From bb5dd248d825940ec6aec7b70eac93c1af51eed1 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 24 Feb 2024 09:04:46 -0600 Subject: [PATCH 001/183] Add compound key repo support --- .../sources/CompleteFullDataSource.java | 3 + .../HighDetailIncompleteFullDataSource.java | 3 + .../LowDetailIncompleteFullDataSource.java | 3 + .../render/ColumnRenderSource.java | 2 +- .../core/file/AbstractDataSourceHandler.java | 3 +- .../core/file/IDataSource.java | 4 +- .../fullDatafile/FullDataFileHandler.java | 2 +- .../GeneratedFullDataFileHandler.java | 2 +- .../core/sql/AbstractDataSourceRepo.java | 13 +- .../core/sql/AbstractDhRepo.java | 45 +++--- .../core/sql/DataSourceDto.java | 20 ++- .../core/sql/DatabaseUpdater.java | 2 +- .../core/sql/FullDataRepo.java | 5 + .../distanthorizons/core/sql/IBaseDTO.java | 5 +- .../core/sql/RenderDataRepo.java | 5 + .../testItems/sql/TestCompoundKeyDto.java | 65 +++++++++ .../testItems/sql/TestCompoundKeyRepo.java | 104 ++++++++++++++ ...tDataRepo.java => TestPrimaryKeyRepo.java} | 21 ++- .../{TestDto.java => TestSingleKeyDto.java} | 12 +- .../src/test/java/tests/DhRepoSqliteTest.java | 133 +++++++++++++----- 20 files changed, 356 insertions(+), 96 deletions(-) create mode 100644 core/src/test/java/testItems/sql/TestCompoundKeyDto.java create mode 100644 core/src/test/java/testItems/sql/TestCompoundKeyRepo.java rename core/src/test/java/testItems/sql/{TestDataRepo.java => TestPrimaryKeyRepo.java} (78%) rename core/src/test/java/testItems/sql/{TestDto.java => TestSingleKeyDto.java} (86%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index c0edc5612..1a1e4f885 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -417,6 +417,9 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu // setters and getters // //=====================// + @Override + public DhSectionPos getKey() { return this.sectionPos; } + @Override public DhSectionPos getSectionPos() { return this.sectionPos; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java index b239a8c18..d1a2959c6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java @@ -447,6 +447,9 @@ public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSo // getters // //=========// + @Override + public DhSectionPos getKey() { return this.sectionPos; } + @Override public DhSectionPos getSectionPos() { return this.sectionPos; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java index a2d188453..e80d4d813 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java @@ -309,6 +309,9 @@ public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor imp // getters and setters // //=====================// + @Override + public DhSectionPos getKey() { return this.sectionPos; } + @Override public DhSectionPos getSectionPos() { return this.sectionPos; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 08ef20f1d..2bc453a88 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -427,7 +427,7 @@ public class ColumnRenderSource implements IDataSource public DhSectionPos getSectionPos() { return this.sectionPos; } @Override - public String getPrimaryKeyString() { return this.sectionPos.serialize(); } + public DhSectionPos getKey() { return this.sectionPos; } public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index 8f2ec3440..30780d27a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -17,7 +17,6 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.channels.ClosedChannelException; -import java.util.ArrayList; import java.util.Enumeration; import java.util.Timer; import java.util.TimerTask; @@ -158,7 +157,7 @@ public abstract class AbstractDataSourceHandler what type of level this data source can be created from */ -public interface IDataSource extends IBaseDTO +public interface IDataSource extends IBaseDTO { DhSectionPos getSectionPos(); - @Override - default String getPrimaryKeyString() { return this.getSectionPos().serialize(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java index 77610a252..96ff63a03 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java @@ -125,7 +125,7 @@ public class FullDataFileHandler while (possibleChildList.size() != 0) { DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1); - if (this.repo.existsWithPrimaryKey(possiblePos.serialize())) + if (this.repo.existsWithKey(possiblePos)) { samplePosList.add(possiblePos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index d9bc2923a..70c26ee5b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -193,7 +193,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler continue; } - if (this.repo.existsWithPrimaryKey(genPos.serialize())) + if (this.repo.existsWithKey(genPos)) { continue; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java index 479da8595..a6d94a053 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java @@ -26,7 +26,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -public abstract class AbstractDataSourceRepo extends AbstractDhRepo +public abstract class AbstractDataSourceRepo extends AbstractDhRepo { public AbstractDataSourceRepo(String databaseType, String databaseLocation) throws SQLException { @@ -35,10 +35,6 @@ public abstract class AbstractDataSourceRepo extends AbstractDhRepo DTO stands for "Data Table Object" + * @param DTO stands for "Data Transfer Object" */ -public abstract class AbstractDhRepo +public abstract class AbstractDhRepo> { public static final int TIMEOUT_SECONDS = 30; private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final ConcurrentHashMap CONNECTIONS_BY_CONNECTION_STRING = new ConcurrentHashMap<>(); - private static final ConcurrentHashMap, String> ACTIVE_CONNECTION_STRINGS_BY_REPO = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap, String> ACTIVE_CONNECTION_STRINGS_BY_REPO = new ConcurrentHashMap<>(); private final String connectionString; private final Connection connection; @@ -107,10 +106,10 @@ public abstract class AbstractDhRepo // high level DB // //===============// - public TDTO get(TDTO dto) { return this.getByPrimaryKey(dto.getPrimaryKeyString()); } - public TDTO getByPrimaryKey(String primaryKey) + public TDTO get(TDTO dto) { return this.getByKey(dto.getKey()); } + public TDTO getByKey(TKey primaryKey) { - Map objectMap = this.queryDictionaryFirst(this.createSelectPrimaryKeySql(primaryKey)); + Map objectMap = this.queryDictionaryFirst(this.createSelectByKeySql(primaryKey)); if (objectMap != null && !objectMap.isEmpty()) { return this.convertDictionaryToDto(objectMap); @@ -124,7 +123,7 @@ public abstract class AbstractDhRepo public void save(TDTO dto) { - if (this.getByPrimaryKey(dto.getPrimaryKeyString()) != null) + if (this.getByKey(dto.getKey()) != null) { this.update(dto); } @@ -141,7 +140,7 @@ public abstract class AbstractDhRepo } catch (DbConnectionClosedException ignored) { - LOGGER.warn("Attempted to insert ["+this.dtoClass.getSimpleName()+"] with primary key ["+(dto != null ? dto.getPrimaryKeyString() : "NULL")+"] on closed repo ["+this.connectionString+"]."); + LOGGER.warn("Attempted to insert ["+this.dtoClass.getSimpleName()+"] with primary key ["+(dto != null ? dto.getKey() : "NULL")+"] on closed repo ["+this.connectionString+"]."); } catch (SQLException e) { @@ -158,7 +157,7 @@ public abstract class AbstractDhRepo } catch (DbConnectionClosedException e) { - LOGGER.warn("Attempted to update ["+this.dtoClass.getSimpleName()+"] with primary key ["+(dto != null ? dto.getPrimaryKeyString() : "NULL")+"] on closed repo ["+this.connectionString+"]."); + LOGGER.warn("Attempted to update ["+this.dtoClass.getSimpleName()+"] with primary key ["+(dto != null ? dto.getKey() : "NULL")+"] on closed repo ["+this.connectionString+"]."); } catch (SQLException e) { @@ -169,10 +168,10 @@ public abstract class AbstractDhRepo } - public void delete(TDTO dto) { this.deleteByPrimaryKey(dto.getPrimaryKeyString()); } - public void deleteByPrimaryKey(String primaryKey) + public void delete(TDTO dto) { this.deleteWithKey(dto.getKey()); } + public void deleteWithKey(TKey key) { - String whereEqualStatement = this.createWherePrimaryKeySql(primaryKey); + String whereEqualStatement = this.createWhereStatement(key); this.queryDictionaryFirst("DELETE FROM "+this.getTableName()+" WHERE "+whereEqualStatement); } @@ -180,10 +179,10 @@ public abstract class AbstractDhRepo public void deleteAll() { this.queryDictionaryFirst("DELETE FROM "+this.getTableName()); } - public boolean exists(TDTO dto) { return this.existsWithPrimaryKey(dto.getPrimaryKeyString()); } - public boolean existsWithPrimaryKey(String primaryKey) + public boolean exists(TDTO dto) { return this.existsWithKey(dto.getKey()); } + public boolean existsWithKey(TKey key) { - String whereEqualStatement = this.createWherePrimaryKeySql(primaryKey); + String whereEqualStatement = this.createWhereStatement(key); Map result = this.queryDictionaryFirst("SELECT EXISTS(SELECT 1 FROM "+this.getTableName()+" WHERE "+whereEqualStatement+") as 'existingCount';"); return result != null && (int)result.get("existingCount") != 0; } @@ -379,10 +378,7 @@ public abstract class AbstractDhRepo // helper methods // //================// - /** Example: Id = '0' */ - public String createWherePrimaryKeySql(TDTO dto) { return this.createWherePrimaryKeySql(dto.getPrimaryKeyString()); } - /** Example: Id = '0' */ - public String createWherePrimaryKeySql(String primaryKeyValue) { return this.getPrimaryKeyName()+" = '"+primaryKeyValue+"'"; } + public String createWhereStatement(TDTO dto) { return this.createWhereStatement(dto.getKey()); } public static List> convertResultSetToDictionaryList(ResultSet resultSet) throws SQLException { @@ -440,12 +436,17 @@ public abstract class AbstractDhRepo //==================// public abstract String getTableName(); - public abstract String getPrimaryKeyName(); @Nullable public abstract TDTO convertDictionaryToDto(Map objectMap) throws ClassCastException; - public abstract String createSelectPrimaryKeySql(String primaryKey); + public String createSelectByKeySql(TKey key) { return "SELECT * FROM "+this.getTableName()+" WHERE "+this.createWhereStatement(key); } + /** + * Example: + * Id = '0' + * ColOne = '0' AND ColTwo = '2' + */ + public abstract String createWhereStatement(TKey key); public abstract PreparedStatement createInsertStatement(TDTO dto) throws SQLException; public abstract PreparedStatement createUpdateStatement(TDTO dto) throws SQLException; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java index 8ad1ef558..a0782c41b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java @@ -31,7 +31,7 @@ import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; /** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */ -public class DataSourceDto implements IBaseDTO +public class DataSourceDto implements IBaseDTO { public DhSectionPos pos; public int checksum; @@ -49,6 +49,11 @@ public class DataSourceDto implements IBaseDTO public final byte[] dataArray; + + //=============// + // constructor // + //=============// + public DataSourceDto(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) { this.pos = pos; @@ -63,9 +68,6 @@ public class DataSourceDto implements IBaseDTO } - @Override - public String getPrimaryKeyString() { return this.pos.serialize(); } - /** @return a stream for the data contained in this DTO. */ public DhDataInputStream getInputStream() throws IOException { @@ -74,4 +76,14 @@ public class DataSourceDto implements IBaseDTO return compressedStream; } + + + //===========// + // overrides // + //===========// + + @Override + public DhSectionPos getKey() { return this.pos; } + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index d41c3cdc1..d71eda424 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -53,7 +53,7 @@ public class DatabaseUpdater // script running // //================// - public static void runAutoUpdateScripts(AbstractDhRepo repo) throws SQLException + public static > void runAutoUpdateScripts(AbstractDhRepo repo) throws SQLException { // get the resource scripts ArrayList scriptList; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java index 7d6e3f633..2ebd85c2a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/FullDataRepo.java @@ -19,6 +19,8 @@ package com.seibel.distanthorizons.core.sql; +import com.seibel.distanthorizons.core.pos.DhSectionPos; + import java.sql.SQLException; public class FullDataRepo extends AbstractDataSourceRepo @@ -35,4 +37,7 @@ public class FullDataRepo extends AbstractDataSourceRepo @Override public String getTableName() { return TABLE_NAME; } + @Override + public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java index e7c1f99af..a692a5b08 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java @@ -23,8 +23,9 @@ package com.seibel.distanthorizons.core.sql; * DTO = DataTable Object
* Any object that's stored in the database should extend this object. */ -public interface IBaseDTO +public interface IBaseDTO { - String getPrimaryKeyString(); + TKey getKey(); + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java index 8cbec7dfe..7e2aee31e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java @@ -19,6 +19,8 @@ package com.seibel.distanthorizons.core.sql; +import com.seibel.distanthorizons.core.pos.DhSectionPos; + import java.sql.SQLException; public class RenderDataRepo extends AbstractDataSourceRepo @@ -35,4 +37,7 @@ public class RenderDataRepo extends AbstractDataSourceRepo @Override public String getTableName() { return TABLE_NAME; } + @Override + public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } + } diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java new file mode 100644 index 000000000..fbabbe52e --- /dev/null +++ b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java @@ -0,0 +1,65 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package testItems.sql; + +import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.IBaseDTO; + +public class TestCompoundKeyDto implements IBaseDTO +{ + public DhChunkPos id; + public String value; + + + + public TestCompoundKeyDto(DhChunkPos id, String value) + { + this.id = id; + this.value = value; + } + + @Override + public DhChunkPos getKey() { return this.id; } + + + @Override + public boolean equals(Object other) + { + if (other.getClass() != this.getClass()) + { + return false; + } + else + { + TestCompoundKeyDto otherDto = (TestCompoundKeyDto) other; + + return otherDto.id.equals(this.id) + && otherDto.value.equals(this.value); + } + } + + @Override + public String toString() + { + return this.id + ", " + this.value; + } + +} diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java new file mode 100644 index 000000000..114e4b57d --- /dev/null +++ b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java @@ -0,0 +1,104 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package testItems.sql; + +import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.sql.AbstractDhRepo; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Map; + +public class TestCompoundKeyRepo extends AbstractDhRepo +{ + + public TestCompoundKeyRepo(String databaseType, String databaseLocation) throws SQLException + { + super(databaseType, databaseLocation, TestCompoundKeyDto.class); + + // note: this should only ever be done with the test repo. + // All long term tables should be created using a sql Script. + String createTableSql = + "CREATE TABLE IF NOT EXISTS "+this.getTableName()+"(\n" + + "XPos INT NOT NULL\n" + + ",ZPos INT NOT NULL\n" + + "\n" + + ",Value TEXT NULL\n" + + "\n" + + ",PRIMARY KEY (XPos, ZPos)" + + ");"; + this.queryDictionaryFirst(createTableSql); + } + + + @Override + public String getTableName() { return "TestCompound"; } + @Override + public String createWhereStatement(DhChunkPos key) { return "XPos = '"+key.x+"' AND ZPos = '"+key.z+"'"; } + + + @Override + public TestCompoundKeyDto convertDictionaryToDto(Map objectMap) throws ClassCastException + { + int xPos = (int) objectMap.get("XPos"); + int zPos = (int) objectMap.get("ZPos"); + String value = (String) objectMap.get("Value"); + + return new TestCompoundKeyDto(new DhChunkPos(xPos, zPos), value); + } + + @Override + public PreparedStatement createInsertStatement(TestCompoundKeyDto dto) throws SQLException + { + String sql = + "INSERT INTO "+this.getTableName()+" \n" + + "(XPos, ZPos, Value) \n" + + "VALUES(?,?,?);"; + PreparedStatement statement = this.createPreparedStatement(sql); + + int i = 1; // post-increment for the win! + statement.setObject(i++, dto.id.x); + statement.setObject(i++, dto.id.z); + + statement.setObject(i++, dto.value); + + return statement; + } + + @Override + public PreparedStatement createUpdateStatement(TestCompoundKeyDto dto) throws SQLException + { + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET \n" + + " Value = ? \n" + + "WHERE XPos = ? AND ZPos = ?"; + PreparedStatement statement = this.createPreparedStatement(sql); + + int i = 1; + statement.setObject(i++, dto.value); + + statement.setObject(i++, dto.id.x); + statement.setObject(i++, dto.id.z); + + return statement; + } + +} diff --git a/core/src/test/java/testItems/sql/TestDataRepo.java b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java similarity index 78% rename from core/src/test/java/testItems/sql/TestDataRepo.java rename to core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java index 38bb3c909..748c3a214 100644 --- a/core/src/test/java/testItems/sql/TestDataRepo.java +++ b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java @@ -25,12 +25,12 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -public class TestDataRepo extends AbstractDhRepo +public class TestPrimaryKeyRepo extends AbstractDhRepo { - public TestDataRepo(String databaseType, String databaseLocation) throws SQLException + public TestPrimaryKeyRepo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, TestDto.class); + super(databaseType, databaseLocation, TestSingleKeyDto.class); // note: this should only ever be done with the test repo. // All long term tables should be created using a sql Script. @@ -49,26 +49,23 @@ public class TestDataRepo extends AbstractDhRepo @Override public String getTableName() { return "Test"; } - @Override - public String getPrimaryKeyName() { return "Id"; } + @Override + public String createWhereStatement(Integer keyString) { return "Id = '"+keyString+"'"; } @Override - public TestDto convertDictionaryToDto(Map objectMap) throws ClassCastException + public TestSingleKeyDto convertDictionaryToDto(Map objectMap) throws ClassCastException { int id = (int) objectMap.get("Id"); String value = (String) objectMap.get("Value"); long longValue = (Long) objectMap.get("LongValue"); byte byteValue = (Byte) objectMap.get("ByteValue"); - return new TestDto(id, value, longValue, byteValue); + return new TestSingleKeyDto(id, value, longValue, byteValue); } - @Override - public String createSelectPrimaryKeySql(String primaryKey) { return "SELECT * FROM "+this.getTableName()+" WHERE Id = '"+primaryKey+"'"; } - @Override - public PreparedStatement createInsertStatement(TestDto dto) throws SQLException + public PreparedStatement createInsertStatement(TestSingleKeyDto dto) throws SQLException { String sql = "INSERT INTO "+this.getTableName()+" \n" + @@ -87,7 +84,7 @@ public class TestDataRepo extends AbstractDhRepo } @Override - public PreparedStatement createUpdateStatement(TestDto dto) throws SQLException + public PreparedStatement createUpdateStatement(TestSingleKeyDto dto) throws SQLException { String sql = "UPDATE "+this.getTableName()+" \n" + diff --git a/core/src/test/java/testItems/sql/TestDto.java b/core/src/test/java/testItems/sql/TestSingleKeyDto.java similarity index 86% rename from core/src/test/java/testItems/sql/TestDto.java rename to core/src/test/java/testItems/sql/TestSingleKeyDto.java index 488184eb5..6de1352f3 100644 --- a/core/src/test/java/testItems/sql/TestDto.java +++ b/core/src/test/java/testItems/sql/TestSingleKeyDto.java @@ -20,16 +20,18 @@ package testItems.sql; import com.seibel.distanthorizons.core.sql.IBaseDTO; -import org.junit.Assert; -public class TestDto implements IBaseDTO +public class TestSingleKeyDto implements IBaseDTO { public int id; public String value; public long longValue; public byte byteValue; - public TestDto(int id, String value, long longValue, byte byteValue) + + + + public TestSingleKeyDto(int id, String value, long longValue, byte byteValue) { this.id = id; this.value = value; @@ -38,7 +40,7 @@ public class TestDto implements IBaseDTO } @Override - public String getPrimaryKeyString() { return this.id+""; } + public Integer getKey() { return this.id; } @Override @@ -50,7 +52,7 @@ public class TestDto implements IBaseDTO } else { - TestDto otherDto = (TestDto) other; + TestSingleKeyDto otherDto = (TestSingleKeyDto) other; return otherDto.id == this.id && otherDto.value.equals(this.value) diff --git a/core/src/test/java/tests/DhRepoSqliteTest.java b/core/src/test/java/tests/DhRepoSqliteTest.java index 0a2030a8a..8d15c5c6a 100644 --- a/core/src/test/java/tests/DhRepoSqliteTest.java +++ b/core/src/test/java/tests/DhRepoSqliteTest.java @@ -19,11 +19,15 @@ package tests; +import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.sql.DatabaseUpdater; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; -import testItems.sql.TestDataRepo; -import testItems.sql.TestDto; +import testItems.sql.TestCompoundKeyRepo; +import testItems.sql.TestCompoundKeyDto; +import testItems.sql.TestPrimaryKeyRepo; +import testItems.sql.TestSingleKeyDto; import java.io.File; import java.sql.SQLException; @@ -35,28 +39,29 @@ import java.util.Map; public class DhRepoSqliteTest { public static String DATABASE_TYPE = "jdbc:sqlite"; + public static String DB_FILE_NAME = "test.sqlite"; - @Test - public void testFileSqlite() + + @BeforeClass + public static void testSetup() { - String dbFileName = "test.sqlite"; - - - File dbFile = new File(dbFileName); + File dbFile = new File(DB_FILE_NAME); if (dbFile.exists()) { Assert.assertTrue("unable to delete old test DB File.", dbFile.delete()); } - - - TestDataRepo testDataRepo = null; + } + + + + @Test + public void testPrimaryKeyRepo() + { + TestPrimaryKeyRepo primaryKeyRepo = null; try { - testDataRepo = new TestDataRepo(DATABASE_TYPE, dbFileName); - - dbFile = new File(dbFileName); - Assert.assertTrue("dbFile not created", dbFile.exists()); + primaryKeyRepo = new TestPrimaryKeyRepo(DATABASE_TYPE, DB_FILE_NAME); @@ -65,15 +70,15 @@ public class DhRepoSqliteTest //==========================// // check that the schema table is created - Map autoUpdateTablePresentResult = testDataRepo.queryDictionaryFirst("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';"); + Map autoUpdateTablePresentResult = primaryKeyRepo.queryDictionaryFirst("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';"); if (autoUpdateTablePresentResult == null || autoUpdateTablePresentResult.get("name") == null) { Assert.fail("Auto DB update table missing."); } // check that the update scripts aren't run multiple times - TestDataRepo altDataRepoOne = new TestDataRepo(DATABASE_TYPE, dbFileName); - TestDataRepo altDataRepoTwo = new TestDataRepo(DATABASE_TYPE, dbFileName); + TestPrimaryKeyRepo altDataRepoOne = new TestPrimaryKeyRepo(DATABASE_TYPE, DB_FILE_NAME); + TestPrimaryKeyRepo altDataRepoTwo = new TestPrimaryKeyRepo(DATABASE_TYPE, DB_FILE_NAME); @@ -82,39 +87,39 @@ public class DhRepoSqliteTest //===========// // insert - TestDto insertDto = new TestDto(0, "a", 0L, (byte) 0); - testDataRepo.save(insertDto); + TestSingleKeyDto insertDto = new TestSingleKeyDto(0, "a", 0L, (byte) 0); + primaryKeyRepo.save(insertDto); // get - TestDto getDto = testDataRepo.getByPrimaryKey("0"); + TestSingleKeyDto getDto = primaryKeyRepo.getByKey(0); Assert.assertNotNull("get failed, null returned", getDto); Assert.assertEquals("get/insert failed, not equal", insertDto, getDto); // exists - DTO present - Assert.assertTrue("DTO exists failed", testDataRepo.exists(insertDto)); - Assert.assertTrue("DTO exists failed", testDataRepo.existsWithPrimaryKey(insertDto.getPrimaryKeyString())); + Assert.assertTrue("DTO exists failed", primaryKeyRepo.exists(insertDto)); + Assert.assertTrue("DTO exists failed", primaryKeyRepo.existsWithKey(insertDto.getKey())); // update - TestDto updateMetaFile = new TestDto(0, "b", Long.MAX_VALUE, Byte.MAX_VALUE); - testDataRepo.save(updateMetaFile); + TestSingleKeyDto updateMetaFile = new TestSingleKeyDto(0, "b", Long.MAX_VALUE, Byte.MAX_VALUE); + primaryKeyRepo.save(updateMetaFile); // get - getDto = testDataRepo.getByPrimaryKey("0"); + getDto = primaryKeyRepo.getByKey(0); Assert.assertNotNull("get failed, null returned", getDto); Assert.assertEquals("get/insert failed, not equal", updateMetaFile, getDto); // delete - testDataRepo.delete(updateMetaFile); + primaryKeyRepo.delete(updateMetaFile); // get - getDto = testDataRepo.getByPrimaryKey("0"); + getDto = primaryKeyRepo.getByKey(0); Assert.assertNull("delete failed, not null returned", getDto); // exists - DTO absent - Assert.assertFalse("DTO exists failed", testDataRepo.exists(insertDto)); - Assert.assertFalse("DTO exists failed", testDataRepo.existsWithPrimaryKey(insertDto.getPrimaryKeyString())); + Assert.assertFalse("DTO exists failed", primaryKeyRepo.exists(insertDto)); + Assert.assertFalse("DTO exists failed", primaryKeyRepo.existsWithKey(insertDto.getKey())); } catch (SQLException e) @@ -123,11 +128,75 @@ public class DhRepoSqliteTest } finally { - if (testDataRepo != null) + if (primaryKeyRepo != null) { - testDataRepo.close(); + primaryKeyRepo.close(); } } } + @Test + public void testCompoundKeyRepo() + { + TestCompoundKeyRepo compoundKeyRepo = null; + try + { + compoundKeyRepo = new TestCompoundKeyRepo(DATABASE_TYPE, DB_FILE_NAME); + + + + //===========// + // DTO tests // + //===========// + + // insert + TestCompoundKeyDto insertDto = new TestCompoundKeyDto(new DhChunkPos(1, 2), "a"); + compoundKeyRepo.save(insertDto); + + // get + TestCompoundKeyDto getDto = compoundKeyRepo.getByKey(new DhChunkPos(1, 2)); + Assert.assertNotNull("get failed, null returned", getDto); + Assert.assertEquals("get/insert failed, not equal", insertDto, getDto); + + // exists - DTO present + Assert.assertTrue("DTO exists failed", compoundKeyRepo.exists(insertDto)); + Assert.assertTrue("DTO exists failed", compoundKeyRepo.existsWithKey(insertDto.getKey())); + + + // update + TestCompoundKeyDto updateMetaFile = new TestCompoundKeyDto(new DhChunkPos(1, 2), "b"); + compoundKeyRepo.save(updateMetaFile); + + // get + getDto = compoundKeyRepo.getByKey(new DhChunkPos(1, 2)); + Assert.assertNotNull("get failed, null returned", getDto); + Assert.assertEquals("get/insert failed, not equal", updateMetaFile, getDto); + + + // delete + compoundKeyRepo.delete(updateMetaFile); + + // get + getDto = compoundKeyRepo.getByKey(new DhChunkPos(1, 2)); + Assert.assertNull("delete failed, not null returned", getDto); + + // exists - DTO absent + Assert.assertFalse("DTO exists failed", compoundKeyRepo.exists(insertDto)); + Assert.assertFalse("DTO exists failed", compoundKeyRepo.existsWithKey(insertDto.getKey())); + + } + catch (SQLException e) + { + Assert.fail(e.getMessage()); + } + finally + { + if (compoundKeyRepo != null) + { + compoundKeyRepo.close(); + } + } + } + + } From 28e230a2db05b18c37e0cad9144b3f4556325e00 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 25 Feb 2024 08:14:25 -0600 Subject: [PATCH 002/183] Fix DhApiScreenResizeEvent incorrect description --- .../methods/events/abstractEvents/DhApiScreenResizeEvent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java index 387981aad..64407bb5c 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java @@ -32,7 +32,7 @@ import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhAp */ public abstract class DhApiScreenResizeEvent implements IDhApiEvent { - /** Fired immediately before Distant Horizons renders any transparent buffers. */ + /** Fired immediately before Distant Horizons handles the screen resize. */ public abstract void onResize(DhApiEventParam event); From 113c0f227f47ddc660b16a47af48ba259b4a5fda Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 08:05:54 -0600 Subject: [PATCH 003/183] Improve debugging for FullDataPointUtil.remap() --- .../core/util/FullDataPointUtil.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index c02f567af..606af3eb6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -44,10 +44,10 @@ import org.jetbrains.annotations.Contract; * YY YY YY YY YY YY YY YY
* YY YY YY YY DP DP DP DP
* DP DP DP DP DP DP DP DP
- * ID ID ID ID ID ID IO ID
- * ID ID ID ID ID ID IO ID
- * ID ID ID ID ID ID IO ID
- * ID ID ID ID ID ID IO ID <-- Bottom bits
+ * ID ID ID ID ID ID ID ID
+ * ID ID ID ID ID ID ID ID
+ * ID ID ID ID ID ID ID ID
+ * ID ID ID ID ID ID ID ID <-- Bottom bits
* * * @see RenderDataPointUtil @@ -106,6 +106,20 @@ public class FullDataPointUtil /** Remaps the biome/blockState ID of the given datapoint */ @Contract(pure = true) - public static long remap(int[] newIdMapping, long data) { return (data & INVERSE_ID_MASK) | newIdMapping[(int) data]; } + public static long remap(int[] newIdMapping, long data) throws IndexOutOfBoundsException + { + int currentId = getId(data); + try + { + int newId = newIdMapping[currentId]; + return (data & INVERSE_ID_MASK) | newId; + } + catch (IndexOutOfBoundsException e) + { + // this try-catch is present to fix an issue where the stack trace is missing + // and to allow for easily attaching a debugger + throw new RuntimeException(e); + } + } } From 9b93125936bc83d4c6a456cc60aa3bebfa2c4f41 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 11:44:50 -0600 Subject: [PATCH 004/183] Add first draft of the new full data system --- .../distanthorizons/core/Initializer.java | 7 - .../methods/data/DhApiTerrainDataRepo.java | 6 +- .../fullData/FullDataDownSampler.java | 159 ----- .../fullData/FullDataPointIdMap.java | 16 +- .../accessor/ChunkSizedFullDataAccessor.java | 79 --- .../accessor/FullDataArrayAccessor.java | 3 +- .../fullData/accessor/IFullDataAccessor.java | 6 +- .../loader/AbstractFullDataSourceLoader.java | 209 ------- .../loader/CompleteFullDataSourceLoader.java | 33 - ...hDetailIncompleteFullDataSourceLoader.java | 33 - ...wDetailIncompleteFullDataSourceLoader.java | 33 - .../sources/CompleteFullDataSource.java | 326 ++++------ .../HighDetailIncompleteFullDataSource.java | 586 ------------------ .../LowDetailIncompleteFullDataSource.java | 439 ------------- .../fullData/sources/NewFullDataSource.java | 447 +++++++++++++ .../sources/interfaces/IFullDataSource.java | 107 ---- .../interfaces/IIncompleteFullDataSource.java | 86 --- .../interfaces/IStreamableFullDataSource.java | 161 ----- .../render/ColumnRenderSource.java | 147 ++--- .../render/ColumnRenderSourceLoader.java | 6 +- .../transformers/ChunkToLodBuilder.java | 16 +- .../FullDataToRenderDataTransformer.java | 81 +-- .../transformers/LodDataBuilder.java | 132 +++- ...a => AbstractLegacyDataSourceHandler.java} | 47 +- .../file/AbstractNewDataSourceHandler.java | 296 +++++++++ .../core/file/IDataSource.java | 35 +- .../core/file/ISourceProvider.java | 12 +- .../fullDatafile/FullDataFileHandler.java | 233 ------- .../fullDatafile/IFullDataSourceProvider.java | 25 +- .../LegacyFullDataFileHandler.java | 134 ++++ .../fullDatafile/NewFullDataFileHandler.java | 276 +++++++++ ...a => NewGeneratedFullDataFileHandler.java} | 161 +++-- ...java => NewRemoteFullDataFileHandler.java} | 6 +- .../renderfile/IRenderSourceProvider.java | 8 +- .../renderfile/RenderSourceFileHandler.java | 44 +- .../SubDimensionLevelMatcher.java | 18 +- .../generation/IWorldGenerationQueue.java | 7 + .../MissingWorldGenPositionFinder.java | 213 ------- .../core/generation/WorldGenerationQueue.java | 38 +- .../tasks/IWorldGenTaskTracker.java | 4 +- .../generation/tasks/WorldGenTaskGroup.java | 8 +- .../core/level/AbstractDhLevel.java | 28 +- .../core/level/ClientLevelModule.java | 50 +- .../core/level/DhClientLevel.java | 16 +- .../core/level/DhClientServerLevel.java | 10 +- .../core/level/DhServerLevel.java | 10 +- .../distanthorizons/core/level/IDhLevel.java | 10 +- .../core/level/IDhWorldGenLevel.java | 4 +- .../core/level/ServerLevelModule.java | 7 +- .../core/level/WorldGenModule.java | 12 +- .../distanthorizons/core/pos/DhLodPos.java | 5 +- .../core/pos/DhSectionPos.java | 16 +- .../core/render/LodQuadTree.java | 5 +- .../core/sql/DatabaseUpdater.java | 2 + .../core/sql/{ => dto}/IBaseDTO.java | 2 +- .../LegacyDataSourceDTO.java} | 9 +- .../core/sql/dto/NewFullDataSourceDTO.java | 256 ++++++++ .../core/sql/{ => repo}/AbstractDhRepo.java | 5 +- .../AbstractLegacyDataSourceRepo.java} | 17 +- .../core/sql/{ => repo}/FullDataRepo.java | 4 +- .../core/sql/repo/NewFullDataSourceRepo.java | 214 +++++++ .../core/sql/{ => repo}/RenderDataRepo.java | 4 +- .../core/util/FullDataPointUtil.java | 3 + ...te-createGeneratedFullDataSourceTables.sql | 25 + .../main/resources/sqlScripts/scriptList.txt | 1 + .../testItems/sql/TestCompoundKeyDto.java | 3 +- .../testItems/sql/TestCompoundKeyRepo.java | 2 +- .../testItems/sql/TestPrimaryKeyRepo.java | 2 +- .../java/testItems/sql/TestSingleKeyDto.java | 2 +- .../src/test/java/tests/DhRepoSqliteTest.java | 3 +- 70 files changed, 2342 insertions(+), 3068 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java rename core/src/main/java/com/seibel/distanthorizons/core/file/{AbstractDataSourceHandler.java => AbstractLegacyDataSourceHandler.java} (85%) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{GeneratedFullDataFileHandler.java => NewGeneratedFullDataFileHandler.java} (59%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{RemoteFullDataFileHandler.java => NewRemoteFullDataFileHandler.java} (74%) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java rename core/src/main/java/com/seibel/distanthorizons/core/sql/{ => dto}/IBaseDTO.java (95%) rename core/src/main/java/com/seibel/distanthorizons/core/sql/{DataSourceDto.java => dto/LegacyDataSourceDTO.java} (83%) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java rename core/src/main/java/com/seibel/distanthorizons/core/sql/{ => repo}/AbstractDhRepo.java (98%) rename core/src/main/java/com/seibel/distanthorizons/core/sql/{AbstractDataSourceRepo.java => repo/AbstractLegacyDataSourceRepo.java} (85%) rename core/src/main/java/com/seibel/distanthorizons/core/sql/{ => repo}/FullDataRepo.java (91%) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java rename core/src/main/java/com/seibel/distanthorizons/core/sql/{ => repo}/RenderDataRepo.java (91%) create mode 100644 core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index e74d7bedd..0684e4cc3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -26,10 +26,7 @@ import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.world.DhApiWorldProxy; import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig; import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.CompleteFullDataSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.HighDetailIncompleteFullDataSourceLoader; import com.seibel.distanthorizons.api.DhApi; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.LowDetailIncompleteFullDataSourceLoader; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; //import io.netty.buffer.ByteBuf; import net.jpountz.lz4.LZ4Compressor; @@ -77,10 +74,6 @@ public class Initializer - CompleteFullDataSourceLoader unused2 = new CompleteFullDataSourceLoader(); // Auto register into the loader system - HighDetailIncompleteFullDataSourceLoader unused3 = new HighDetailIncompleteFullDataSourceLoader(); // Auto register - LowDetailIncompleteFullDataSourceLoader unused4 = new LowDetailIncompleteFullDataSourceLoader(); // Auto register - // link Core's config to the API DhApi.Delayed.configs = DhApiConfig.INSTANCE; DhApi.Delayed.terrainRepo = DhApiTerrainDataRepo.INSTANCE; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index e2d2c8d66..0cc87f208 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -28,7 +28,7 @@ import com.seibel.distanthorizons.api.objects.math.DhApiVec3i; import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhLodPos; @@ -212,7 +212,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo try { // attempt to get/generate the data source for this section - IFullDataSource dataSource = level.getFileHandler().getAsync(sectionPos).get(); + NewFullDataSource dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + sectionPos + "]."); @@ -221,7 +221,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.getMapping(); - SingleColumnFullDataAccessor dataColumn = dataSource.tryGet(relativePos.x, relativePos.z); + SingleColumnFullDataAccessor dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { int dataColumnIndexCount = dataColumn.getSingleLength(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java deleted file mode 100644 index ac8862668..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataDownSampler.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData; - -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.LodUtil; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.concurrent.CompletableFuture; - -public class FullDataDownSampler -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - public static CompletableFuture createDownSamplingFuture(DhSectionPos newTarget, IFullDataSourceProvider provider) - { - // TODO: Make this future somehow run with lowest priority (to ensure ram usage stays low) - return createDownSamplingFuture(CompleteFullDataSource.createEmpty(newTarget), provider); - } - - public static CompletableFuture createDownSamplingFuture(CompleteFullDataSource target, IFullDataSourceProvider provider) - { - int sectionSizeNeeded = 1 << target.getDataDetailLevel(); - - ArrayList> futures; - DhLodPos basePos = target.getSectionPos().getSectionBBoxPos().getCornerLodPos(CompleteFullDataSource.SECTION_SIZE_OFFSET); - - - if (sectionSizeNeeded <= CompleteFullDataSource.SECTION_SIZE_OFFSET) - { - futures = new ArrayList<>(sectionSizeNeeded * sectionSizeNeeded); - for (int xOffset = 0; xOffset < sectionSizeNeeded; xOffset++) - { - for (int zOffset = 0; zOffset < sectionSizeNeeded; zOffset++) - { - CompletableFuture future = provider.getAsync(new DhSectionPos( - CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset, basePos.z + zOffset)); - future = future.whenComplete((source, ex) -> { - if (ex == null && source != null && source instanceof CompleteFullDataSource) - { - downSample(target, (CompleteFullDataSource) source); - } - else if (ex != null) - { - LOGGER.error("Error while down sampling", ex); - } - }); - futures.add(future); - } - } - } - else - { - futures = new ArrayList<>(CompleteFullDataSource.WIDTH * CompleteFullDataSource.WIDTH); - int multiplier = sectionSizeNeeded / CompleteFullDataSource.WIDTH; - for (int xOffset = 0; xOffset < CompleteFullDataSource.WIDTH; xOffset++) - { - for (int zOffset = 0; zOffset < CompleteFullDataSource.WIDTH; zOffset++) - { - CompletableFuture future = provider.getAsync(new DhSectionPos( - CompleteFullDataSource.SECTION_SIZE_OFFSET, basePos.x + xOffset * multiplier, basePos.z + zOffset * multiplier)); - future = future.whenComplete((source, ex) -> { - if (ex == null && source != null && source instanceof CompleteFullDataSource) - { - downSample(target, (CompleteFullDataSource) source); - } - else if (ex != null) - { - LOGGER.error("Error while down sampling", ex); - } - }); - futures.add(future); - } - } - } - return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).thenApply(v -> target); - } - - public static void downSample(CompleteFullDataSource target, CompleteFullDataSource source) - { - LodUtil.assertTrue(target.getSectionPos().overlapsExactly(source.getSectionPos())); - LodUtil.assertTrue(target.getDataDetailLevel() > source.getDataDetailLevel()); - - byte detailDiff = (byte) (target.getDataDetailLevel() - source.getDataDetailLevel()); - DhSectionPos trgPos = target.getSectionPos(); - DhSectionPos srcPos = source.getSectionPos(); - - if (detailDiff >= CompleteFullDataSource.SECTION_SIZE_OFFSET) - { - // The source occupies only 1 datapoint in the target - // FIXME: TEMP method for down-sampling: take only the corner column - int sourceSectionPerTargetData = 1 << (detailDiff - CompleteFullDataSource.SECTION_SIZE_OFFSET); - if (srcPos.getX() % sourceSectionPerTargetData != 0 || srcPos.getZ() % sourceSectionPerTargetData != 0) - { - return; - } - DhLodPos trgOffset = trgPos.getMinCornerLodPos(target.getDataDetailLevel()); - DhLodPos srcOffset = srcPos.getSectionBBoxPos().convertToDetailLevel(target.getDataDetailLevel()); - int offsetX = trgOffset.x - srcOffset.x; - int offsetZ = trgOffset.z - srcOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < CompleteFullDataSource.WIDTH - && offsetZ >= 0 && offsetZ < CompleteFullDataSource.WIDTH); - target.markNotEmpty(); - source.get(0, 0).deepCopyTo(target.get(offsetX, offsetZ)); - - } - else if (detailDiff > 0) - { - // The source occupies multiple data-points in the target - int srcDataPerTrgData = 1 << detailDiff; - int overlappedTrgDataSize = CompleteFullDataSource.WIDTH / srcDataPerTrgData; - - DhLodPos trgOffset = trgPos.getMinCornerLodPos(target.getDataDetailLevel()); - DhLodPos srcOffset = srcPos.getSectionBBoxPos().getCornerLodPos(target.getDataDetailLevel()); - int offsetX = trgOffset.x - srcOffset.x; - int offsetZ = trgOffset.z - srcOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < CompleteFullDataSource.WIDTH - && offsetZ >= 0 && offsetZ < CompleteFullDataSource.WIDTH); - target.markNotEmpty(); - - for (int ox = 0; ox < overlappedTrgDataSize; ox++) - { - for (int oz = 0; oz < overlappedTrgDataSize; oz++) - { - SingleColumnFullDataAccessor column = target.get(ox + offsetX, oz + offsetZ); - column.downsampleFrom(source.subView(srcDataPerTrgData, ox * srcDataPerTrgData, oz * srcDataPerTrgData)); - } - } - } - else - { - LodUtil.assertNotReach(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index b276882b0..a0d391554 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -165,19 +165,23 @@ public class FullDataPointIdMap /** * Adds each entry from the given map to this map. + * + * Note: when using this function be careful about re-mapping the + * same data source multiple times. + * Doing so may cause indexOutOfBounds issues. * * @return an array of each added entry's ID in this map in order */ - public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap target) + public int[] mergeAndReturnRemappedEntityIds(FullDataPointIdMap inputMap) { try { - LOGGER.trace("merging {" + this.pos + ", " + this.entryList.size() + "} and {" + target.pos + ", " + target.entryList.size() + "}"); + LOGGER.trace("merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); - target.readWriteLock.readLock().lock(); + inputMap.readWriteLock.readLock().lock(); this.readWriteLock.writeLock().lock(); - ArrayList entriesToMerge = target.entryList; + ArrayList entriesToMerge = inputMap.entryList; int[] remappedEntryIds = new int[entriesToMerge.size()]; for (int i = 0; i < entriesToMerge.size(); i++) { @@ -191,9 +195,9 @@ public class FullDataPointIdMap finally { this.readWriteLock.writeLock().unlock(); - target.readWriteLock.readLock().unlock(); + inputMap.readWriteLock.readLock().unlock(); - LOGGER.trace("finished merging {" + this.pos + ", " + this.entryList.size() + "} and {" + target.pos + ", " + target.entryList.size() + "}"); + LOGGER.trace("finished merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java deleted file mode 100644 index dcc02e536..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/ChunkSizedFullDataAccessor.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; - -import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; - -/** - * A more specific version of {@link FullDataArrayAccessor} - * that only contains full data for a single chunk. - * - * @see FullDataPointUtil - */ -public class ChunkSizedFullDataAccessor extends FullDataArrayAccessor -{ - public final DhChunkPos chunkPos; - public final DhSectionPos sectionPos; - - // TODO replace this var with LodUtil.BLOCK_DETAIL_LEVEL - public final byte detailLevel = LodUtil.BLOCK_DETAIL_LEVEL; - - - - public ChunkSizedFullDataAccessor(DhChunkPos chunkPos) - { - super(new FullDataPointIdMap(new DhSectionPos(chunkPos)), - new long[LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH][0], - LodUtil.CHUNK_WIDTH); - - this.chunkPos = chunkPos; - // TODO the fact this is using a LodUtil detail level instead of the DhSectionPos detail level may cause confusion and trouble down the line - this.sectionPos = new DhSectionPos(LodUtil.CHUNK_DETAIL_LEVEL, this.chunkPos.x, this.chunkPos.z); - } - - - - public void setSingleColumn(long[] data, int xRelative, int zRelative) { this.dataArrays[xRelative * LodUtil.CHUNK_WIDTH + zRelative] = data; } - - public long nonEmptyCount() - { - long count = 0; - for (long[] data : this.dataArrays) - { - if (data.length != 0) - { - count += 1; - } - } - return count; - } - - public long emptyCount() { return (LodUtil.CHUNK_WIDTH * LodUtil.CHUNK_WIDTH) - this.nonEmptyCount(); } - - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public String toString() { return this.chunkPos + " " + this.nonEmptyCount(); } - -} \ No newline at end of file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java index db8eec620..123a29653 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java @@ -23,15 +23,14 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; /** * Contains Full Data points and basic methods for getting and setting them.
* Can be used standalone or as the base for Full data sources. * * @see CompleteFullDataSource - * @see LowDetailIncompleteFullDataSource */ +@Deprecated public class FullDataArrayAccessor implements IFullDataAccessor { protected final FullDataPointIdMap mapping; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java index 6f45e1707..62cd58cc4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java @@ -19,8 +19,8 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.FullDataPointUtil; @@ -28,12 +28,12 @@ import java.util.Iterator; /** * Contains raw full data points, which must be interpreted by the {@link FullDataPointUtil}.
- * Often used by {@link IFullDataSource}'s. + * Often used by {@link CompleteFullDataSource}'s. * - * @see IFullDataSource * @see FullDataArrayAccessor * @see FullDataPointUtil */ +@Deprecated public interface IFullDataAccessor { FullDataPointIdMap getMapping(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java deleted file mode 100644 index 406d64b97..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/AbstractFullDataSourceLoader.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.google.common.collect.HashMultimap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.locks.ReentrantLock; - -public abstract class AbstractFullDataSourceLoader -{ - public static final HashMultimap, AbstractFullDataSourceLoader> LOADER_REGISTRY = HashMultimap.create(); - public static final HashMap> DATATYPE_REGISTRY = new HashMap<>(); - - - public final Class fullDataSourceClass; - - public final String datatype; - public final byte[] loaderSupportedVersions; - - /** used when pooling data sources */ - private final ArrayList cachedSources = new ArrayList<>(); - private final ReentrantLock cacheLock = new ReentrantLock(); - - - - //=============// - // constructor // - //=============// - - public AbstractFullDataSourceLoader(Class fullDataSourceClass, String datatype, byte[] loaderSupportedVersions) - { - this.datatype = datatype; - this.loaderSupportedVersions = loaderSupportedVersions; - Arrays.sort(loaderSupportedVersions); // sort to allow fast access - this.fullDataSourceClass = fullDataSourceClass; - - if (DATATYPE_REGISTRY.containsKey(datatype) && DATATYPE_REGISTRY.get(datatype) != fullDataSourceClass) - { - throw new IllegalArgumentException("Loader for datatype: [" + datatype + "] already registered with different class: " - + DATATYPE_REGISTRY.get(datatype) + " != " + fullDataSourceClass); - } - - Set loaders = LOADER_REGISTRY.get(fullDataSourceClass); - if (loaders.stream().anyMatch(other -> - { - // see if any loaderSupportsVersion conflicts with this one - for (byte otherVer : other.loaderSupportedVersions) - { - if (Arrays.binarySearch(loaderSupportedVersions, otherVer) >= 0) - { - return true; - } - } - return false; - })) - { - throw new IllegalArgumentException("Loader for class " + fullDataSourceClass + " that supports one of the version in " - + Arrays.toString(loaderSupportedVersions) + " already registered!"); - } - - DATATYPE_REGISTRY.put(datatype, fullDataSourceClass); - LOADER_REGISTRY.put(fullDataSourceClass, this); - } - - - - //================// - // loader getters // - //================// - - public static AbstractFullDataSourceLoader getLoader(String dataType, byte dataVersion) - { - return LOADER_REGISTRY.get(DATATYPE_REGISTRY.get(dataType)).stream() - .filter(loader -> Arrays.binarySearch(loader.loaderSupportedVersions, dataVersion) >= 0) - .findFirst().orElse(null); - } - - public static AbstractFullDataSourceLoader getLoader(Class clazz, byte dataVersion) - { - return LOADER_REGISTRY.get(clazz).stream() - .filter(loader -> Arrays.binarySearch(loader.loaderSupportedVersions, dataVersion) >= 0) - .findFirst().orElse(null); - } - - - - //==================// - // abstract methods // - //==================// - - protected abstract IFullDataSource createEmptyDataSource(DhSectionPos pos); - - - - //==============// - // data loading // - //==============// - - /** Should be used in conjunction with {@link AbstractFullDataSourceLoader#returnPooledDataSource} to return the pooled sources. */ - public IFullDataSource loadTemporaryDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException - { - IFullDataSource dataSource = this.tryGetPooledSource(); - if (dataSource != null) - { - dataSource.repopulateFromStream(dto, dto.getInputStream(), level); - } - else - { - dataSource = this.loadDataSource(dto, level); - } - - return dataSource; - } - - /** - * Can return null if any of the requirements aren't met. - * - * @throws InterruptedException if the loader thread is interrupted, generally happens when the level is shutting down - */ - public IFullDataSource loadDataSource(DataSourceDto dto, IDhLevel level) throws IOException, InterruptedException - { - IFullDataSource dataSource = this.createEmptyDataSource(dto.pos); - dataSource.populateFromStream(dto, dto.getInputStream(), level); - return dataSource; - } - - - - //=====================// - // data source pooling // - //=====================// - - /** @return null if no pooled source exists */ - public IFullDataSource tryGetPooledSource() - { - try - { - this.cacheLock.lock(); - - int index = this.cachedSources.size() - 1; - if (index == -1) - { - return null; - } - else - { - return this.cachedSources.remove(index); - } - } - finally - { - this.cacheLock.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link AbstractFullDataSourceLoader#tryGetPooledSource} is called. - */ - public void returnPooledDataSource(IFullDataSource dataSource) - { - if (dataSource == null) - { - return; - } - else if (dataSource.getClass() != this.fullDataSourceClass) - { - return; - } - else if (this.cachedSources.size() > 25) - { - return; - } - - try - { - this.cacheLock.lock(); - this.cachedSources.add(dataSource); - } - finally - { - this.cacheLock.unlock(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java deleted file mode 100644 index d9088d20d..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/CompleteFullDataSourceLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; - -public class CompleteFullDataSourceLoader extends AbstractFullDataSourceLoader -{ - public CompleteFullDataSourceLoader() { super(CompleteFullDataSource.class, CompleteFullDataSource.DATA_TYPE_NAME, new byte[]{CompleteFullDataSource.DATA_FORMAT_VERSION}); } - - @Override - protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return CompleteFullDataSource.createEmpty(pos); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java deleted file mode 100644 index 9dcfb4f58..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/HighDetailIncompleteFullDataSourceLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; - -public class HighDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader -{ - public HighDetailIncompleteFullDataSourceLoader() { super(HighDetailIncompleteFullDataSource.class, HighDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{HighDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } - - @Override - protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return HighDetailIncompleteFullDataSource.createEmpty(pos); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java deleted file mode 100644 index 49793c958..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/loader/LowDetailIncompleteFullDataSourceLoader.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.loader; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; - -public class LowDetailIncompleteFullDataSourceLoader extends AbstractFullDataSourceLoader -{ - public LowDetailIncompleteFullDataSourceLoader() { super(LowDetailIncompleteFullDataSource.class, LowDetailIncompleteFullDataSource.DATA_TYPE_NAME, new byte[]{LowDetailIncompleteFullDataSource.DATA_FORMAT_VERSION}); } - - @Override - protected IFullDataSource createEmptyDataSource(DhSectionPos pos) { return LowDetailIncompleteFullDataSource.createEmpty(pos); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 1a1e4f885..4ed5e4cc7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -20,17 +20,13 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; +import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhBlockPos2D; -import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -47,10 +43,8 @@ import java.util.Arrays; * This data source contains every datapoint over its given {@link DhSectionPos}. * * @see FullDataPointUtil - * @see LowDetailIncompleteFullDataSource - * @see HighDetailIncompleteFullDataSource */ -public class CompleteFullDataSource extends FullDataArrayAccessor implements IFullDataSource, IStreamableFullDataSource +public class CompleteFullDataSource extends FullDataArrayAccessor implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -60,8 +54,15 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu public static final byte DATA_FORMAT_VERSION = 3; public static final String DATA_TYPE_NAME = "CompleteFullDataSource"; - @Override - public String getDataTypeName() { return DATA_TYPE_NAME; } + + /** + * This is the byte put between different sections in the binary save file. + * The presence and absence of this byte indicates if the file is correctly formatted. + */ + private static final int DATA_GUARD_BYTE = 0xFFFFFFFF; + /** indicates the binary save file represents an empty data source */ + private static final int NO_DATA_FLAG_BYTE = 0x00000001; + private DhSectionPos sectionPos; @@ -92,11 +93,100 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu - //=================// - // stream handling // - //=================// + //======// + // data // + //======// + + @Deprecated + @Override + public void update(NewFullDataSource dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } + + + + //=====================// + // setters and getters // + //=====================// @Override + public DhSectionPos getKey() { return this.sectionPos; } + + @Override + public DhSectionPos getSectionPos() { return this.sectionPos; } + + public void resizeDataStructuresForRepopulation(DhSectionPos pos) + { + // no data structures need to be changed, only the source's position + this.sectionPos = pos; + } + + @Override + public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } + + @Override + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } + + @Override + public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } + @Override + public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { return this.worldGenStep; } + + public boolean isEmpty() { return this.isEmpty; } + + + + //=================// + // stream handling // + //=================// + + /** + * Clears and then overwrites any data in this object with the data from the given file and stream. + * This is expected to be used with an existing {@link CompleteFullDataSource} and can be used in place of a constructor to reuse an existing {@link CompleteFullDataSource} object. + */ + public void repopulateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + { + // clear/overwrite the old data + this.resizeDataStructuresForRepopulation(dto.pos); + this.getMapping().clear(dto.pos); + + // set the new data + this.populateFromStream(dto, inputStream, level); + } + + /** + * Overwrites any data in this object with the data from the given file and stream. + * This is expected to be used with an empty {@link CompleteFullDataSource} and functions similar to a constructor. + */ + public void populateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + { + FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level); + this.setSourceSummaryData(summaryData); + + + long[][] dataPoints = this.readDataPoints(summaryData.dataWidth, inputStream); + if (dataPoints == null) + { + return; + } + this.setDataPoints(dataPoints); + + + FullDataPointIdMap mapping = this.readIdMappings(inputStream, level.getLevelWrapper()); + this.setIdMapping(mapping); + + } + + + // low level stream methods // + + @Deprecated + @Override + public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException + { + throw new UnsupportedOperationException("Deprecated"); + } + + /** unused, just here for reference as to how the data was written */ + @Deprecated public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException { outputStream.writeInt(this.getDataDetailLevel()); @@ -105,8 +195,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu outputStream.writeByte(this.worldGenStep.value); } - @Override - public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetail = inputStream.readInt(); if (dataDetail != dto.dataDetailLevel) @@ -137,21 +226,18 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return new FullDataSourceSummaryData(width, worldGenStep); } - public void setSourceSummaryData(FullDataSourceSummaryData summaryData) - { - this.worldGenStep = summaryData.worldGenStep; - } + public void setSourceSummaryData(FullDataSourceSummaryData summaryData) { this.worldGenStep = summaryData.worldGenStep; } - - @Override + /** unused, just here for reference as to how the data was written */ + @Deprecated public boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException { if (this.isEmpty()) { - outputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); + outputStream.writeInt(NO_DATA_FLAG_BYTE); return false; } - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(DATA_GUARD_BYTE); @@ -167,7 +253,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu // Data array content (only on non-empty columns) - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(DATA_GUARD_BYTE); for (int x = 0; x < this.width; x++) { for (int z = 0; z < this.width; z++) @@ -187,19 +273,18 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return true; } - @Override - public long[][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream dataInputStream) throws IOException + public long[][] readDataPoints(int width, DhDataInputStream dataInputStream) throws IOException { // Data array length int dataPresentFlag = dataInputStream.readInt(); - if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) + if (dataPresentFlag == NO_DATA_FLAG_BYTE) { // Section is empty return null; } - else if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) + else if (dataPresentFlag != DATA_GUARD_BYTE) { - throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + IFullDataSource.NO_DATA_FLAG_BYTE + "] or (data present) [" + IFullDataSource.DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); + throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + NO_DATA_FLAG_BYTE + "] or (data present) [" + DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); } @@ -211,7 +296,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu } else { - dataPointArrays = new long[width * width][]; + dataPointArrays = new long[width * width][]; } for (int x = 0; x < width; x++) @@ -238,7 +323,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu // check if the array start flag is present int arrayStartFlag = dataInputStream.readInt(); - if (arrayStartFlag != IFullDataSource.DATA_GUARD_BYTE) + if (arrayStartFlag != DATA_GUARD_BYTE) { throw new IOException("invalid data length end guard"); } @@ -258,7 +343,6 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu return dataPointArrays; } - @Override public void setDataPoints(long[][] dataPoints) { LodUtil.assertTrue(this.dataArrays.length == dataPoints.length, "Data point array length mismatch."); @@ -268,183 +352,47 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IFu } - @Override + /** unused, just here for reference as to how the data was written */ + @Deprecated public void writeIdMappings(DhDataOutputStream outputStream) throws IOException { - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); + outputStream.writeInt(DATA_GUARD_BYTE); this.mapping.serialize(outputStream); } - @Override - public FullDataPointIdMap readIdMappings(long[][] dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataPointIdMap readIdMappings(DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException { int guardByte = inputStream.readInt(); - if (guardByte != IFullDataSource.DATA_GUARD_BYTE) + if (guardByte != DATA_GUARD_BYTE) { throw new IOException("Invalid data content end guard for ID mapping"); } return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper); } - @Override public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - //======// - // data // - //======// + //================// + // helper classes // + //================// - @Override - public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.get(relativeX, relativeZ); } - @Override - public SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ) { return this.get(relativeX, relativeZ); } - - @Override - public void update(ChunkSizedFullDataAccessor chunkDataView) + /** + * This holds information that is relevant to the entire source and isn't stored in the data points.
+ * Example: minimum height, detail level, source type, etc. + */ + private static class FullDataSourceSummaryData { - LodUtil.assertTrue(this.sectionPos.overlapsExactly(chunkDataView.getSectionPos())); - if (this.getDataDetailLevel() == LodUtil.BLOCK_DETAIL_LEVEL) + public final int dataWidth; + public EDhApiWorldGenerationStep worldGenStep; + + + public FullDataSourceSummaryData(int dataWidth, EDhApiWorldGenerationStep worldGenStep) { - DhBlockPos2D chunkBlockPos = new DhBlockPos2D(chunkDataView.chunkPos.x * LodUtil.CHUNK_WIDTH, chunkDataView.chunkPos.z * LodUtil.CHUNK_WIDTH); - DhBlockPos2D blockOffset = chunkBlockPos.subtract(this.sectionPos.getMinCornerLodPos().getCornerBlockPos()); - LodUtil.assertTrue(blockOffset.x >= 0 && blockOffset.x < WIDTH && blockOffset.z >= 0 && blockOffset.z < WIDTH); - this.isEmpty = false; - - chunkDataView.shadowCopyTo(this.subView(LodUtil.CHUNK_WIDTH, blockOffset.x, blockOffset.z)); - - // DEBUG ASSERTION - { - for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++) - { - for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++) - { - SingleColumnFullDataAccessor column = this.get(x + blockOffset.x, z + blockOffset.z); - LodUtil.assertTrue(column.doesColumnExist()); - } - } - } - } - else if (this.getDataDetailLevel() < LodUtil.CHUNK_DETAIL_LEVEL) - { - int dataPerFull = 1 << this.getDataDetailLevel(); - int fullSize = LodUtil.CHUNK_WIDTH / dataPerFull; - DhLodPos dataOffset = chunkDataView.getSectionPos().getMinCornerLodPos(this.getDataDetailLevel()); - DhLodPos baseOffset = this.sectionPos.getMinCornerLodPos(this.getDataDetailLevel()); - - int offsetX = dataOffset.x - baseOffset.x; - int offsetZ = dataOffset.z - baseOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < WIDTH && offsetZ >= 0 && offsetZ < WIDTH); - - this.isEmpty = false; - for (int xOffset = 0; xOffset < fullSize; xOffset++) - { - for (int zOffset = 0; zOffset < fullSize; zOffset++) - { - SingleColumnFullDataAccessor column = this.get(xOffset + offsetX, zOffset + offsetZ); - column.downsampleFrom(chunkDataView.subView(dataPerFull, xOffset * dataPerFull, zOffset * dataPerFull)); - } - } - } - else if (this.getDataDetailLevel() >= LodUtil.CHUNK_DETAIL_LEVEL) - { - //FIXME: TEMPORARY - int chunkPerFull = 1 << (this.getDataDetailLevel() - LodUtil.CHUNK_DETAIL_LEVEL); - if (chunkDataView.chunkPos.x % chunkPerFull != 0 || chunkDataView.chunkPos.z % chunkPerFull != 0) - { - return; - } - - DhLodPos baseOffset = this.sectionPos.getMinCornerLodPos(this.getDataDetailLevel()); - DhSectionPos dataOffset = chunkDataView.getSectionPos().convertNewToDetailLevel(this.getDataDetailLevel()); - - int offsetX = dataOffset.getX() - baseOffset.x; - int offsetZ = dataOffset.getZ() - baseOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < WIDTH && offsetZ >= 0 && offsetZ < WIDTH); - - this.isEmpty = false; - chunkDataView.get(0, 0).deepCopyTo(this.get(offsetX, offsetZ)); - } - else - { - LodUtil.assertNotReach(); - //TODO + this.dataWidth = dataWidth; + this.worldGenStep = worldGenStep; } } - - - //================// - // helper methods // - //================// - - /** Returns whether data at the given posToWrite can effect the target region file at posToTest. */ - public static boolean firstDataPosCanAffectSecond(DhSectionPos posToWrite, DhSectionPos posToTest) - { - if (!posToWrite.overlapsExactly(posToTest)) - { - // the testPosition is outside the writePosition - return false; - } - else if (posToTest.getDetailLevel() > posToWrite.getDetailLevel()) - { - // the testPosition is larger (aka is less detailed) than the writePosition, - // more detailed sections shouldn't be updated by lower detail sections - return false; - } - else if (posToWrite.getDetailLevel() - posToTest.getDetailLevel() <= SECTION_SIZE_OFFSET) - { - // if the difference in detail levels is very large, the posToWrite - // may be skipped, due to how we sample large detail levels by only - // getting the corners. - - // In this case the difference isn't very large, so return true - return true; - } - else - { - // the difference in detail levels is very large, - // check if the posToWrite is in a corner of posToTest - byte sectPerData = (byte) BitShiftUtil.powerOfTwo(posToWrite.getDetailLevel() - posToTest.getDetailLevel() - SECTION_SIZE_OFFSET); - LodUtil.assertTrue(sectPerData != 0); - return posToTest.getX() % sectPerData == 0 && posToTest.getZ() % sectPerData == 0; - } - } - - - - //=====================// - // setters and getters // - //=====================// - - @Override - public DhSectionPos getKey() { return this.sectionPos; } - - @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public void resizeDataStructuresForRepopulation(DhSectionPos pos) - { - // no data structures need to be changed, only the source's position - this.sectionPos = pos; - } - - @Override - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - - @Override - public boolean isEmpty() { return this.isEmpty; } - @Override - public void markNotEmpty() { this.isEmpty = false; } - - @Override - public int getWidthInDataPoints() { return this.width; } - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java deleted file mode 100644 index d1a2959c6..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/HighDetailIncompleteFullDataSource.java +++ /dev/null @@ -1,586 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import org.apache.logging.log4j.Logger; - -import java.io.*; -import java.util.BitSet; - -/** - * Used for small incomplete LOD blocks.
- * Handles incomplete full data with a detail level equal to or lower than - * {@link HighDetailIncompleteFullDataSource#MAX_SECTION_DETAIL}.

- * - * Compared to other {@link IIncompleteFullDataSource}'s, this object doesn't extend {@link FullDataArrayAccessor}, - * instead it contains several "sections" of data, represented by {@link FullDataArrayAccessor}s.

- * - * Formerly "SparseFullDataSource". - * - * @see LowDetailIncompleteFullDataSource - * @see CompleteFullDataSource - * @see FullDataPointUtil - */ -public class HighDetailIncompleteFullDataSource implements IIncompleteFullDataSource, IStreamableFullDataSource -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - // TODO James would like to rename, comment, and potentially remove some of these constants. - // But he doesn't currently have the understanding to do so. - public static final byte SPARSE_UNIT_DETAIL = LodUtil.CHUNK_DETAIL_LEVEL; - public static final byte SPARSE_UNIT_SIZE = (byte) BitShiftUtil.powerOfTwo(SPARSE_UNIT_DETAIL); - - public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - public static final int SECTION_SIZE = (byte) BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - /** aka max detail level */ - public static final byte MAX_SECTION_DETAIL = SECTION_SIZE_OFFSET + SPARSE_UNIT_DETAIL; - - public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_TYPE_NAME = "HighDetailIncompleteFullDataSource"; - @Override - public String getDataTypeName() { return DATA_TYPE_NAME; } - - - protected final FullDataPointIdMap mapping; - private DhSectionPos sectionPos; - private FullDataArrayAccessor[] sparseData; - private DhLodPos chunkPos; - - public int sectionCount; - public int dataPointsPerSection; - public boolean isEmpty = true; - public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; - private boolean isPromoted = false; - - - - //==============// - // constructors // - //==============// - - public static HighDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new HighDetailIncompleteFullDataSource(pos); } - private HighDetailIncompleteFullDataSource(DhSectionPos sectionPos) - { - LodUtil.assertTrue(sectionPos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(sectionPos.getDetailLevel() <= MAX_SECTION_DETAIL); - - this.sectionPos = sectionPos; - this.sectionCount = BitShiftUtil.powerOfTwo(sectionPos.getDetailLevel() - SPARSE_UNIT_DETAIL); - this.dataPointsPerSection = SECTION_SIZE / this.sectionCount; - - this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount]; - this.chunkPos = sectionPos.getMinCornerLodPos(SPARSE_UNIT_DETAIL); - this.mapping = new FullDataPointIdMap(sectionPos); - } - - protected HighDetailIncompleteFullDataSource(DhSectionPos sectionPos, FullDataPointIdMap mapping, FullDataArrayAccessor[] data) - { - LodUtil.assertTrue(sectionPos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(sectionPos.getDetailLevel() <= MAX_SECTION_DETAIL); - - this.sectionPos = sectionPos; - this.sectionCount = 1 << (byte) (sectionPos.getDetailLevel() - SPARSE_UNIT_DETAIL); - this.dataPointsPerSection = SECTION_SIZE / this.sectionCount; - - LodUtil.assertTrue(this.sectionCount * this.sectionCount == data.length); - this.sparseData = data; - this.chunkPos = sectionPos.getMinCornerLodPos(SPARSE_UNIT_DETAIL); - this.isEmpty = false; - this.mapping = mapping; - } - - - - //=================// - // stream handling // - //=================// - - - @Override - public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream dataOutputStream) throws IOException - { - dataOutputStream.writeShort(this.getDataDetailLevel()); - dataOutputStream.writeShort(SPARSE_UNIT_DETAIL); - dataOutputStream.writeInt(SECTION_SIZE); - dataOutputStream.writeInt(level.getMinY()); - dataOutputStream.writeByte(this.worldGenStep.value); - - } - @Override - public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException - { - LodUtil.assertTrue(dto.pos.getDetailLevel() > SPARSE_UNIT_DETAIL); - LodUtil.assertTrue(dto.pos.getDetailLevel() <= MAX_SECTION_DETAIL); - - int dataDetailLevel = inputStream.readShort(); - if (dataDetailLevel != dto.dataDetailLevel) - { - throw new IOException("Data level mismatch: ["+dataDetailLevel+"] != ["+dto.dataDetailLevel+"]"); - } - - // confirm that the detail level is correct - int sparseDetail = inputStream.readShort(); - if (sparseDetail != SPARSE_UNIT_DETAIL) - { - throw new IOException("Unexpected sparse detail level: ["+sparseDetail+"] != ["+SPARSE_UNIT_DETAIL+"]"); - } - - // confirm the scale of the data points is correct - int sectionSize = inputStream.readInt(); - if (sectionSize != SECTION_SIZE) - { - throw new IOException("Section size mismatch: ["+sectionSize+"] != ["+SECTION_SIZE+"] (Currently only 1 section size is supported)"); - } - - int minY = inputStream.readInt(); - if (minY != level.getMinY()) - { - LOGGER.warn("Data minY mismatch: [" + minY + "] != [" + level.getMinY() + "]. Will ignore data's y level"); - } - - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); - if (worldGenStep == null) - { - worldGenStep = EDhApiWorldGenerationStep.SURFACE; - LOGGER.warn("Missing WorldGenStep, defaulting to: " + worldGenStep.name()); - } - - - return new FullDataSourceSummaryData(-1, worldGenStep); - } - public void setSourceSummaryData(FullDataSourceSummaryData summaryData) { this.worldGenStep = summaryData.worldGenStep; } - - - @Override - public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException - { - if (this.isEmpty) - { - dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); - return false; - } - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - - - // sparse array existence bitset - BitSet dataArrayIndexHasData = new BitSet(this.sparseData.length); - for (int i = 0; i < this.sparseData.length; i++) - { - dataArrayIndexHasData.set(i, this.sparseData[i] != null); - } - byte[] bytes = dataArrayIndexHasData.toByteArray(); - dataOutputStream.writeInt(bytes.length); - dataOutputStream.write(bytes); - - - // Data array content (only non-empty data is written) - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - for (int dataArrayIndex = dataArrayIndexHasData.nextSetBit(0); - dataArrayIndex >= 0; - dataArrayIndex = dataArrayIndexHasData.nextSetBit(dataArrayIndex + 1)) - { - // column data length - FullDataArrayAccessor array = this.sparseData[dataArrayIndex]; - LodUtil.assertTrue(array != null); - for (int x = 0; x < array.width(); x++) - { - for (int z = 0; z < array.width(); z++) - { - SingleColumnFullDataAccessor columnAccessor = array.get(x, z); - int columnLength = 0; - if (columnAccessor != null) - { - columnLength = columnAccessor.getSingleLength(); - } - dataOutputStream.writeInt(columnLength); - } - } - - // column data - for (int x = 0; x < array.width(); x++) - { - for (int z = 0; z < array.width(); z++) - { - SingleColumnFullDataAccessor column = array.get(x, z); - LodUtil.assertTrue(column.getMapping() == this.mapping); // the mappings must be exactly equal! - - if (column.doesColumnExist()) - { - long[] rawDataPoints = column.getRaw(); - for (long dataPoint : rawDataPoints) - { - dataOutputStream.writeLong(dataPoint); - } - } - } - } - } - - - return true; - } - @Override - public long[][][] readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException - { - // calculate the number of chunks and dataPoints based on the sparseDetail and sectionSize - // TODO these values should be constant, should we still be calculating them like this? - int chunks = BitShiftUtil.powerOfTwo(dto.pos.getDetailLevel() - SPARSE_UNIT_DETAIL); - int dataPointsPerChunk = SECTION_SIZE / chunks; - - - // check if this file has any data - int dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) - { - // this file is empty - return null; - } - else if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + IFullDataSource.NO_DATA_FLAG_BYTE + "] or (data present) [" + IFullDataSource.DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); - } - - - // get the number of columns (IE the bitSet from before) - int numberOfDataColumns = inputStream.readInt(); - // validate the number of data columns - int maxNumberOfDataColumns = (chunks * chunks / 8 + 64) * 2; // TODO what do these values represent? - if (numberOfDataColumns < 0 || numberOfDataColumns > maxNumberOfDataColumns) - { - throw new IOException(LodUtil.formatLog("Sparse Flag BitSet size outside reasonable range: {} (expects {} to {})", - numberOfDataColumns, 1, maxNumberOfDataColumns)); - } - - // read in the presence of each data column - byte[] bytes = new byte[numberOfDataColumns]; - inputStream.readFully(bytes, 0, numberOfDataColumns); - BitSet dataArrayIndexHasData = BitSet.valueOf(bytes); - - - - //====================// - // Data array content // - //====================// - - // (only on non-empty columns) - int dataArrayStartByte = inputStream.readInt(); - // confirm the column data is starting - if (dataArrayStartByte != IFullDataSource.DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("invalid data length end guard"); - } - - - // read in each column that has data written to it - long[][][] rawFullDataArrays = new long[chunks * chunks][][]; - for (int fullDataIndex = dataArrayIndexHasData.nextSetBit(0); - fullDataIndex >= 0 && // TODO why does this happen? - fullDataIndex < rawFullDataArrays.length; - fullDataIndex = dataArrayIndexHasData.nextSetBit(fullDataIndex + 1)) - { - long[][] dataColumn = new long[dataPointsPerChunk * dataPointsPerChunk][]; - - // get the column data lengths - rawFullDataArrays[fullDataIndex] = dataColumn; - for (int x = 0; x < dataColumn.length; x++) - { - // this should be zero if the column doesn't have any data - int dataColumnLength = inputStream.readInt(); - dataColumn[x] = new long[dataColumnLength]; - } - - // get the column data - for (int x = 0; x < dataColumn.length; x++) - { - if (dataColumn[x].length != 0) - { - // read in the data columns - for (int z = 0; z < dataColumn[x].length; z++) - { - dataColumn[x][z] = inputStream.readLong(); - } - } - } - } - - - return rawFullDataArrays; - } - @Override - public void setDataPoints(long[][][] dataPoints) - { - LodUtil.assertTrue(this.sparseData.length == dataPoints.length, "Data point array length mismatch."); - - this.isEmpty = false; - - - for (int arrayAccessorIndex = 0; arrayAccessorIndex < dataPoints.length; arrayAccessorIndex++) - { - if (dataPoints[arrayAccessorIndex] == null) - { - this.sparseData[arrayAccessorIndex] = null; - } - else if (this.sparseData[arrayAccessorIndex] == null) - { - int width = (int) Math.sqrt(dataPoints[arrayAccessorIndex].length); - this.sparseData[arrayAccessorIndex] = new FullDataArrayAccessor(this.mapping, dataPoints[arrayAccessorIndex], width); - } - else - { - for (int dataPointColIndex = 0; dataPointColIndex < dataPoints[arrayAccessorIndex].length; dataPointColIndex++) - { - long[] incomingColumn = dataPoints[arrayAccessorIndex][dataPointColIndex]; - long[] destinationColumn = this.sparseData[arrayAccessorIndex].get(dataPointColIndex).getRaw(); - - // use the existing arrays if possible - if (incomingColumn.length == destinationColumn.length) - { - System.arraycopy(incomingColumn, 0, destinationColumn, 0, incomingColumn.length); - } - else - { - this.sparseData[arrayAccessorIndex].get(dataPointColIndex).setNew(incomingColumn); - } - } - } - } - } - - - @Override - public FullDataPointIdMap readIdMappings(long[][][] dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException - { - // mark the start of the ID data - int idMappingStartByte = inputStream.readInt(); - if (idMappingStartByte != DATA_GUARD_BYTE) - { - // the file format is incorrect - throw new IOException("invalid data content end guard"); - } - - // deserialize the ID data - return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper); - } - @Override - public void writeIdMappings(DhDataOutputStream dataOutputStream) throws IOException - { - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(dataOutputStream); - } - @Override - public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - - - - //======// - // data // - //======// - - @Override - public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, false); } - @Override - public SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, true); } - private SingleColumnFullDataAccessor tryGetOrCreate(int relativeX, int relativeZ, boolean createIfMissing) - { - LodUtil.assertTrue(relativeX >= 0 && relativeX < SECTION_SIZE && relativeZ >= 0 && relativeZ < SECTION_SIZE); - int chunkX = relativeX / this.dataPointsPerSection; - int chunkZ = relativeZ / this.dataPointsPerSection; - FullDataArrayAccessor accessor = this.sparseData[chunkX * this.sectionCount + chunkZ]; - if (accessor == null) - { - if (createIfMissing) - { - // create the missing data so the following get() will succeed - accessor = new FullDataArrayAccessor(this.mapping, new long[this.dataPointsPerSection * this.dataPointsPerSection][], this.dataPointsPerSection); - this.sparseData[chunkX * this.sectionCount + chunkZ] = accessor; - } - else - { - return null; - } - } - - return accessor.get(relativeX % this.dataPointsPerSection, relativeZ % this.dataPointsPerSection); - } - - - - //=========// - // getters // - //=========// - - @Override - public DhSectionPos getKey() { return this.sectionPos; } - - @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public void resizeDataStructuresForRepopulation(DhSectionPos pos) - { - // update the position - this.sectionPos = pos; - this.sectionCount = BitShiftUtil.powerOfTwo(this.sectionPos.getDetailLevel() - SPARSE_UNIT_DETAIL); - this.dataPointsPerSection = SECTION_SIZE / this.sectionCount; - - this.chunkPos = this.sectionPos.getMinCornerLodPos(SPARSE_UNIT_DETAIL); - - - // update the data container - int dataPointCount = this.sectionCount * this.sectionCount; - if (this.sparseData.length != dataPointCount) - { - this.sparseData = new FullDataArrayAccessor[this.sectionCount * this.sectionCount]; - } - - } - - @Override - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - - @Override - public FullDataPointIdMap getMapping() { return this.mapping; } - - @Override - public boolean isEmpty() { return this.isEmpty; } - @Override - public void markNotEmpty() { this.isEmpty = false; } - - @Override - public int getWidthInDataPoints() { return SECTION_SIZE; } - - - private int calculateOffset(int chunkX, int chunkZ) - { - int offsetX = chunkX - this.chunkPos.x; - int offsetZ = chunkZ - this.chunkPos.z; - LodUtil.assertTrue(offsetX >= 0 && offsetZ >= 0 && offsetX < this.sectionCount && offsetZ < this.sectionCount); - return offsetX * this.sectionCount + offsetZ; - } - - - - //=============// - // data update // - //=============// - - @Override - public void update(ChunkSizedFullDataAccessor chunkDataView) - { - int arrayOffset = this.calculateOffset(chunkDataView.chunkPos.x, chunkDataView.chunkPos.z); - FullDataArrayAccessor newArray = new FullDataArrayAccessor(this.mapping, new long[this.dataPointsPerSection * this.dataPointsPerSection][], this.dataPointsPerSection); - if (this.getDataDetailLevel() == chunkDataView.detailLevel) - { - chunkDataView.shadowCopyTo(newArray); - } - else - { - int count = this.dataPointsPerSection; - int dataPerCount = SPARSE_UNIT_SIZE / this.dataPointsPerSection; - - for (int xOffset = 0; xOffset < count; xOffset++) - { - for (int zOffset = 0; zOffset < count; zOffset++) - { - SingleColumnFullDataAccessor column = newArray.get(xOffset, zOffset); - column.downsampleFrom(chunkDataView.subView(dataPerCount, xOffset * dataPerCount, zOffset * dataPerCount)); - } - } - } - - this.isEmpty = false; - this.sparseData[arrayOffset] = newArray; - } - - - // data sampling // - - private void applyToFullDataSource(CompleteFullDataSource dataSource) - { - LodUtil.assertTrue(dataSource.getSectionPos().equals(this.sectionPos)); - LodUtil.assertTrue(dataSource.getDataDetailLevel() == this.getDataDetailLevel()); - for (int x = 0; x < this.sectionCount; x++) - { - for (int z = 0; z < this.sectionCount; z++) - { - FullDataArrayAccessor array = this.sparseData[x * this.sectionCount + z]; - if (array == null) - continue; - - // Otherwise, apply data to dataSource - dataSource.markNotEmpty(); - FullDataArrayAccessor view = dataSource.subView(this.dataPointsPerSection, x * this.dataPointsPerSection, z * this.dataPointsPerSection); - array.shadowCopyTo(view); - } - } - } - - public IFullDataSource tryPromotingToCompleteDataSource() - { - if (this.isEmpty) - { - return this; - } - - // promotion can only succeed if every data column is present - for (FullDataArrayAccessor array : this.sparseData) - { - if (array == null) - { - return this; - } - } - this.isPromoted = true; - CompleteFullDataSource fullDataSource = CompleteFullDataSource.createEmpty(this.sectionPos); - this.applyToFullDataSource(fullDataSource); - return fullDataSource; - } - - @Override - public boolean hasBeenPromoted() { return this.isPromoted; } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java deleted file mode 100644 index e80d4d813..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/LowDetailIncompleteFullDataSource.java +++ /dev/null @@ -1,439 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IStreamableFullDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import org.apache.logging.log4j.Logger; - -import java.io.*; -import java.util.BitSet; - -/** - * Used for large incomplete LOD blocks.
- * Handles incomplete full data with a detail level higher than - * {@link HighDetailIncompleteFullDataSource#MAX_SECTION_DETAIL}.

- * - * Formerly "SpottyFullDataSource". - * - * @see HighDetailIncompleteFullDataSource - * @see CompleteFullDataSource - * @see FullDataPointUtil - */ -public class LowDetailIncompleteFullDataSource extends FullDataArrayAccessor implements IIncompleteFullDataSource, IStreamableFullDataSource -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - /** measured in dataPoints */ - public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - - public static final byte DATA_FORMAT_VERSION = 3; - public static final String DATA_TYPE_NAME = "LowDetailIncompleteFullDataSource"; - @Override - public String getDataTypeName() { return DATA_TYPE_NAME; } - - private DhSectionPos sectionPos; - - private final BitSet isColumnNotEmpty; - - private boolean isEmpty = true; - public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; - private boolean isPromoted = false; - - - - //==============// - // constructors // - //==============// - - public static LowDetailIncompleteFullDataSource createEmpty(DhSectionPos pos) { return new LowDetailIncompleteFullDataSource(pos); } - private LowDetailIncompleteFullDataSource(DhSectionPos sectionPos) - { - super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH); - LodUtil.assertTrue(sectionPos.getDetailLevel() > HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL); - - this.sectionPos = sectionPos; - this.isColumnNotEmpty = new BitSet(WIDTH * WIDTH); - this.worldGenStep = EDhApiWorldGenerationStep.EMPTY; - } - - private LowDetailIncompleteFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, EDhApiWorldGenerationStep worldGenStep, BitSet isColumnNotEmpty, long[][] data) - { - super(mapping, data, WIDTH); - LodUtil.assertTrue(data.length == WIDTH * WIDTH); - - this.sectionPos = pos; - this.isColumnNotEmpty = isColumnNotEmpty; - this.worldGenStep = worldGenStep; - this.isEmpty = false; - } - - - - //=================// - // stream handling // - //=================// - - - @Override - public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException - { - outputStream.writeInt(this.getDataDetailLevel()); - outputStream.writeInt(this.width); - outputStream.writeInt(level.getMinY()); - outputStream.writeByte(this.worldGenStep.value); - - } - @Override - public FullDataSourceSummaryData readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException - { - int dataDetailLevel = inputStream.readInt(); - if (dataDetailLevel != dto.dataDetailLevel) - { - throw new IOException(LodUtil.formatLog("Data level mismatch: " + dataDetailLevel + " != " + dto.dataDetailLevel)); - } - - int width = inputStream.readInt(); - if (width != WIDTH) - { - throw new IOException(LodUtil.formatLog("Section size mismatch: " + width + " != " + WIDTH + " (Currently only 1 section size is supported)")); - } - - int minY = inputStream.readInt(); - if (minY != level.getMinY()) - { - LOGGER.warn("Data minY mismatch: " + minY + " != " + level.getMinY() + ". Will ignore data's y level"); - } - - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); - if (worldGenStep == null) - { - worldGenStep = EDhApiWorldGenerationStep.SURFACE; - LOGGER.warn("Missing WorldGenStep, defaulting to: " + worldGenStep.name()); - } - - - return new FullDataSourceSummaryData(this.width, worldGenStep); - } - public void setSourceSummaryData(FullDataSourceSummaryData summaryData) - { - this.worldGenStep = summaryData.worldGenStep; - } - - - @Override - public boolean writeDataPoints(DhDataOutputStream dataOutputStream) throws IOException - { - if (this.isEmpty) - { - dataOutputStream.writeInt(IFullDataSource.NO_DATA_FLAG_BYTE); - return false; - } - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - - - // data column presence - byte[] bytes = this.isColumnNotEmpty.toByteArray(); - dataOutputStream.writeInt(bytes.length); - dataOutputStream.write(bytes); - - - // Data content - dataOutputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - for (int i = this.isColumnNotEmpty.nextSetBit(0); i >= 0; i = this.isColumnNotEmpty.nextSetBit(i + 1)) - { - dataOutputStream.writeByte(this.dataArrays[i].length); - for (long dataPoint : this.dataArrays[i]) - { - dataOutputStream.writeLong(dataPoint); - } - } - - - return true; - } - @Override - public StreamDataPointContainer readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException - { - // is source empty flag - int dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag == IFullDataSource.NO_DATA_FLAG_BYTE) - { - // Section is empty - return null; - } - else if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - throw new IOException("Invalid file format. Data Points guard byte expected: (no data) [" + IFullDataSource.NO_DATA_FLAG_BYTE + "] or (data present) [" + IFullDataSource.DATA_GUARD_BYTE + "], but found [" + dataPresentFlag + "]."); - } - - - // data column presence - int length = inputStream.readInt(); - if (length < 0 || length > (WIDTH * WIDTH / 8 + 64) * 2) // TODO replace magic numbers or comment what they mean - { - throw new IOException(LodUtil.formatLog("Spotty Flag BitSet size outside reasonable range: {} (expects {} to {})", - length, 1, WIDTH * WIDTH / 8 + 63)); - } - - byte[] bytes = new byte[length]; - inputStream.readFully(bytes, 0, length); - BitSet isColumnNotEmpty = BitSet.valueOf(bytes); - - - - // Data array content - long[][] dataPointArray = new long[WIDTH * WIDTH][]; - dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - throw new IOException("invalid spotty flag end guard"); - } - - for (int xz = isColumnNotEmpty.nextSetBit(0); xz >= 0; xz = isColumnNotEmpty.nextSetBit(xz + 1)) - { - long[] array = new long[inputStream.readByte()]; - for (int y = 0; y < array.length; y++) - { - array[y] = inputStream.readLong(); - } - dataPointArray[xz] = array; - } - - - return new StreamDataPointContainer(dataPointArray, isColumnNotEmpty); - } - @Override - public void setDataPoints(StreamDataPointContainer streamDataPointContainer) - { - long[][] dataPoints = streamDataPointContainer.dataPoints; - - // copy over the datapoints - LodUtil.assertTrue(this.dataArrays.length == dataPoints.length, "Data point array length mismatch."); - System.arraycopy(dataPoints, 0, this.dataArrays, 0, dataPoints.length); - - // overwrite the bitset - for (int i = 0; i < streamDataPointContainer.isColumnNotEmpty.length(); i++) - { - this.isColumnNotEmpty.set(i, streamDataPointContainer.isColumnNotEmpty.get(i)); - } - - this.isEmpty = false; - } - - - @Override - public void writeIdMappings(DhDataOutputStream outputStream) throws IOException - { - outputStream.writeInt(IFullDataSource.DATA_GUARD_BYTE); - this.mapping.serialize(outputStream); - - } - @Override - public FullDataPointIdMap readIdMappings(StreamDataPointContainer streamDataPointContainer, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException - { - // Id mapping - int dataPresentFlag = inputStream.readInt(); - if (dataPresentFlag != IFullDataSource.DATA_GUARD_BYTE) - { - throw new IOException("invalid ID mapping end guard"); - } - return FullDataPointIdMap.deserialize(inputStream, this.sectionPos, levelWrapper); - } - @Override - public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } - - - - //======// - // data // - //======// - - @Override - public SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, false); } - @Override - public SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ) { return this.tryGetOrCreate(relativeX, relativeZ, true); } - private SingleColumnFullDataAccessor tryGetOrCreate(int relativeX, int relativeZ, boolean createIfMissing) - { - int notEmptyIndex = relativeX * WIDTH + relativeZ; - boolean columnEmpty = this.isColumnNotEmpty.get(notEmptyIndex); - - // "create" the missing column if necessary - if (columnEmpty && createIfMissing) - { - this.isColumnNotEmpty.set(notEmptyIndex, true); - columnEmpty = false; - } - - return !columnEmpty ? this.get(relativeX, relativeZ) : null; - } - - - - //=====================// - // getters and setters // - //=====================// - - @Override - public DhSectionPos getKey() { return this.sectionPos; } - - @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } - - @Override - public void resizeDataStructuresForRepopulation(DhSectionPos pos) - { - // no data structures need to be changed, only the source's position - this.sectionPos = pos; - } - - @Override - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - - @Override - public boolean isEmpty() { return this.isEmpty; } - @Override - public void markNotEmpty() { this.isEmpty = false; } - - @Override - public int getWidthInDataPoints() { return WIDTH; } - - - - //===============// - // Data updating // - //===============// - - @Override - public void update(ChunkSizedFullDataAccessor data) - { - LodUtil.assertTrue(this.sectionPos.overlapsExactly(data.getSectionPos())); - - if (this.getDataDetailLevel() >= 4) - { - //FIXME: TEMPORARY - int chunkPerFull = 1 << (this.getDataDetailLevel() - 4); - if (data.chunkPos.x % chunkPerFull != 0 || data.chunkPos.z % chunkPerFull != 0) - { - return; - } - - DhLodPos baseOffset = this.sectionPos.getMinCornerLodPos(this.getDataDetailLevel()); - DhSectionPos dataOffset = data.getSectionPos().convertNewToDetailLevel(this.getDataDetailLevel()); - int offsetX = dataOffset.getX() - baseOffset.x; - int offsetZ = dataOffset.getZ() - baseOffset.z; - LodUtil.assertTrue(offsetX >= 0 && offsetX < WIDTH && offsetZ >= 0 && offsetZ < WIDTH); - this.isEmpty = false; - - SingleColumnFullDataAccessor columnFullDataAccessor = this.get(offsetX, offsetZ); - data.get(0, 0).deepCopyTo(columnFullDataAccessor); - - this.isColumnNotEmpty.set(offsetX * WIDTH + offsetZ, columnFullDataAccessor.doesColumnExist()); - } - else - { - LodUtil.assertNotReach(); - //TODO; - } - - } - - @Override - public IFullDataSource tryPromotingToCompleteDataSource() - { - // promotion can only be completed if every column has data - if (this.isEmpty) - { - return this; - } - else if (this.isColumnNotEmpty.cardinality() != WIDTH * WIDTH) - { - return this; - } - this.isPromoted = true; - return new CompleteFullDataSource(this.sectionPos, this.mapping, this.dataArrays); - } - - @Override - public boolean hasBeenPromoted() { return this.isPromoted; } - - - - //================// - // helper classes // - //================// - - /** used when reading the datapoints to and from the {@link IStreamableFullDataSource} */ - public static class StreamDataPointContainer - { - public long[][] dataPoints; - public BitSet isColumnNotEmpty; - - public StreamDataPointContainer(long[][] dataPoints, BitSet isColumnNotEmpty) - { - this.dataPoints = dataPoints; - this.isColumnNotEmpty = isColumnNotEmpty; - } - - } - - - - //========// - // unused // - //========// - - public static boolean neededForPosition(DhSectionPos posToWrite, DhSectionPos posToTest) - { - if (!posToWrite.overlapsExactly(posToTest)) - return false; - if (posToTest.getDetailLevel() > posToWrite.getDetailLevel()) - return false; - if (posToWrite.getDetailLevel() - posToTest.getDetailLevel() <= SECTION_SIZE_OFFSET) - return true; - byte sectPerData = (byte) (1 << (posToWrite.getDetailLevel() - posToTest.getDetailLevel() - SECTION_SIZE_OFFSET)); - return posToTest.getX() % sectPerData == 0 && posToTest.getZ() % sectPerData == 0; - } - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java new file mode 100644 index 000000000..8eddbcc50 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -0,0 +1,447 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.dataObjects.fullData.sources; + +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; +import com.seibel.distanthorizons.core.file.IDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; +import com.seibel.distanthorizons.coreapi.ModInfo; +import org.apache.logging.log4j.Logger; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.concurrent.locks.ReentrantLock; + +/** + * This data source contains every datapoint over its given {@link DhSectionPos}. + * + * TODO create a child object that extends AutoClosable + * that can be pooled so we don't have GC overhead + * + * @see FullDataPointUtil + * @see CompleteFullDataSource + */ +public class NewFullDataSource implements IDataSource +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + /** measured in data columns */ + public static final int WIDTH = 64; + + public static final byte DATA_FORMAT_VERSION = 1; + + + + // TODO make these fields private + private DhSectionPos pos; + @Override + public DhSectionPos getKey() { return this.pos; } + + public long lastModifiedUnixDateTime; + public long createdUnixDateTime; + + public int levelMinY; + + /** + * stores how far each column has been generated should start with {@link EDhApiWorldGenerationStep#EMPTY} + * @see EDhApiWorldGenerationStep + */ + public byte[] columnGenerationSteps; + + /** stored x/z, y */ + public long[][] dataPoints; + private boolean isEmpty; + + private FullDataPointIdMap mapping; + public FullDataPointIdMap getMapping() { return this.mapping; } + + public boolean applyToParent = false; + + + + //==============// + // constructors // + //==============// + + public static NewFullDataSource createEmpty(DhSectionPos pos) { return new NewFullDataSource(pos); } + private NewFullDataSource(DhSectionPos pos) + { + this.pos = pos; + this.dataPoints = new long[WIDTH * WIDTH][]; + this.mapping = new FullDataPointIdMap(pos); + this.isEmpty = true; + + // doesn't need to be populated since nothing has been generated yet + // the default value of all 0's is adequate + this.columnGenerationSteps = new byte[WIDTH * WIDTH]; + } + + public static NewFullDataSource createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new NewFullDataSource(pos, mapping, data, columnGenerationStep); } + private NewFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) + { + LodUtil.assertTrue(data.length == WIDTH * WIDTH); + + this.pos = pos; + this.dataPoints = data; + this.mapping = mapping; + this.isEmpty = false; + + this.columnGenerationSteps = columnGenerationSteps; + } + + public static NewFullDataSource createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } + + public static NewFullDataSource createFromCompleteDataSource(CompleteFullDataSource legacyData) + { + byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; + long[][] dataPoints = new long[WIDTH * WIDTH][]; + for (int x = 0; x < WIDTH; x++) + { + for (int z = 0; z < WIDTH; z++) + { + int index = relativePosToIndex(x, z); + + SingleColumnFullDataAccessor accessor = legacyData.get(x, z); + + if (accessor.doesColumnExist()) + { + dataPoints[index] = accessor.getRaw(); + columnGenerationSteps[index] = legacyData.getWorldGenStep().value; + } + } + } + + return NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps); + } + + + + //======// + // data // + //======// + + public SingleColumnFullDataAccessor get(int relX, int relZ) { return new SingleColumnFullDataAccessor(this.mapping, this.dataPoints, relativePosToIndex(relX, relZ)); } + + @Override + public void update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { this.update(inputDataSource); } + public void update(NewFullDataSource inputDataSource) + { + byte thisDetailLevel = this.pos.getDetailLevel(); + byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); + + + // determine the mapping changes necessary for the input to map onto this datasource + int[] remappedIds = this.mapping.mergeAndReturnRemappedEntityIds(inputDataSource.mapping); + + boolean dataChanged = false; + if (inputDetailLevel == thisDetailLevel) + { + dataChanged = this.updateFromSameDetailLevel(inputDataSource, remappedIds); + } + else if (inputDetailLevel + 1 == thisDetailLevel) + { + dataChanged = this.updateFromOneBelowDetailLevel(inputDataSource, remappedIds); + } + else + { + // TODO what should happen here? + + // other detail levels aren't supported since it would be more difficult to maintain + // and would lead to edge cases that don't necessarily need to be supported + // (IE what do you do when the input is smaller than a single datapoint in the receiving data source?) + // instead it's better to just percolate the updates up + //throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); + } + + if (dataChanged && this.pos.getDetailLevel() < NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL) + { + // mark that this data source should be applied to its parent + this.applyToParent = true; + } + } + public boolean updateFromSameDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) + { + // both data sources should have the same detail level + if (inputDataSource.pos.getDetailLevel() != this.pos.getDetailLevel()) + { + throw new IllegalArgumentException("Both data sources must have the same detail level. Expected ["+this.pos.getDetailLevel()+"], received ["+inputDataSource.pos.getDetailLevel()+"]."); + } + + // copy over everything from the input data source into this one + // provided there is data to copy and the world generation step is the same or more complete + boolean dataChanged = false; + for (int x = 0; x < WIDTH; x++) + { + for (int z = 0; z < WIDTH; z++) + { + int index = relativePosToIndex(x, z); + + long[] newDataArray = inputDataSource.dataPoints[index]; + if (newDataArray != null) + { + byte thisGenState = this.columnGenerationSteps[index]; + byte inputGenState = inputDataSource.columnGenerationSteps[index]; + + if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value + && thisGenState <= inputGenState) + { + this.dataPoints[index] = new long[newDataArray.length]; + System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); + this.remapDataColumn(index, remappedIds); + + this.columnGenerationSteps[index] = inputGenState; + + dataChanged = true; // TODO contents of the arrays should be compared to prevent re-writing the same data + } + } + } + } + + return dataChanged; + } + public boolean updateFromOneBelowDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) + { + if (inputDataSource.pos.getDetailLevel() + 1 != this.pos.getDetailLevel()) + { + throw new IllegalArgumentException("Input data source must be exactly 1 detail level below this data source. Expected ["+(this.pos.getDetailLevel() - 1)+"], received ["+inputDataSource.pos.getDetailLevel()+"]."); + } + + // input is one detail level lower (higher detail) + // so 2x2 input data points will be converted into 1 recipient data point + + + // determine where in the input data source should be written to + // since the input is one detail level below it will be one of this position's 4 children + int minChildXPos = this.pos.getChildByIndex(0).getX(); + int recipientOffsetX = (inputDataSource.pos.getX() == minChildXPos) ? 0 : (WIDTH / 2); + int minChildZPos = this.pos.getChildByIndex(0).getZ(); + int recipientOffsetZ = (inputDataSource.pos.getZ() == minChildZPos) ? 0 : (WIDTH / 2); + + + + // merge the input's data points + // into this data source's + boolean dataChanged = false; + for (int x = 0; x < WIDTH; x += 2) + { + for (int z = 0; z < WIDTH; z += 2) + { + int inputIndex = relativePosToIndex(x, z); + + long[] inputDataArray = inputDataSource.dataPoints[inputIndex]; + if (inputDataArray != null) + { + byte inputGenStep = inputDataSource.columnGenerationSteps[inputIndex]; + + // TODO downsample instad of grabbing the column nearest to (-inf, -inf) + int recipientX = (x / 2) + recipientOffsetX; + int recipientZ = (z / 2) + recipientOffsetZ; + int recipientIndex = relativePosToIndex(recipientX, recipientZ); + + this.columnGenerationSteps[recipientIndex] = inputGenStep; + this.dataPoints[recipientIndex] = inputDataArray; + this.remapDataColumn(recipientIndex, remappedIds); + + this.isEmpty = false; + + dataChanged = true; // TODO contents of the arrays should probably be compared or something + } + } + } + + return dataChanged; + } + /** + * Only update the ID once it's been added to this data source. + * Updating the incoming data source will cause issues if it is applied + * to anything else due to multiple remapping. + */ + private void remapDataColumn(int dataPointIndex, int[] remappedIds) + { + long[] dataColumn = this.dataPoints[dataPointIndex]; + for (int i = 0; i < dataColumn.length; i++) + { + dataColumn[i] = FullDataPointUtil.remap(remappedIds, dataColumn[i]); + } + } + + + + //================// + // helper methods // + //================// + + // TODO make private, any external logic should go through a method, not interact with the arrays directly + public static int relativePosToIndex(int relX, int relZ) throws IndexOutOfBoundsException + { + if (relX < 0 || relZ < 0 || + relX > WIDTH || relZ > WIDTH) + { + throw new IndexOutOfBoundsException("Relative data source positions must be between [0] and ["+WIDTH+"] (inclusive) the relative pos: ["+relX+","+relZ+"] is outside of those boundaries."); + } + + return (relX * WIDTH) + relZ; + } + + + + //=====================// + // setters and getters // + //=====================// + + @Override + public DhSectionPos getSectionPos() { return this.pos; } + + @Override + public byte getDataDetailLevel() { return (byte) (this.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); } + + @Override + public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } + + @Deprecated + @Override + public EDhApiWorldGenerationStep getWorldGenStep() { return this.getWorldGenStepAtRelativePos(0, 0); } + @Override + public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) + { + int index = relativePosToIndex(relX, relZ); + return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); + } + + public boolean isEmpty() { return this.isEmpty; } + public void markNotEmpty() { this.isEmpty = false; } + + public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + { + int index = relativePosToIndex(relX, relZ); + this.dataPoints[index] = longArray; + this.columnGenerationSteps[index] = worldGenStep.value; + + + // validate the incoming ID's + // shouldn't normally happen and can be disabled for release builds + if (ModInfo.IS_DEV_BUILD) + { + int maxValidId = this.mapping.getMaxValidId(); + for (int i = 0; i < longArray.length; i++) + { + long dataPoint = longArray[i]; + int id = FullDataPointUtil.getId(dataPoint); + if (id > maxValidId) + { + LodUtil.assertNotReach("Column set with higher than possible ID. ID [" + id + "], max valid ID [" + maxValidId + "]."); + } + } + } + } + + @Override + public String toString() { return this.pos.toString(); } + + + + //============// + // deprecated // + //============// + + @Deprecated + @Override + public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) + { + throw new UnsupportedOperationException("deprecated"); + } + + + + //=========// + // pooling // + //=========// + + // TODO add pooled data sources + private static class Pooling + { + /** used when pooling data sources */ + private final ArrayList cachedSources = new ArrayList<>(); + private final ReentrantLock cacheLock = new ReentrantLock(); + + + /** @return null if no pooled source exists */ + public CompleteFullDataSource tryGetPooledSource() + { + try + { + this.cacheLock.lock(); + + int index = this.cachedSources.size() - 1; + if (index == -1) + { + return null; + } + else + { + return this.cachedSources.remove(index); + } + } + finally + { + this.cacheLock.unlock(); + } + } + + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. + */ + public void returnPooledDataSource(CompleteFullDataSource dataSource) + { + if (dataSource == null) + { + return; + } + else if (this.cachedSources.size() > 25) + { + return; + } + + try + { + this.cacheLock.lock(); + this.cachedSources.add(dataSource); + } + finally + { + this.cacheLock.unlock(); + } + } + + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java deleted file mode 100644 index 1dcaeed37..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IFullDataSource.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; - -import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.IFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import org.jetbrains.annotations.Nullable; - -import java.io.IOException; - -/** - * Base for all Full Data Source objects.

- * - * Contains full DH data, methods related to file/stream reading/writing, and the data necessary to create {@link ColumnRenderSource}'s.
- * {@link IFullDataSource}'s will either implement or contain {@link IFullDataAccessor}'s. - * - * @see IFullDataAccessor - * @see IIncompleteFullDataSource - * @see IStreamableFullDataSource - */ -public interface IFullDataSource extends IDataSource -{ - /** - * This is the byte put between different sections in the binary save file. - * The presence and absence of this byte indicates if the file is correctly formatted. - */ - int DATA_GUARD_BYTE = 0xFFFFFFFF; - /** indicates the binary save file represents an empty data source */ - int NO_DATA_FLAG_BYTE = 0x00000001; - - - - default void update(ChunkSizedFullDataAccessor chunkData, IDhLevel level) { this.update(chunkData); } - void update(ChunkSizedFullDataAccessor data); - - boolean isEmpty(); - void markNotEmpty(); - - /** AKA; the max relative position that {@link IFullDataSource#tryGet(int, int)} can accept for either X or Z */ - int getWidthInDataPoints(); - - - - //======// - // data // - //======// - - /** - * Attempts to get the data column for the given relative x and z position. - * - * @return null if the data doesn't exist - */ - @Nullable - SingleColumnFullDataAccessor tryGet(int relativeX, int relativeZ); - /** - * Attempts to get the data column for the given relative x and z position.
- * If no data exists yet an empty data column will be created. - */ - SingleColumnFullDataAccessor getOrCreate(int relativeX, int relativeZ); - - FullDataPointIdMap getMapping(); - - - - //=======================// - // basic stream handling // - //=======================// - - /** - * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * - * @see IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) - */ - void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; - - /** - * Should only be implemented by {@link IStreamableFullDataSource} to prevent potential stream read/write inconsistencies. - * - * @see IStreamableFullDataSource#repopulateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) - */ - void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException; - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java deleted file mode 100644 index 4514e5f45..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IIncompleteFullDataSource.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; - -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.pos.DhLodPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.LodUtil; - -public interface IIncompleteFullDataSource extends IFullDataSource -{ - /** - * Overwrites data in this object with non-null data from the input {@link IFullDataSource}.

- * - * This can be used to either merge same sized data sources or downsample to - */ - default void sampleFrom(IFullDataSource inputSource) - { - DhSectionPos inputPos = inputSource.getSectionPos(); - DhSectionPos thisPos = this.getSectionPos(); - LodUtil.assertTrue(inputPos.getDetailLevel() < thisPos.getDetailLevel(), "input data source at pos: ["+inputPos+"] has a lower detail level than this: ["+thisPos+"]."); - LodUtil.assertTrue(inputPos.overlapsExactly(this.getSectionPos()), "input source at pos: ["+inputPos+"] (converted to ["+inputPos.convertNewToDetailLevel(thisPos.getDetailLevel())+"]) doesn't overlap with this source's pos: ["+thisPos+"]."); - - if (inputSource.isEmpty()) - { - return; - } - - - this.markNotEmpty(); - - DhLodPos baseOffset = thisPos.getMinCornerLodPos(this.getDataDetailLevel()); - DhSectionPos inputOffset = inputPos.convertNewToDetailLevel(this.getDataDetailLevel()); - int offsetX = inputOffset.getX() - baseOffset.x; - int offsetZ = inputOffset.getZ() - baseOffset.z; - - - int numberOfDataPointsToUpdate = this.getWidthInDataPoints() / thisPos.getWidthCountForLowerDetailedSection(inputSource.getSectionPos().getDetailLevel()); // can be 0 if the input source is significantly smaller than this data source - // should be 1 at minimum, to prevent divide by zero errors (and because trying to get 0 or a fractional data point doesn't make any sense) - numberOfDataPointsToUpdate = Math.max(1, numberOfDataPointsToUpdate); - - - int inputFractionWidth = inputSource.getWidthInDataPoints() / numberOfDataPointsToUpdate; - for (int x = 0; x < numberOfDataPointsToUpdate; x++) - { - for (int z = 0; z < numberOfDataPointsToUpdate; z++) - { - SingleColumnFullDataAccessor thisDataColumn = this.getOrCreate(offsetX + x, offsetZ + z); - SingleColumnFullDataAccessor inputDataColumn = inputSource.tryGet(inputFractionWidth * x, inputFractionWidth * z); - - if (inputDataColumn != null) - { - inputDataColumn.deepCopyTo(thisDataColumn); - } - } - } - } - - /** - * Attempts to convert this {@link IIncompleteFullDataSource} into a {@link CompleteFullDataSource}. - * - * @return a new {@link CompleteFullDataSource} if successful, returns itself if not. - */ - IFullDataSource tryPromotingToCompleteDataSource(); - - boolean hasBeenPromoted(); - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java deleted file mode 100644 index e1ce32bef..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/interfaces/IStreamableFullDataSource.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; - -import java.io.IOException; - -/** - * This interface holds the complete method list necessary for reading and writing a {@link IFullDataSource} - * to and from data streams.

- * - * This interface's purpose is to reduce the chance of accidentally mismatching read/write operation data types or content by splitting - * up each read/write method into small easy to understand chunks. - * - * @param defines the object holding this data source's summary data, extends {@link IStreamableFullDataSource.FullDataSourceSummaryData}. - * @param defines the object holding the data points, probably long[][] or long[][][]. - * {@link IStreamableFullDataSource#populateFromStream(DataSourceDto, DhDataInputStream, IDhLevel) populateFromStream} - * for the full reasoning. - */ -public interface IStreamableFullDataSource extends IFullDataSource -{ - - //=================// - // stream handling // - //=================// - - /** - * Clears and then overwrites any data in this object with the data from the given file and stream. - * This is expected to be used with an existing {@link IStreamableFullDataSource} and can be used in place of a constructor to reuse an existing {@link IStreamableFullDataSource} object. - * - * @see IStreamableFullDataSource#populateFromStream - */ - @Override - default void repopulateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException - { - // clear/overwrite the old data - this.resizeDataStructuresForRepopulation(dto.pos); - this.getMapping().clear(dto.pos); - - // set the new data - this.populateFromStream(dto, inputStream, level); - } - - /** - * Overwrites any data in this object with the data from the given file and stream. - * This is expected to be used with an empty {@link IStreamableFullDataSource} and functions similar to a constructor. - */ - @Override - default void populateFromStream(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException - { - SummaryDataType summaryData = this.readSourceSummaryInfo(dto, inputStream, level); - this.setSourceSummaryData(summaryData); - - - DataContainerType dataPoints = this.readDataPoints(dto, summaryData.dataWidth, inputStream); - if (dataPoints == null) - { - return; - } - this.setDataPoints(dataPoints); - - - FullDataPointIdMap mapping = this.readIdMappings(dataPoints, inputStream, level.getLevelWrapper()); - this.setIdMapping(mapping); - - } - - @Override - default void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException - { - this.writeSourceSummaryInfo(level, outputStream); - - boolean hasData = this.writeDataPoints(outputStream); - if (!hasData) - { - return; - } - - this.writeIdMappings(outputStream); - } - - - - /** Note: this should only be used if the data source is being reused. Normally data sources shouldn't change. */ - void resizeDataStructuresForRepopulation(DhSectionPos pos); - - /** - * Includes information about the source file that doesn't need to be saved in each data point. Like the source's size and y-level. - */ - void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException; - /** - * Confirms that the given {@link DataSourceDto} is valid for this {@link IStreamableFullDataSource}.
- * This specifically checks any fields that should be set when the {@link IStreamableFullDataSource} was first constructed. - * - * @throws IOException if the {@link DataSourceDto} isn't valid for this object. - */ - SummaryDataType readSourceSummaryInfo(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException; - void setSourceSummaryData(SummaryDataType summaryData); - - - /** @return true if any data points were present and written, false if this object was empty */ - boolean writeDataPoints(DhDataOutputStream outputStream) throws IOException; - /** @return null if no data points were present */ - DataContainerType readDataPoints(DataSourceDto dto, int width, DhDataInputStream inputStream) throws IOException; - void setDataPoints(DataContainerType dataPoints); - - - void writeIdMappings(DhDataOutputStream outputStream) throws IOException; - FullDataPointIdMap readIdMappings(DataContainerType dataPoints, DhDataInputStream inputStream, ILevelWrapper levelWrapper) throws IOException, InterruptedException; - void setIdMapping(FullDataPointIdMap mappings); - - - - //================// - // helper classes // - //================// - - /** - * This holds information that is relevant to the entire source and isn't stored in the data points.
- * Example: minimum height, detail level, source type, etc. - */ - class FullDataSourceSummaryData - { - public final int dataWidth; - public EDhApiWorldGenerationStep worldGenStep; - - - public FullDataSourceSummaryData(int dataWidth, EDhApiWorldGenerationStep worldGenStep) - { - this.dataWidth = dataWidth; - this.worldGenStep = worldGenStep; - } - - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 2bc453a88..fdebf427e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -20,14 +20,14 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhLodPos; +import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; @@ -61,8 +61,6 @@ public class ColumnRenderSource implements IDataSource public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } public static final String DATA_NAME = "ColumnRenderSource"; - @Override - public String getDataTypeName() { return DATA_NAME; } /** * This is the byte put between different sections in the binary save file. @@ -292,117 +290,60 @@ public class ColumnRenderSource implements IDataSource } @Override - public void update(ChunkSizedFullDataAccessor chunkDataView, IDhClientLevel level) + public void update(NewFullDataSource inputDataSource, IDhClientLevel level) { - final String errorMessagePrefix = "Unable to complete fastWrite for RenderSource pos: [" + this.sectionPos + "] and chunk pos: [" + chunkDataView.chunkPos + "]. Error:"; - - final DhSectionPos renderSourcePos = this.getSectionPos(); - - final int sourceBlockX = renderSourcePos.getMinCornerLodPos().getCornerBlockPos().x; - final int sourceBlockZ = renderSourcePos.getMinCornerLodPos().getCornerBlockPos().z; - - // offset between the incoming chunk data and this render source - final int blockOffsetX = (chunkDataView.chunkPos.x * LodUtil.CHUNK_WIDTH) - sourceBlockX; - final int blockOffsetZ = (chunkDataView.chunkPos.z * LodUtil.CHUNK_WIDTH) - sourceBlockZ; - - final int sourceDataPointBlockWidth = BitShiftUtil.powerOfTwo(this.getDataDetailLevel()); + final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputDataSource.getSectionPos() + "]. Error:"; boolean dataChanged = false; - - if (chunkDataView.detailLevel == this.getDataDetailLevel()) + if (inputDataSource.getSectionPos().getDetailLevel() == this.sectionPos.getDetailLevel()) { - this.markNotEmpty(); - // confirm the render source contains this chunk - if (blockOffsetX < 0 - || blockOffsetX + LodUtil.CHUNK_WIDTH > this.getWidthInDataPoints() - || blockOffsetZ < 0 - || blockOffsetZ + LodUtil.CHUNK_WIDTH > this.getWidthInDataPoints()) + try { - LOGGER.warn(errorMessagePrefix+"Data offset is out of bounds."); - return; - } - - - if (Thread.interrupted()) - { - LOGGER.warn(errorMessagePrefix+"write interrupted."); - return; - } - - - for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++) - { - for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++) + if (Thread.interrupted()) { - ColumnArrayView columnArrayView = this.getVerticalDataPointView(blockOffsetX + x, blockOffsetZ + z); - int hash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = chunkDataView.get(x, z); - FullDataToRenderDataTransformer.convertColumnData(level, - sourceBlockX + sourceDataPointBlockWidth * (blockOffsetX + x), - sourceBlockZ + sourceDataPointBlockWidth * (blockOffsetZ + z), - columnArrayView, fullArrayView); - dataChanged |= hash != columnArrayView.getDataHash(); + LOGGER.warn(errorMessagePrefix + "write interrupted."); + return; + } + + + + DhBlockPos2D centerBlockPos = inputDataSource.getSectionPos().getCenterBlockPos(); + int halfBlockWidth = inputDataSource.getSectionPos().getBlockWidth() / 2; + DhBlockPos2D minBlockPos = new DhBlockPos2D(centerBlockPos.x - halfBlockWidth, centerBlockPos.z - halfBlockWidth); + + for (int x = 0; x < NewFullDataSource.WIDTH; x++) + { + for (int z = 0; z < NewFullDataSource.WIDTH; z++) + { + ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); + int columnHash = columnArrayView.getDataHash(); + + SingleColumnFullDataAccessor fullArrayView = inputDataSource.get(x, z); + EDhApiWorldGenerationStep worldGenStep = inputDataSource.getWorldGenStepAtRelativePos(x, z); + if (fullArrayView != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) + { + FullDataToRenderDataTransformer.convertColumnData(level, + minBlockPos.x + x, + minBlockPos.z + z, + columnArrayView, fullArrayView); + dataChanged |= columnHash != columnArrayView.getDataHash(); + + this.fillDebugFlag(x, z, 1, 1, ColumnRenderSource.DebugSourceFlag.DIRECT); + } + } } } - this.fillDebugFlag(blockOffsetX, blockOffsetZ, LodUtil.CHUNK_WIDTH, LodUtil.CHUNK_WIDTH, ColumnRenderSource.DebugSourceFlag.DIRECT); - } - else if (chunkDataView.detailLevel < this.getDataDetailLevel() && this.getDataDetailLevel() <= chunkDataView.getSectionPos().getDetailLevel()) - { - this.markNotEmpty(); - // multiple chunk data points converting to 1 column data point - DhLodPos dataCornerPos = chunkDataView.getSectionPos().getMinCornerLodPos(chunkDataView.detailLevel); - DhLodPos sourceCornerPos = renderSourcePos.getMinCornerLodPos(this.getDataDetailLevel()); - DhLodPos sourceStartingChangePos = dataCornerPos.convertToDetailLevel(this.getDataDetailLevel()); - int relStartX = Math.floorMod(sourceStartingChangePos.x, this.getWidthInDataPoints()); - int relStartZ = Math.floorMod(sourceStartingChangePos.z, this.getWidthInDataPoints()); - int dataToSourceScale = sourceCornerPos.getWidthAtDetail(chunkDataView.detailLevel); - int columnsInChunk = chunkDataView.getSectionPos().getWidthCountForLowerDetailedSection(this.getDataDetailLevel()); - - for (int xOffset = 0; xOffset < columnsInChunk; xOffset++) + catch (Exception e) { - for (int zOffset = 0; zOffset < columnsInChunk; zOffset++) - { - int relSourceX = relStartX + xOffset; - int relSourceZ = relStartZ + zOffset; - ColumnArrayView columnArrayView = this.getVerticalDataPointView(relSourceX, relSourceZ); - int hash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = chunkDataView.get(xOffset * dataToSourceScale, zOffset * dataToSourceScale); - FullDataToRenderDataTransformer.convertColumnData(level, - sourceBlockX + sourceDataPointBlockWidth * relSourceX, - sourceBlockZ + sourceDataPointBlockWidth * relSourceZ, - columnArrayView, fullArrayView); - dataChanged |= hash != columnArrayView.getDataHash(); - } + LOGGER.error(errorMessagePrefix + e.getMessage(), e); } - this.fillDebugFlag(relStartX, relStartZ, columnsInChunk, columnsInChunk, ColumnRenderSource.DebugSourceFlag.DIRECT); - } - else if (chunkDataView.getSectionPos().getDetailLevel() < this.getDataDetailLevel()) - { - // The entire chunk is being converted to a single column data point, possibly. - DhLodPos dataCornerPos = chunkDataView.getSectionPos().getMinCornerLodPos(chunkDataView.detailLevel); - DhLodPos sourceCornerPos = renderSourcePos.getMinCornerLodPos(this.getDataDetailLevel()); - DhLodPos sourceStartingChangePos = dataCornerPos.convertToDetailLevel(this.getDataDetailLevel()); - int chunksPerColumn = sourceStartingChangePos.getWidthAtDetail(chunkDataView.getSectionPos().getDetailLevel()); - if (chunkDataView.getSectionPos().getX() % chunksPerColumn != 0 || chunkDataView.getSectionPos().getZ() % chunksPerColumn != 0) - { - return; // not a multiple of the column size, so no change - } - int relStartX = Math.floorMod(sourceStartingChangePos.x, this.getWidthInDataPoints()); - int relStartZ = Math.floorMod(sourceStartingChangePos.z, this.getWidthInDataPoints()); - ColumnArrayView columnArrayView = this.getVerticalDataPointView(relStartX, relStartZ); - int hash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = chunkDataView.get(0, 0); - FullDataToRenderDataTransformer.convertColumnData(level, dataCornerPos.x * sourceDataPointBlockWidth, - dataCornerPos.z * sourceDataPointBlockWidth, - columnArrayView, fullArrayView); - dataChanged = hash != columnArrayView.getDataHash(); - this.fillDebugFlag(relStartX, relStartZ, 1, 1, ColumnRenderSource.DebugSourceFlag.DIRECT); } if (dataChanged) { this.localVersion.incrementAndGet(); + this.markNotEmpty(); } } @@ -432,7 +373,9 @@ public class ColumnRenderSource implements IDataSource public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; } + public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; } + @Override + public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { return EDhApiWorldGenerationStep.EMPTY; } /** @return how many data points wide this {@link ColumnRenderSource} is. */ public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); } @@ -493,7 +436,6 @@ public class ColumnRenderSource implements IDataSource this.debugSourceFlags[x * SECTION_SIZE + z] = flag; } } - localVersion.incrementAndGet(); } public DebugSourceFlag debugGetFlag(int ox, int oz) { return this.debugSourceFlags[ox * SECTION_SIZE + oz]; } @@ -548,7 +490,6 @@ public class ColumnRenderSource implements IDataSource { FULL(ColorUtil.BLUE), DIRECT(ColorUtil.WHITE), - SPARSE(ColorUtil.YELLOW), FILE(ColorUtil.BROWN); public final int color; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java index 538dac67d..94caa4724 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.sql.DataSourceDto; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -31,7 +31,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; /** - * Handles loading and parsing {@link DataSourceDto}s to create {@link ColumnRenderSource}s.

+ * Handles loading and parsing {@link LegacyDataSourceDTO}s to create {@link ColumnRenderSource}s.

* * Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what * file versions this class can handle. @@ -48,7 +48,7 @@ public class ColumnRenderSourceLoader - public ColumnRenderSource loadRenderSource(DataSourceDto dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataFileVersion = dto.binaryDataFormatVersion; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index 3fd433a62..8d27273c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -23,7 +23,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.pos.DhChunkPos; @@ -57,7 +57,7 @@ public class ChunkToLodBuilder implements AutoCloseable // data generation // //=================// - public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) + public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) { if (chunkWrapper == null) { @@ -74,7 +74,7 @@ public class ChunkToLodBuilder implements AutoCloseable } // Otherwise, it means we're the first to do so. Let's submit our task to this entry. - CompletableFuture future = new CompletableFuture<>(); + CompletableFuture future = new CompletableFuture<>(); this.concurrentTaskToBuildList.addLast(new Task(chunkWrapper.getChunkPos(), future)); return future; } @@ -158,10 +158,10 @@ public class ChunkToLodBuilder implements AutoCloseable { if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - ChunkSizedFullDataAccessor data = LodDataBuilder.createChunkData(latestChunk); - if (data != null) + NewFullDataSource dataSource = LodDataBuilder.createGeneratedDataSource(latestChunk); + if (dataSource != null) { - task.future.complete(data); + task.future.complete(dataSource); continue; } } @@ -233,11 +233,11 @@ public class ChunkToLodBuilder implements AutoCloseable private static class Task { public final DhChunkPos chunkPos; - public final CompletableFuture future; + public final CompletableFuture future; /** This is tracked so impossible tasks can be removed from the queue */ public long generationAttemptExpirationTimeMs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); - Task(DhChunkPos chunkPos, CompletableFuture future) + Task(DhChunkPos chunkPos, CompletableFuture future) { this.chunkPos = chunkPos; this.future = future; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index a3b3d54d1..6c6451ecb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -22,11 +22,8 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import com.seibel.distanthorizons.api.enums.config.EBlocksToAvoid; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -35,7 +32,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -46,8 +42,7 @@ import org.apache.logging.log4j.Logger; import java.util.HashSet; /** - * Handles converting {@link ChunkSizedFullDataAccessor}, {@link IIncompleteFullDataSource}, - * and {@link IFullDataSource}'s to {@link ColumnRenderSource}. + * Handles converting {@link NewFullDataSource}'s to {@link ColumnRenderSource}. */ public class FullDataToRenderDataTransformer { @@ -62,7 +57,7 @@ public class FullDataToRenderDataTransformer // public transformer interface // //==============================// - public static ColumnRenderSource transformFullDataToRenderSource(IFullDataSource fullDataSource, IDhClientLevel level) + public static ColumnRenderSource transformFullDataToRenderSource(NewFullDataSource fullDataSource, IDhClientLevel level) { if (fullDataSource == null) { @@ -77,17 +72,7 @@ public class FullDataToRenderDataTransformer try { - if (fullDataSource instanceof CompleteFullDataSource) - { - return transformCompleteFullDataToColumnData(level, (CompleteFullDataSource) fullDataSource); - } - else if (fullDataSource instanceof IIncompleteFullDataSource) - { - return transformIncompleteFullDataToColumnData(level, (IIncompleteFullDataSource) fullDataSource); - } - - LodUtil.assertNotReach("Unimplemented Full Data transformer for "+IFullDataSource.class.getSimpleName()+" of type ["+fullDataSource.getClass().getSimpleName()+"]."); - return null; + return transformCompleteFullDataToColumnData(level, fullDataSource); } catch (InterruptedException e) { @@ -108,7 +93,7 @@ public class FullDataToRenderDataTransformer * @throws InterruptedException Can be caused by interrupting the thread upstream. * Generally thrown if the method is running after the client leaves the current world. */ - private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, CompleteFullDataSource fullDataSource) throws InterruptedException + private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, NewFullDataSource fullDataSource) throws InterruptedException { final DhSectionPos pos = fullDataSource.getSectionPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); @@ -149,56 +134,6 @@ public class FullDataToRenderDataTransformer return columnSource; } - /** - * @throws InterruptedException Can be caused by interrupting the thread upstream. - * Generally thrown if the method is running after the client leaves the current world. - */ - private static ColumnRenderSource transformIncompleteFullDataToColumnData(IDhClientLevel level, IIncompleteFullDataSource data) throws InterruptedException - { - final DhSectionPos pos = data.getSectionPos(); - final byte dataDetail = data.getDataDetailLevel(); - final int vertSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetailLevel()); - final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY()); - if (data.isEmpty()) - { - return columnSource; - } - - columnSource.markNotEmpty(); - - if (dataDetail == columnSource.getDataDetailLevel()) - { - int baseX = pos.getMinCornerLodPos().getCornerBlockPos().x; - int baseZ = pos.getMinCornerLodPos().getCornerBlockPos().z; - - int width = pos.getWidthCountForLowerDetailedSection(dataDetail); - for (int x = 0; x < width; x++) - { - for (int z = 0; z < width; z++) - { - throwIfThreadInterrupted(); - - SingleColumnFullDataAccessor fullArrayView = data.tryGet(x, z); - if (fullArrayView == null) - { - continue; - } - - ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView); - - columnSource.fillDebugFlag(x, z, 1, 1, ColumnRenderSource.DebugSourceFlag.SPARSE); - } - } - } - else - { - throw new UnsupportedOperationException("To be implemented"); - //FIXME: Implement different size creation of renderData - } - return columnSource; - } - //================// @@ -222,7 +157,9 @@ public class FullDataToRenderDataTransformer // TODO what does this mean? - private static void iterateAndConvert(IDhClientLevel level, int blockX, int blockZ, ColumnArrayView column, SingleColumnFullDataAccessor data) + private static void iterateAndConvert(IDhClientLevel level, + int blockX, int blockZ, + ColumnArrayView column, SingleColumnFullDataAccessor data) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); @@ -335,7 +272,7 @@ public class FullDataToRenderDataTransformer // TODO what does this mean? public static void convertColumnData(IDhClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleColumnFullDataAccessor fullArrayView) { - if (!fullArrayView.doesColumnExist()) + if (fullArrayView == null || !fullArrayView.doesColumnExist()) { return; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 8ac100f70..65c043fda 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -21,12 +21,14 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import java.util.List; +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -40,6 +42,8 @@ public class LodDataBuilder { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IBlockStateWrapper AIR = SingletonInjector.INSTANCE.get(IWrapperFactory.class).getAirBlockStateWrapper(); + /** how many chunks wide the {@link NewFullDataSource} is. */ + private static final int NUMB_OF_CHUNKS_WIDE = NewFullDataSource.WIDTH / LodUtil.CHUNK_WIDTH; private static boolean getTopErrorLogged = false; @@ -49,7 +53,7 @@ public class LodDataBuilder // converters // //============// - public static ChunkSizedFullDataAccessor createChunkData(IChunkWrapper chunkWrapper) + public static NewFullDataSource createGeneratedDataSource(IChunkWrapper chunkWrapper) { if (!canGenerateLodFromChunk(chunkWrapper)) { @@ -57,26 +61,89 @@ public class LodDataBuilder } - ChunkSizedFullDataAccessor chunkData = new ChunkSizedFullDataAccessor(chunkWrapper.getChunkPos()); - int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); - for (int x = 0; x < LodUtil.CHUNK_WIDTH; x++) + // get the section position + int sectionPosX = chunkWrapper.getChunkPos().x; + // negative positions start at -1 so the logic there is slightly different + sectionPosX = (sectionPosX < 0) ? ((sectionPosX + 1) / NUMB_OF_CHUNKS_WIDE) - 1 : (sectionPosX / NUMB_OF_CHUNKS_WIDE); + int sectionPosZ = chunkWrapper.getChunkPos().z; + sectionPosZ = (sectionPosZ < 0) ? ((sectionPosZ + 1) / NUMB_OF_CHUNKS_WIDE) - 1 : (sectionPosZ / NUMB_OF_CHUNKS_WIDE); + DhSectionPos pos = new DhSectionPos(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ); + + NewFullDataSource dataSource = NewFullDataSource.createEmpty(pos); + dataSource.markNotEmpty(); + + + + // compute the chunk dataSource offset + // this offset is used to determine where in the dataSource this chunk's data should go + int chunkOffsetX = chunkWrapper.getChunkPos().x; + if (chunkWrapper.getChunkPos().x < 0) { - for (int z = 0; z < LodUtil.CHUNK_WIDTH; z++) + // expected offset positions: + // chunkPos -> offset + // 5 -> 1 + // 4 -> 0 --- + // 3 -> 3 + // 2 -> 2 + // 1 -> 1 + // 0 -> 0 === + // -1 -> 3 + // -2 -> 2 + // -3 -> 1 + // -4 -> 0 --- + // -5 -> 3 + chunkOffsetX = ((chunkOffsetX) % NUMB_OF_CHUNKS_WIDE); + if (chunkOffsetX != 0) + { + chunkOffsetX += NUMB_OF_CHUNKS_WIDE; + } + } + else + { + chunkOffsetX %= NUMB_OF_CHUNKS_WIDE; + } + chunkOffsetX *= LodUtil.CHUNK_WIDTH; + + int chunkOffsetZ = chunkWrapper.getChunkPos().z; + if (chunkWrapper.getChunkPos().z < 0) + { + chunkOffsetZ = ((chunkOffsetZ) % NUMB_OF_CHUNKS_WIDE); + if (chunkOffsetZ != 0) + { + chunkOffsetZ += NUMB_OF_CHUNKS_WIDE; + } + } + else + { + chunkOffsetZ %= NUMB_OF_CHUNKS_WIDE; + } + chunkOffsetZ *= LodUtil.CHUNK_WIDTH; + + + + //==========================// + // populate the data source // + //==========================// + + int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); + for (int chunkX = 0; chunkX < LodUtil.CHUNK_WIDTH; chunkX++) + { + for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++) { LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); int lastY = chunkWrapper.getMaxBuildHeight(); - IBiomeWrapper biome = chunkWrapper.getBiome(x, lastY, z); + IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; - int mappedId = chunkData.getMapping().addIfNotPresentAndGetId(biome, blockState); - // FIXME: The +1 offset to reproduce the old behavior. Remove this when we get per-face lighting - byte light = (byte) ((chunkWrapper.getBlockLight(x, lastY + 1, z) << 4) + chunkWrapper.getSkyLight(x, lastY + 1, z)); + int mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); + // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting + byte light = (byte) ((chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ) << 4) + chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ)); // determine the starting Y Pos - int y = chunkWrapper.getLightBlockingHeightMapValue(x,z); + int y = chunkWrapper.getLightBlockingHeightMapValue(chunkX,chunkZ); // go up until we reach open air or the world limit - IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(x, y, z); + IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) { try @@ -84,13 +151,13 @@ public class LodDataBuilder // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. y++; - topBlockState = chunkWrapper.getBlockState(x, y, z); + topBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); } catch (Exception e) { if (!getTopErrorLogged) { - LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + x + "," + y + "," + z + "] error: " + e.getMessage(), e); + LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + chunkX + "," + y + "," + chunkZ + "] error: " + e.getMessage(), e); getTopErrorLogged = true; } @@ -102,39 +169,38 @@ public class LodDataBuilder for (; y >= minBuildHeight; y--) { - IBiomeWrapper newBiome = chunkWrapper.getBiome(x, y, z); - IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(x, y, z); - byte newLight = (byte) ((chunkWrapper.getBlockLight(x, y + 1, z) << 4) + chunkWrapper.getSkyLight(x, y + 1, z)); + IBiomeWrapper newBiome = chunkWrapper.getBiome(chunkX, y, chunkZ); + IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); + byte newLight = (byte) ((chunkWrapper.getBlockLight(chunkX, y + 1, chunkZ) << 4) + chunkWrapper.getSkyLight(chunkX, y + 1, chunkZ)); if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) { longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), light)); biome = newBiome; blockState = newBlockState; - mappedId = chunkData.getMapping().addIfNotPresentAndGetId(biome, blockState); + mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); light = newLight; lastY = y; } -// else if (newLight != light) { -// longs.add(FullFormat.encode(mappedId, lastY-y, y+1 - chunk.getMinBuildHeight(), light)); -// light = newLight; -// lastY = y; -// } } longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), light)); - chunkData.setSingleColumn(longs.toLongArray(), x, z); + dataSource.setSingleColumn(longs.toLongArray(), + chunkX + chunkOffsetX, + chunkZ + chunkOffsetZ, + EDhApiWorldGenerationStep.LIGHT); } } - if (!canGenerateLodFromChunk(chunkWrapper)) return null; - LodUtil.assertTrue(chunkData.emptyCount() == 0); - return chunkData; + + LodUtil.assertTrue(!dataSource.isEmpty()); + return dataSource; } + /** @throws ClassCastException if an API user returns the wrong object type(s) */ - public static ChunkSizedFullDataAccessor createApiChunkData(DhApiChunk dataPoints) throws ClassCastException + public static NewFullDataSource createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException { - ChunkSizedFullDataAccessor accessor = new ChunkSizedFullDataAccessor(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ)); + NewFullDataSource accessor = NewFullDataSource.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ))); for (int relZ = 0; relZ < LodUtil.CHUNK_WIDTH; relZ++) { for (int relX = 0; relX < LodUtil.CHUNK_WIDTH; relX++) @@ -156,23 +222,23 @@ public class LodDataBuilder int id = accessor.getMapping().addIfNotPresentAndGetId( (IBiomeWrapper) (dataPoint.biomeWrapper), (IBlockStateWrapper) (dataPoint.blockStateWrapper) - ); + ); packedDataPoints[index] = FullDataPointUtil.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, (byte) (dataPoint.lightLevel) - ); + ); } - accessor.setSingleColumn(packedDataPoints, relX, relZ); + accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); } } return accessor; } - + //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java similarity index 85% rename from core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 30780d27a..5b7a4f97f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -1,11 +1,13 @@ package com.seibel.distanthorizons.core.file; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.*; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; @@ -29,7 +31,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; -public abstract class AbstractDataSourceHandler, TDhLevel extends IDhLevel> implements ISourceProvider +public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> + implements ISourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); @@ -54,7 +57,7 @@ public abstract class AbstractDataSourceHandler updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkDataView) + public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) { ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) @@ -199,11 +202,11 @@ public abstract class AbstractDataSourceHandler { - DhSectionPos bottomPos = chunkDataView.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + DhSectionPos bottomPos = inputDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); bottomPos.forEachPosUpToDetailLevel( this.topSectionDetailLevelRef.byteValue(), - (pos) -> this.updateDataSourceAtPos(pos, chunkDataView) ); + (pos) -> this.updateDataSourceAtPos(pos, inputDataSource) ); }, executor); } @@ -213,7 +216,7 @@ public abstract class AbstractDataSourceHandler
* * This prevents repeatedly reading/writing the same data source to/from disk if said @@ -279,17 +282,17 @@ public abstract class AbstractDataSourceHandler, + TDTO extends IBaseDTO, + TDhLevel extends IDhLevel> + implements ISourceProvider +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); + /** How long a data source must remain un-modified before being written to disk. */ + private static final int SAVE_DELAY_IN_MS = 4_000; + + /** + * The highest numerical detail level possible. + * Used when determining which positions to update. + * + * @see AbstractNewDataSourceHandler#MIN_SECTION_DETAIL_LEVEL + */ + public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL; // TODO add "section" to detail level + /** + * The lowest numerical detail level possible. + * + * @see AbstractNewDataSourceHandler#TOP_SECTION_DETAIL_LEVEL + * */ + public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + + protected final ReentrantLock[] updateLockArray; + protected final ReentrantLock closeLock = new ReentrantLock(); + protected volatile boolean isShutdown = false; + + protected final TDhLevel level; + protected final File saveDir; + + public final AbstractDhRepo repo; + + public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); + + + + //=============// + // constructor // + //=============// + + public AbstractNewDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public AbstractNewDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + { + this.level = level; + this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; + if (!this.saveDir.exists() && !this.saveDir.mkdirs()) + { + LOGGER.warn("Unable to create full data folder, file saving may fail."); + } + + // the lock array's length is double the number of CPU cores so the number of collisions + // should be relatively low without having too many extra locks + int lockCount = Runtime.getRuntime().availableProcessors() * 2; + this.updateLockArray = new ReentrantLock[lockCount]; + for (int i = 0; i < lockCount; i++) + { + this.updateLockArray[i] = new ReentrantLock(); + } + + this.repo = this.createRepo(); + } + + + + + //==================// + // abstract methods // + //==================// + + /** When this is called the parent folders should be created */ + protected abstract AbstractDhRepo createRepo(); + + protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException; + protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); + + /** Creates a new data source using any DTOs already present in the database. */ + protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos); + + protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); + + + + //==============// + // data reading // + //==============// + + /** + * Returns the {@link TDataSource} for the given section position.
+ * The returned data source may be null if there was a problem.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. + */ + @Override + public CompletableFuture getAsync(DhSectionPos pos) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + /** + * Should only be used in internal file handler methods where we are already running on a file handler thread. + * Can return null if there was a problem. + * @see AbstractNewDataSourceHandler#getAsync(DhSectionPos) + */ + public TDataSource get(DhSectionPos pos) + { + TDataSource dataSource = null; + try + { + TDTO dto = this.repo.getByKey(pos); + if (dto != null) + { + // load from file + dataSource = this.createDataSourceFromDto(dto); + } + else + { + // attempt to create from any existing files + dataSource = this.createNewDataSourceFromExistingDtos(pos); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); + } + + return dataSource; + } + + + + //===============// + // data updating // + //===============// + + @Override + public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) + { + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + + try + { + // run file handling on a separate thread + return CompletableFuture.runAsync(() -> + { + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource, true); + + }, executor); + } + catch (RejectedExecutionException ignore) + { + // can happen if the executor was shutdown while this task was queued + return CompletableFuture.completedFuture(null); + } + } + /** + * @param pos the position to update + * @param lockOnPosition Can be disabled by inheriting children to allow for their own locking logic. + * This is important if the child has its own position specific logic that shouldn't be done concurrently. + */ + protected void updateDataSourceAtPos(DhSectionPos pos, NewFullDataSource inputData, boolean lockOnPosition) + { + // a lock is necessary to prevent two threads from writing to the same position at once, + // if that happens only the second update will apply and the LOD will end up with hole(s) + ReentrantLock updateLock = this.getUpdateLockForPos(pos); + + try + { + if (lockOnPosition) + { + updateLock.lock(); + } + + + // get or create the data source + TDataSource dataSource = this.get(pos); + dataSource.update(inputData, this.level); // TODO only write to database and send update signal if data was changed + + + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(dataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + { + if (listener != null) + { + listener.OnDataSourceUpdated(dataSource); + } + } + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + } + finally + { + if (lockOnPosition) + { + updateLock.unlock(); + } + } + } + + + + //================// + // helper methods // + //================// + + /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ + protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } + + + + //=========// + // cleanup // + //=========// + + @Override + public void close() + { + try + { + this.closeLock.lock(); + this.isShutdown = true; + + // wait a moment so any queued saves can finish queuing, + // otherwise we might not see everything that needs saving and attempt to use a closed repo + Thread.sleep(200); + + LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "]."); + + this.repo.close(); + } + catch (InterruptedException ignore) { } + finally + { + this.closeLock.unlock(); + } + } + + + + //================// + // helper classes // + //================// + + @FunctionalInterface + public interface IDataSourceUpdateFunc + { + void OnDataSourceUpdated(TDataSource updatedFullDataSource); + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index d1ab9424c..5a61dd2e7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -1,12 +1,11 @@ package com.seibel.distanthorizons.core.file; +import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import java.io.IOException; @@ -14,11 +13,10 @@ import java.io.IOException; /** * Base for all data sources. * - * @param what type of level this data source can be created from + * @param there are times when we need specifically a client level vs a more generic level */ public interface IDataSource extends IBaseDTO { - DhSectionPos getSectionPos(); @@ -27,8 +25,10 @@ public interface IDataSource extends IBaseDTO extends IBaseDTO - * Primarily by {@link AbstractFullDataSourceLoader#getLoader(String, byte)} to determine how to parse - * the binary data when read from file. + /** + * Returns the detail level of the data contained by this data source. + * IE: 0 for block, 1 for 2x2 blocks, etc. + * + * @see EDhApiDetailLevel */ - String getDataTypeName(); - /** Defines how the binary data is formatted and which {@link AbstractFullDataSourceLoader} should be used when loading from file. */ + byte getDataDetailLevel(); + + @Deprecated // TODO only necessary for full data sources + EDhApiWorldGenerationStep getWorldGenStep(); + EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ); + + /** Defines how the binary data is formatted. */ byte getDataFormatVersion(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java index 1d2b15ef4..cd9d3c363 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java @@ -1,15 +1,23 @@ package com.seibel.distanthorizons.core.file; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.util.concurrent.CompletableFuture; +/** + * Base for all data source providers + * + * @see IFullDataSourceProvider + * @see IRenderSourceProvider + */ public interface ISourceProvider, TDhLevel extends IDhLevel> extends AutoCloseable { CompletableFuture getAsync(DhSectionPos pos); - CompletableFuture updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkData); + CompletableFuture updateDataSourceAsync(NewFullDataSource inputData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java deleted file mode 100644 index 96ff63a03..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandler.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.fullDatafile; - -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.loader.AbstractFullDataSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.HighDetailIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.LowDetailIncompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IIncompleteFullDataSource; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.AbstractDataSourceRepo; -import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.awt.*; -import java.io.File; -import java.io.IOException; -import java.sql.SQLException; -import java.util.*; - -public class FullDataFileHandler - extends AbstractDataSourceHandler - implements IFullDataSourceProvider, IDebugRenderable -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - - //=============// - // constructor // - //=============// - - public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public FullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) - { - super(level, saveStructure, saveDirOverride); - - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); - } - - - - //====================// - // Abstract overrides // - //====================// - - @Override - protected AbstractDataSourceRepo createRepo() - { - try - { - return new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); - } - catch (SQLException e) - { - // should only happen if there is an issue with the database (it's locked or can't be created if missing) - // or the database update failed - throw new RuntimeException(e); - } - } - - @Override - protected IFullDataSource createDataSourceFromDto(DataSourceDto dto) throws InterruptedException, IOException - { - AbstractFullDataSourceLoader loader = AbstractFullDataSourceLoader.getLoader(dto.dataType, dto.binaryDataFormatVersion); - IFullDataSource dataSource = loader.loadDataSource(dto, this.level); - return dataSource; - } - /** Creates a new data source using any DTOs already present in the database. */ - @Override - protected IFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) - { - IIncompleteFullDataSource newFullDataSource = this.makeEmptyDataSource(pos); - return this.updateFromDataSourceFromExistingDtos(newFullDataSource); - } - protected IFullDataSource updateFromDataSourceFromExistingDtos(IIncompleteFullDataSource newFullDataSource) - { - DhSectionPos pos = newFullDataSource.getSectionPos(); - - boolean showFullDataFileSampling = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); - if (showFullDataFileSampling) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA), - 0.2, 32f)); - } - - - // get all non-empty sections to sample from - ArrayList samplePosList = new ArrayList<>(); - ArrayList possibleChildList = new ArrayList<>(); - pos.forEachChild((childPos) -> - { - if (childPos.getDetailLevel() >= this.minDetailLevel) - { - possibleChildList.add(childPos); - } - }); - while (possibleChildList.size() != 0) - { - DhSectionPos possiblePos = possibleChildList.remove(possibleChildList.size()-1); - if (this.repo.existsWithKey(possiblePos)) - { - samplePosList.add(possiblePos); - } - else - { - possiblePos.forEachChild((childPos) -> - { - if (childPos.getDetailLevel() >= this.minDetailLevel) - { - possibleChildList.add(childPos); - } - }); - } - } - - - // read in the existing data - for (int i = 0; i < samplePosList.size(); i++) - { - DhSectionPos samplePos = samplePosList.get(i); - IFullDataSource sampleDataSource = this.get(samplePos); - if (sampleDataSource == null) - { - // no file was found, this is unexpected, but can be ignored - continue; - } - - if (showFullDataFileSampling) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(newFullDataSource.getSectionPos(), 64f, 72f, 0.03f, Color.MAGENTA.darker()), - 0.2, 32f)); - } - - try - { - newFullDataSource.sampleFrom(sampleDataSource); - } - catch (Exception e) - { - LOGGER.warn("Unable to sample "+sampleDataSource.getSectionPos()+" into "+newFullDataSource.getSectionPos(), e); - } - } - - - // promotion may happen if all children are fully populated - return newFullDataSource.tryPromotingToCompleteDataSource(); - } - - - - @Override - protected IIncompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) - { - return pos.getDetailLevel() <= HighDetailIncompleteFullDataSource.MAX_SECTION_DETAIL ? - HighDetailIncompleteFullDataSource.createEmpty(pos) : - LowDetailIncompleteFullDataSource.createEmpty(pos); - } - - - - //===================// - // extension methods // - //===================// - - @Override - public void writeDataSourceToFile(IFullDataSource fullDataSource) throws IOException - { - // doing this here guarantees that all changes are caught and promoted - if (fullDataSource instanceof IIncompleteFullDataSource) - { - fullDataSource = ((IIncompleteFullDataSource) fullDataSource).tryPromotingToCompleteDataSource(); - } - - super.writeDataSourceToFile(fullDataSource); - - // save has completed - boolean showFullDataFileStatus = Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get(); - if (showFullDataFileStatus) - { - DebugRenderer.makeParticle(new DebugRenderer.BoxParticle( - new DebugRenderer.Box(fullDataSource.getSectionPos(), 64f, 70f, 0.02f, Color.YELLOW), - 0.2, 16f)); - } - } - - @Override - public int getUnsavedDataSourceCount() { return this.unsavedDataSourceBySectionPos.size(); } - - - - //===========// - // overrides // - //===========// - - @Override - public void debugRender(DebugRenderer renderer) - { - this.saveTimerTasksBySectionPos.keySet() - .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); - - } - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index b487b6a20..1c6e40ccd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -19,28 +19,33 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.ISourceProvider; -import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.FullDataRepo; +import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; import java.util.concurrent.CompletableFuture; /** - * Handles reading, writing, and updating {@link IFullDataSource}'s.
+ * Handles reading, writing, and updating {@link NewFullDataSource}'s.
* Should be backed by a database handled by a {@link FullDataRepo}. */ -public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable +public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable { - CompletableFuture getAsync(DhSectionPos pos); - IFullDataSource get(DhSectionPos pos); + CompletableFuture getAsync(DhSectionPos pos); + NewFullDataSource get(DhSectionPos pos); - CompletableFuture updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkData); + /** + * If this provider has the ability to create (world gen) or get (networking) + * missing data sources this method will queue the given position + * for generation or retrieval. + */ + void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos); + CompletableFuture updateDataSourceAsync(NewFullDataSource chunkData); + + @Deprecated int getUnsavedDataSourceCount(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java new file mode 100644 index 000000000..95a1e06c4 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java @@ -0,0 +1,134 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.file.fullDatafile; + +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; +import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; + +import java.awt.*; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; + +public class LegacyFullDataFileHandler + extends AbstractLegacyDataSourceHandler + implements IDebugRenderable +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + + + //=============// + // constructor // + //=============// + + public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + { + super(level, saveStructure, saveDirOverride); + + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); + } + + + + //====================// + // Abstract overrides // + //====================// + + @Override + protected AbstractLegacyDataSourceRepo createRepo() + { + try + { + return new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + } + catch (SQLException e) + { + // should only happen if there is an issue with the database (it's locked or can't be created if missing) + // or the database update failed + throw new RuntimeException(e); + } + } + + @Override + protected CompleteFullDataSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException + { + CompleteFullDataSource dataSource = CompleteFullDataSource.createEmpty(dto.pos); + dataSource.populateFromStream(dto, dto.getInputStream(), this.level); + return dataSource; + } + /** Creates a new data source using any DTOs already present in the database. */ + @Deprecated + @Override + protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) + { + throw new UnsupportedOperationException("Deprecated"); + } + + + @Deprecated + @Override + protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) + { + throw new UnsupportedOperationException("Deprecated"); + } + + + + //===================// + // extension methods // + //===================// + + @Deprecated + @Override + public void writeDataSourceToFile(CompleteFullDataSource fullDataSource) throws IOException + { + throw new UnsupportedOperationException("Deprecated"); + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + this.saveTimerTasksBySectionPos.keySet() + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); + + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java new file mode 100644 index 000000000..6754f1d00 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -0,0 +1,276 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.file.fullDatafile; + +import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; +import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; +import com.seibel.distanthorizons.core.util.ThreadUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; + +import java.awt.*; +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +public class NewFullDataFileHandler + extends AbstractNewDataSourceHandler + implements IFullDataSourceProvider, IDebugRenderable +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + private static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 20; + /** how many parent update tasks can be in the queue at once */ + private static final int MAX_PARENT_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); + + /** indicates how long the update queue thread should wait between queuing ticks */ + private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 1_000; + + + // TODO add a debug view + Set parentApplicationPositionSet = ConcurrentHashMap.newKeySet(); + private final ThreadPoolExecutor updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Update Queue Processor"); + private final AtomicBoolean updateQueueThreadRunningRef = new AtomicBoolean(false); + + + + //=============// + // constructor // + //=============// + + public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + { + super(level, saveStructure, saveDirOverride); + + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); + } + + + + //====================// + // Abstract overrides // + //====================// + + @Override + protected AbstractDhRepo createRepo() + { + try + { + return new NewFullDataSourceRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + } + catch (SQLException e) + { + // should only happen if there is an issue with the database (it's locked or the folder path is missing) + // or the database update failed + throw new RuntimeException(e); + } + } + + @Override + protected NewFullDataSourceDTO createDtoFromDataSource(NewFullDataSource dataSource) + { + try + { + return NewFullDataSourceDTO.CreateFromDataSource(dataSource); + } + catch (IOException e) + { + LOGGER.warn("Unable to create DTO, error: "+e.getMessage(), e); + return null; + } + } + + @Override + protected NewFullDataSource createDataSourceFromDto(NewFullDataSourceDTO dto) throws InterruptedException, IOException + { return dto.createDataSource(this.level.getLevelWrapper()); } + @Override + protected NewFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) + { + // TODO maybe just set children update flags to true? + return NewFullDataSource.createEmpty(pos); + } + + @Override + protected NewFullDataSource makeEmptyDataSource(DhSectionPos pos) { return NewFullDataSource.createEmpty(pos); } + + @Deprecated + @Override + public int getUnsavedDataSourceCount() { return 0; } + + + + //================// + // parent updates // + //================// + + @Override + public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) + { + // Do nothing. + // This file handler doesn't have the ability to generate or retrieve data sources + // that aren't already in the database + } + + @Override + protected void updateDataSourceAtPos(DhSectionPos pos, NewFullDataSource inputData, boolean lockOnPosition) + { + ReentrantLock updateLock = this.getUpdateLockForPos(pos); + + try + { + if (lockOnPosition) + { + updateLock.lock(); + } + + super.updateDataSourceAtPos(pos, inputData, false); + this.tryQueueParentUpdates(); + + this.parentApplicationPositionSet.remove(inputData.getSectionPos()); + if (pos.getDetailLevel() != inputData.getSectionPos().getDetailLevel()) + { + // mark that the update has completed + ((NewFullDataSourceRepo) this.repo).setApplyToParent(inputData.getSectionPos(), false); // TODO remove casting + } + } + catch (Exception e) + { + LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + } + finally + { + if (lockOnPosition) + { + updateLock.unlock(); + } + } + } + /** Queues some of the parent updates listed in the database. */ + private void tryQueueParentUpdates() + { + // the update thread is already running, + // we don't need multiple running + if (this.updateQueueThreadRunningRef.getAndSet(true)) + { + return; + } + + + this.updateQueueProcessor.execute(() -> + { + try + { + ArrayList updatePosList = null; + + while ( // continue queuing update positions as long as there are positions to queue + (updatePosList == null || updatePosList.size() != 0) + // only add more items to the queue if half or more of the previous tasks have been completed + && this.parentApplicationPositionSet.size() < (MAX_PARENT_UPDATE_TASK_COUNT / 2)) + { + // prevent hitting the database more often than is necessary + Thread.sleep(UPDATE_QUEUE_THREAD_DELAY_IN_MS); + + + // get the positions that need to be applied to their parents + updatePosList = ((NewFullDataSourceRepo) this.repo).getPositionsToUpdate(MAX_PARENT_UPDATE_TASK_COUNT); + if (updatePosList.size() != 0) + { + // stop if the file handler has been shut down + ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + this.updateQueueThreadRunningRef.set(false); + return; + } + + + // queue each update + int queueCount = 0; + for (DhSectionPos pos : updatePosList) + { + if (this.parentApplicationPositionSet.add(pos)) + { + queueCount++; + executor.execute(() -> + { + NewFullDataSource inputData = this.get(pos); + // update the parent position with this new data + this.updateDataSourceAtPos(pos.getParentPos(), inputData, true); + + // TODO add comparable interface to make this low priority + }); + } + } + + // can be used for debugging + if (queueCount != 0) + { + LOGGER.trace("Queued [" + queueCount + "] out of ["+updatePosList.size()+"] parent updates."); + } + } + } + } + catch (Exception e) + { + LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e); + } + finally + { + this.updateQueueThreadRunningRef.set(false); + } + }); + } + + + + //===========// + // overrides // + //===========// + + @Override + public void debugRender(DebugRenderer renderer) + { + if (Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get()) + { + this.parentApplicationPositionSet + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); + } + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java similarity index 59% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 70c26ee5b..5b38ed4e8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -19,11 +19,10 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; -import com.seibel.distanthorizons.core.generation.MissingWorldGenPositionFinder; import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -34,11 +33,12 @@ import org.apache.logging.log4j.Logger; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; -public class GeneratedFullDataFileHandler extends FullDataFileHandler +public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -46,8 +46,11 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); - /** Used to prevent world gen tasks from being queued multiple times. */ - private final Set generatingDataPos = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + // TODO name better + // this is just the list of section pos that have had their world generation + // calculated and queued this session. + private final Set genHandledPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); @@ -55,7 +58,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // constructor // //=============// - public GeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public NewGeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } @@ -64,25 +67,22 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler //===========// @Override - public IFullDataSource get(DhSectionPos pos) { return this.get(pos, true); } - public IFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck) + public NewFullDataSource get(DhSectionPos pos) { return this.get(pos, true); } + public NewFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck) { - IFullDataSource dataSource = super.get(pos); + NewFullDataSource dataSource = super.get(pos); if (runWorldGenCheck) { - // add world gen tasks for missing columns in the data source - // if this position hasn't already been queued for generation - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null && !this.generatingDataPos.contains(pos)) - { - this.queueWorldGenForMissingColumnsInDataSource(worldGenQueue, pos, dataSource); - } + this.tryQueueSection(pos); } return dataSource; } + @Override + public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) { this.tryQueueSection(pos); } + //==================// @@ -103,17 +103,25 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public void clearGenerationQueue() { this.worldGenQueueRef.set(null); - this.generatingDataPos.clear(); // clear the incomplete data sources + this.genHandledPosSet.clear(); } /** Can be used to remove positions that are outside the player's render distance. */ public void removeGenRequestIf(Function removeIf) { - this.generatingDataPos.forEach((pos) -> + // TODO there has to be a better way to do this + + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) + { + worldGenQueue.removeGenRequestIf(removeIf); + } + + this.genHandledPosSet.forEach((pos) -> { if (removeIf.apply(pos)) { - this.generatingDataPos.remove(pos); + this.genHandledPosSet.remove(pos); } }); } @@ -162,6 +170,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler } } + // TODO only fire after the section has finished generated or once every X seconds private void fireOnGenPosSuccessListeners(DhSectionPos pos) { // fire the event listeners @@ -177,47 +186,97 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler // helper methods // //================// - private void queueWorldGenForMissingColumnsInDataSource(IWorldGenerationQueue worldGenQueue, DhSectionPos pos, IFullDataSource dataSource) + /** does nothing if this section or one of it's parents has already been queued */ + public void tryQueueSection(DhSectionPos pos) { - // get the un-generated pos list - byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - ArrayList genPosList = MissingWorldGenPositionFinder.getUngeneratedPosList(dataSource, minGeneratorSectionDetailLevel, true); - - // start each pos generating - ArrayList> taskFutureList = new ArrayList<>(); - for (DhSectionPos genPos : genPosList) + IWorldGenerationQueue tempWorldGenQueue = this.worldGenQueueRef.get(); + if (tempWorldGenQueue == null) { - // try not to re-queue already generating tasks - if (this.generatingDataPos.contains(genPos)) + return; + } + + if (this.genHandledPosSet.contains(pos)) + { + return; + } + + + + AtomicBoolean positionAlreadyHandled = new AtomicBoolean(false); + pos.forEachPosUpToDetailLevel(NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL, (parentPos) -> + { + if (!positionAlreadyHandled.get()) { - continue; + if (this.genHandledPosSet.contains(parentPos)) + { + positionAlreadyHandled.set(true); + } + } + }); + + if (positionAlreadyHandled.get()) + { + return; + } + this.genHandledPosSet.add(pos); + + + + // get the un-generated pos list + byte minGeneratorSectionDetailLevel = (byte) (tempWorldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + + pos.forEachChildAtDetailLevel(minGeneratorSectionDetailLevel, (genPos) -> + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return; } if (this.repo.existsWithKey(genPos)) { - continue; + // TODO only pull in the generation steps + NewFullDataSource potentialDataSource = this.get(genPos, false); + + EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT; + checkWorldGenLoop: + for (int x = 0; x < NewFullDataSource.WIDTH; x++) + { + for (int z = 0; z < NewFullDataSource.WIDTH; z++) + { + int index = NewFullDataSource.relativePosToIndex(x, z); + byte genStepValue = potentialDataSource.columnGenerationSteps[index]; + + if (genStepValue < currentMinWorldGenStep.value) + { + EDhApiWorldGenerationStep newWorldGenStep = EDhApiWorldGenerationStep.fromValue(genStepValue); + if (newWorldGenStep != null && newWorldGenStep.value < currentMinWorldGenStep.value) + { + currentMinWorldGenStep = newWorldGenStep; + } + } + + if (currentMinWorldGenStep == EDhApiWorldGenerationStep.EMPTY) + { + // queue the task + break checkWorldGenLoop; + } + } + } + + if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY) + { + // no world gen needed + return; + } } // queue each new gen task - GenTask genTask = new GenTask(dataSource.getSectionPos()); - CompletableFuture worldGenFuture = worldGenQueue.submitGenTask(genPos, dataSource.getDataDetailLevel(), genTask); + GenTask genTask = new GenTask(genPos); + CompletableFuture worldGenFuture = tempWorldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask); worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex)); - - taskFutureList.add(worldGenFuture); - } - - - // mark the data source as generating if necessary - if (taskFutureList.size() != 0) - { - this.generatingDataPos.add(pos); - CompletableFuture.allOf(taskFutureList.toArray(new CompletableFuture[0])) - .whenComplete((voidObj, ex) -> - { - this.generatingDataPos.remove(pos); - }); - } + }); } @@ -242,11 +301,11 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandler public boolean isMemoryAddressValid() { return true; } @Override - public Consumer getChunkDataConsumer() + public Consumer getChunkDataConsumer() { return (chunkSizedFullDataSource) -> { - GeneratedFullDataFileHandler.this.level.updateDataSourcesWithChunkData(chunkSizedFullDataSource); + NewGeneratedFullDataFileHandler.this.level.updateDataSources(chunkSizedFullDataSource); }; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewRemoteFullDataFileHandler.java similarity index 74% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewRemoteFullDataFileHandler.java index f3ef059cb..d3287db9d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewRemoteFullDataFileHandler.java @@ -25,9 +25,9 @@ import org.jetbrains.annotations.Nullable; import java.io.File; -public class RemoteFullDataFileHandler extends FullDataFileHandler +public class NewRemoteFullDataFileHandler extends NewFullDataFileHandler { - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } + public NewRemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public NewRemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java index 2e31e1794..3c2147a2f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java @@ -19,14 +19,12 @@ package com.seibel.distanthorizons.core.file.renderfile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.RenderDataRepo; +import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; import java.util.concurrent.CompletableFuture; @@ -38,7 +36,7 @@ public interface IRenderSourceProvider extends ISourceProvider getAsync(DhSectionPos pos); - CompletableFuture updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkData); + CompletableFuture updateDataSourceAsync(NewFullDataSource dataSource); /** Deletes any data stored in the render cache so it can be re-created */ void deleteRenderCache(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 8b85153e4..b50dcc1d3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -19,39 +19,35 @@ package com.seibel.distanthorizons.core.file.renderfile; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; +import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.sql.AbstractDataSourceRepo; -import com.seibel.distanthorizons.core.sql.DataSourceDto; -import com.seibel.distanthorizons.core.sql.FullDataRepo; -import com.seibel.distanthorizons.core.sql.RenderDataRepo; +import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; import com.seibel.distanthorizons.core.util.threading.ThreadPools; import org.apache.logging.log4j.Logger; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.*; -public class RenderSourceFileHandler extends AbstractDataSourceHandler implements IRenderSourceProvider +public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler implements IRenderSourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private final F3Screen.NestedMessage threadPoolMsg; - private final IFullDataSourceProvider fullDataSourceProvider; + public final IFullDataSourceProvider fullDataSourceProvider; @@ -78,7 +74,7 @@ public class RenderSourceFileHandler extends AbstractDataSourceHandler updateDataSourcesWithChunkDataAsync(ChunkSizedFullDataAccessor chunkDataView) + public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) { - return CompletableFuture.allOf( - super.updateDataSourcesWithChunkDataAsync(chunkDataView), - this.fullDataSourceProvider.updateDataSourcesWithChunkDataAsync(chunkDataView) - ); + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); + return CompletableFuture.completedFuture(null); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 85fdb0c58..c48569e79 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -21,10 +21,9 @@ package com.seibel.distanthorizons.core.file.subDimMatching; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.structure.ClientOnlySaveStructure; import com.seibel.distanthorizons.core.generation.DhLightingEngine; @@ -190,14 +189,9 @@ public class SubDimensionLevelMatcher implements AutoCloseable LOGGER.warn("unable to build lod for chunk:"+newlyLoadedChunk.getChunkPos()); return null; } - ChunkSizedFullDataAccessor newChunkSizedFullDataView = LodDataBuilder.createChunkData(newlyLoadedChunk); - if (newChunkSizedFullDataView == null) - { - LOGGER.warn("Unexpected LOD build error for chunk:"+newlyLoadedChunk.getChunkPos()); - return null; - } + NewFullDataSource newChunkSizedFullDataView = NewFullDataSource.createFromChunk(newlyLoadedChunk); // convert to a data source for easier comparing - CompleteFullDataSource newDataSource = CompleteFullDataSource.createEmpty(new DhSectionPos(this.playerData.playerBlockPos)); + NewFullDataSource newDataSource = NewFullDataSource.createEmpty(new DhSectionPos(this.playerData.playerBlockPos)); newDataSource.update(newChunkSizedFullDataView); @@ -219,7 +213,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable { // get the data source to compare against IDhLevel tempLevel = new DhClientLevel(new ClientOnlySaveStructure(), this.currentClientLevel, testLevelFolder, false); - IFullDataSource testFullDataSource = tempLevel.getFileHandler().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); + NewFullDataSource testFullDataSource = tempLevel.getFullDataProvider().getAsync(new DhSectionPos(this.playerData.playerBlockPos)).join(); if (testFullDataSource == null) { continue; @@ -240,8 +234,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable { for (int z = 0; z < CompleteFullDataSource.WIDTH; z++) { - SingleColumnFullDataAccessor newColumn = newDataSource.tryGet(x, z); - SingleColumnFullDataAccessor testColumn = testFullDataSource.tryGet(x, z); + SingleColumnFullDataAccessor newColumn = newDataSource.get(x, z); + SingleColumnFullDataAccessor testColumn = testFullDataSource.get(x, z); if (newColumn != null && testColumn != null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index af9c28755..52caedbef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -26,7 +26,10 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.io.Closeable; import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +// TODO does this need a interface? +@Deprecated public interface IWorldGenerationQueue extends Closeable { /** the largest numerical detail level */ @@ -45,4 +48,8 @@ public interface IWorldGenerationQueue extends Closeable CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning); void close(); + void removeGenRequestIf(Function removeIf); + void removeGenTask(DhSectionPos pos); + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java deleted file mode 100644 index f6e554dd8..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/MissingWorldGenPositionFinder.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.generation; - -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.LinkedList; - -/** - * Handles finding any positions in a given {@link IFullDataSource} that - * aren't generated. - */ -public class MissingWorldGenPositionFinder -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - /** - * @param generatorDetailLevel the detail level that the un-generated positions should be split into. - * @param onlyReturnPositionsTheGeneratorCanAccept - * If true this will only return positions with the same detail level as generatorDetailLevel.
- * If false this will return the lowest detail level possible for totally empty sections.
- * As of 2023-9-28 both have been tested and confirmed working with the Batch world generator, the only difference is that - * the task list will be deceptively small if this value is false. - * - * @return the list of {@link DhSectionPos} that aren't generated in this data source. - */ - public static ArrayList getUngeneratedPosList(IFullDataSource dataSource, byte generatorDetailLevel, boolean onlyReturnPositionsTheGeneratorCanAccept) - { - ArrayList posArray = getUngeneratedPosListForQuadrant(dataSource, dataSource.getSectionPos(), generatorDetailLevel); - - if (onlyReturnPositionsTheGeneratorCanAccept) - { - LinkedList posList = new LinkedList<>(posArray); - - // subdivide positions until they match the generatorDetailLevel - ArrayList cleanedPosArray = new ArrayList<>(); - while (posList.size() > 0) - { - DhSectionPos pos = posList.remove(); - if (pos.getDetailLevel() > generatorDetailLevel) - { - pos.forEachChild((childPos) -> posList.push(childPos)); - } - else - { - cleanedPosArray.add(pos); - } - } - - return cleanedPosArray; - } - else - { - return posArray; - } - } - private static ArrayList getUngeneratedPosListForQuadrant(IFullDataSource dataSource, DhSectionPos quadrantPos, byte generatorDetailLevel) - { - ArrayList ungeneratedPosList = new ArrayList<>(); - - int sourceRelWidthInDataPoints = dataSource.getWidthInDataPoints(); - - - if (quadrantPos.getDetailLevel() == generatorDetailLevel) - { - // we are at the highest detail level the world generator can accept, - // we either need to generate this whole section, or not at all - - ESectionPopulationState populationState = getPopulationStateForPos(dataSource, quadrantPos, sourceRelWidthInDataPoints); - if (populationState != ESectionPopulationState.COMPLETE) - { - // at least 1 data point is missing, - // this whole section must be generated - ungeneratedPosList.add(quadrantPos); - } - } - else if (quadrantPos.getDetailLevel() > generatorDetailLevel) - { - // detail level too low for world generator, check child positions - for (int i = 0; i < 4; i++) - { - DhSectionPos inputPos = quadrantPos.getChildByIndex(i); - - ESectionPopulationState populationState = getPopulationStateForPos(dataSource, inputPos, sourceRelWidthInDataPoints); - if (populationState == ESectionPopulationState.COMPLETE) - { - // no generation necessary - continue; - } - else if (populationState == ESectionPopulationState.EMPTY) - { - // nothing exists for this sub quadrant, add this sub-quadrant's position - ungeneratedPosList.add(inputPos); - } - else if (populationState == ESectionPopulationState.PARTIAL) - { - // some data exists in this quadrant, but not all that we need - // recurse down to determine which sub-quadrant positions will need generation - ungeneratedPosList.addAll(getUngeneratedPosListForQuadrant(dataSource, inputPos, generatorDetailLevel)); - } - } - } - else - { - throw new IllegalArgumentException("detail level lower than world generator can accept."); - } - - return ungeneratedPosList; - } - private static ESectionPopulationState getPopulationStateForPos(IFullDataSource dataSource, DhSectionPos inputPos, int sourceRelWidthInDataPoints) - { - // TODO comment - - byte childDetailLevel = inputPos.getDetailLevel(); - - int quadrantDetailLevelDiff = dataSource.getSectionPos().getDetailLevel() - childDetailLevel; - int widthInSecPos = BitShiftUtil.powerOfTwo(quadrantDetailLevelDiff); - int relWidthForSecPos = sourceRelWidthInDataPoints / widthInSecPos; - - DhSectionPos minSecPos = dataSource.getSectionPos().convertNewToDetailLevel(childDetailLevel); - - - - int minRelX = inputPos.getX() - minSecPos.getX(); - int maxRelX = minRelX + 1; - - int minRelZ = inputPos.getZ() - minSecPos.getZ(); - int maxRelZ = minRelZ + 1; - - minRelX = minRelX * relWidthForSecPos; - maxRelX = maxRelX * relWidthForSecPos; - - minRelZ = minRelZ * relWidthForSecPos; - maxRelZ = maxRelZ * relWidthForSecPos; - - - - boolean quadrantFullyGenerated = true; - boolean quadrantEmpty = true; - for (int relX = minRelX; relX < maxRelX; relX++) - { - for (int relZ = minRelZ; relZ < maxRelZ; relZ++) - { - SingleColumnFullDataAccessor column = dataSource.tryGet(relX, relZ); - if (column == null || !column.doesColumnExist()) - { - // no data for this relative position - quadrantFullyGenerated = false; - } - else - { - // data exists for this pos - quadrantEmpty = false; - } - } - } - - - if (quadrantFullyGenerated) - { - // no generation necessary - return ESectionPopulationState.COMPLETE; - } - else if (quadrantEmpty) - { - // nothing exists for this sub quadrant, add this sub-quadrant's position - return ESectionPopulationState.EMPTY; - } - else - { - // some data exists in this quadrant, but not all that we need - // recurse down to determine which sub-quadrant positions will need generation - return ESectionPopulationState.PARTIAL; - } - } - - - - //================// - // helper classes // - //================// - - private enum ESectionPopulationState - { - COMPLETE, - EMPTY, - PARTIAL - } -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 7c3b68744..822a86606 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGenerat import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratorReturnType; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.generation.tasks.*; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -47,6 +47,7 @@ import java.awt.*; import java.util.*; import java.util.concurrent.*; import java.util.function.Consumer; +import java.util.function.Function; public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRenderable { @@ -159,6 +160,29 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender return future; } + @Override + public void removeGenRequestIf(Function removeIf) + { + this.waitingTasks.forEachKey(100, (genPos) -> + { + if (removeIf.apply(genPos)) + { + this.waitingTasks.remove(genPos); + } + }); + } + + @Override + public void removeGenTask(DhSectionPos pos) + { + WorldGenTask task = this.waitingTasks.remove(pos); + if (task != null) + { + task.future.cancel(true); + } + } + + //===============// @@ -407,7 +431,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender DhChunkPos chunkPosMin, byte granularity, byte targetDataDetail, - Consumer chunkDataConsumer + Consumer chunkDataConsumer ) { EDhApiDistantGeneratorMode generatorMode = Config.Client.Advanced.WorldGenerator.distantGeneratorMode.get(); @@ -428,9 +452,9 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender try { IChunkWrapper chunk = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray); - ChunkSizedFullDataAccessor chunkDataAccessor = LodDataBuilder.createChunkData(chunk); - LodUtil.assertTrue(chunkDataAccessor != null); - chunkDataConsumer.accept(chunkDataAccessor); + NewFullDataSource dataSource = LodDataBuilder.createGeneratedDataSource(chunk); + LodUtil.assertTrue(dataSource != null); + chunkDataConsumer.accept(dataSource); } catch (ClassCastException e) { @@ -453,8 +477,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender { try { - ChunkSizedFullDataAccessor chunkDataAccessor = LodDataBuilder.createApiChunkData(dataPoints); - chunkDataConsumer.accept(chunkDataAccessor); + NewFullDataSource dataSource = LodDataBuilder.createFromApiChunkData(dataPoints); + chunkDataConsumer.accept(dataSource); } catch (ClassCastException e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java index d30653121..4568f9821 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.generation.tasks; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import java.util.function.Consumer; @@ -32,6 +32,6 @@ public interface IWorldGenTaskTracker /** Returns true if the task hasn't been garbage collected. */ boolean isMemoryAddressValid(); - Consumer getChunkDataConsumer(); + Consumer getChunkDataConsumer(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java index fdc188d55..39c145c01 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java @@ -19,8 +19,7 @@ package com.seibel.distanthorizons.core.generation.tasks; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.pos.DhLodPos; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.util.Iterator; @@ -31,6 +30,7 @@ import java.util.function.Consumer; * @author Leetom * @version 2022-11-25 */ +@Deprecated // TODO look into how these are used and if they should continue to be used public final class WorldGenTaskGroup { public final DhSectionPos pos; @@ -46,13 +46,13 @@ public final class WorldGenTaskGroup this.dataDetail = dataDetail; } - public void consumeChunkData(ChunkSizedFullDataAccessor chunkSizedFullDataView) + public void consumeChunkData(NewFullDataSource chunkSizedFullDataView) { Iterator tasks = this.worldGenTasks.iterator(); while (tasks.hasNext()) { WorldGenTask task = tasks.next(); - Consumer chunkDataConsumer = task.taskTracker.getChunkDataConsumer(); + Consumer chunkDataConsumer = task.taskTracker.getChunkDataConsumer(); if (chunkDataConsumer == null) { tasks.remove(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java index 72e3324c7..24e76ba78 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java @@ -20,13 +20,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkModifiedEvent; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.transformers.ChunkToLodBuilder; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector; -import java.util.concurrent.CompletableFuture; - public abstract class AbstractDhLevel implements IDhLevel { public final ChunkToLodBuilder chunkToLodBuilder; @@ -48,23 +46,17 @@ public abstract class AbstractDhLevel implements IDhLevel @Override public void updateChunkAsync(IChunkWrapper chunk) { - CompletableFuture future = this.chunkToLodBuilder.tryGenerateData(chunk); - if (future != null) + NewFullDataSource dataSource = NewFullDataSource.createFromChunk(chunk); + if (dataSource == null) { - future.thenAccept((chunkSizedFullDataAccessor) -> - { - if (chunkSizedFullDataAccessor == null) - { - // This can happen if, among other reasons, a chunk save is superceded by a later event - return; - } - - this.updateDataSourcesWithChunkData(chunkSizedFullDataAccessor); - ApiEventInjector.INSTANCE.fireAllEvents( - DhApiChunkModifiedEvent.class, - new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z)); - }); + // This can happen if, among other reasons, a chunk save is superseded by a later event + return; } + + this.updateDataSources(dataSource); + ApiEventInjector.INSTANCE.fireAllEvents( + DhApiChunkModifiedEvent.class, + new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z)); } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index e83aa706f..23a358d7e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -22,9 +22,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -38,25 +40,36 @@ import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; -import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import org.apache.logging.log4j.Logger; import java.io.Closeable; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule implements Closeable +public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); private final IDhClientLevel parentClientLevel; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); public final F3Screen.NestedMessage f3Message; + + + + //=============// + // constructor // + //=============// + public ClientLevelModule(IDhClientLevel parentClientLevel) { this.parentClientLevel = parentClientLevel; this.f3Message = new F3Screen.NestedMessage(this::f3Log); + + NewFullDataFileHandler fileHandler = this.parentClientLevel.getFullDataProvider(); + fileHandler.dateSourceUpdateListeners.add(this); } + + //==============// // tick methods // //==============// @@ -92,7 +105,7 @@ public class ClientLevelModule implements Closeable } clientRenderState.close(); - clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFileHandler(), this.parentClientLevel.getSaveStructure()); + clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, clientRenderState)) { //FIXME: How to handle this? @@ -124,7 +137,7 @@ public class ClientLevelModule implements Closeable /** @return if the {@link ClientRenderState} was successfully swapped */ public boolean startRenderer(IClientLevelWrapper clientLevelWrapper) { - ClientRenderState ClientRenderState = new ClientRenderState(parentClientLevel, clientLevelWrapper, parentClientLevel.getFileHandler(), parentClientLevel.getSaveStructure()); + ClientRenderState ClientRenderState = new ClientRenderState(parentClientLevel, clientLevelWrapper, parentClientLevel.getFullDataProvider(), parentClientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState)) { LOGGER.warn("Failed to start renderer due to concurrency"); @@ -185,22 +198,23 @@ public class ClientLevelModule implements Closeable ClientRenderState.close(); } + + //===============// // data handling // //===============// - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) + + public void updateDataSources(NewFullDataSource data) { this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } + @Override + public void OnDataSourceUpdated(NewFullDataSource updatedFullDataSource) { ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { ClientRenderState.renderSourceFileHandler - .updateDataSourcesWithChunkDataAsync(data) + .updateDataSourceAsync(updatedFullDataSource) // wait for the update to finish before triggering a reload to prevent holes in the world - .thenRun(() -> ClientRenderState.quadtree.reloadPos(data.sectionPos)); - } - else - { - this.parentClientLevel.getFileHandler().updateDataSourcesWithChunkDataAsync(data); + .thenRun(() -> ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos())); } } @@ -231,16 +245,10 @@ public class ClientLevelModule implements Closeable - //=======================// // misc helper functions // //=======================// - public void dumpRamUsage() - { - //TODO - } - /** Returns what should be displayed in Minecraft's F3 debug menu */ protected String[] f3Log() { @@ -274,6 +282,12 @@ public class ClientLevelModule implements Closeable } } + + + //================// + // helper classes // + //================// + public static class ClientRenderState { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 1c69e5c83..500588d13 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -20,9 +20,10 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewRemoteFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; @@ -44,7 +45,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public final ClientLevelModule clientside; public final IClientLevelWrapper levelWrapper; public final AbstractSaveStructure saveStructure; - public final RemoteFullDataFileHandler dataFileHandler; + public final NewRemoteFullDataFileHandler dataFileHandler; @@ -57,7 +58,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel { this.levelWrapper = clientLevelWrapper; this.saveStructure = saveStructure; - this.dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); + this.dataFileHandler = new NewRemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); this.clientside = new ClientLevelModule(this); if (enableRendering) @@ -117,7 +118,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public ILevelWrapper getLevelWrapper() { return levelWrapper; } @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); } + public void updateDataSources(NewFullDataSource data) { this.clientside.updateDataSources(data); } @Override public int getMinY() { return levelWrapper.getMinHeight(); } @@ -136,10 +137,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel //=======================// @Override - public IFullDataSourceProvider getFileHandler() - { - return dataFileHandler; - } + public NewFullDataFileHandler getFullDataProvider() { return this.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index be70e9f95..8a063f933 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -20,9 +20,10 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.render.LodRenderSection; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; @@ -173,10 +174,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public IFullDataSourceProvider getFileHandler() - { - return serverside.dataFileHandler; - } + public NewFullDataFileHandler getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() @@ -188,7 +186,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public boolean hasSkyLight() { return this.serverLevelWrapper.hasSkyLight(); } @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) { this.clientside.updateDataSourcesWithChunkData(data); } + public void updateDataSources(NewFullDataSource data) { this.clientside.updateDataSources(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index 9b12a3627..e8cdd66de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -50,11 +50,11 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public void serverTick() { this.chunkToLodBuilder.tick(); } @Override - public void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data) + public void updateDataSources(NewFullDataSource data) { DhSectionPos pos = data.getSectionPos(); pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - this.getFileHandler().updateDataSourcesWithChunkDataAsync(data); + this.getFullDataProvider().updateDataSourceAsync(data); } @Override @@ -97,7 +97,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public IFullDataSourceProvider getFileHandler() { return serverside.dataFileHandler; } + public NewFullDataFileHandler getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index f614dad86..02331409b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -19,14 +19,12 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.ChunkSizedFullDataAccessor; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; -import java.util.concurrent.CompletableFuture; - public interface IDhLevel extends AutoCloseable { int getMinY(); @@ -39,12 +37,12 @@ public interface IDhLevel extends AutoCloseable void updateChunkAsync(IChunkWrapper chunk); - IFullDataSourceProvider getFileHandler(); + NewFullDataFileHandler getFullDataProvider(); AbstractSaveStructure getSaveStructure(); boolean hasSkyLight(); - void updateDataSourcesWithChunkData(ChunkSizedFullDataAccessor data); + void updateDataSources(NewFullDataSource data); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java index 94108b380..118d4a272 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; -public interface IDhWorldGenLevel extends IDhLevel, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener +public interface IDhWorldGenLevel extends IDhLevel, NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener { void doWorldGen(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java index e73deca38..cdc32f285 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java @@ -22,12 +22,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.distanthorizons.core.config.AppliedConfigState; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.BatchGenerator; import com.seibel.distanthorizons.core.generation.WorldGenerationQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.WorldGeneratorInjector; import org.apache.logging.log4j.Logger; @@ -37,7 +36,7 @@ public class ServerLevelModule public final IDhServerLevel parentServerLevel; public final AbstractSaveStructure saveStructure; - public final GeneratedFullDataFileHandler dataFileHandler; + public final NewGeneratedFullDataFileHandler dataFileHandler; public final AppliedConfigState worldGeneratorEnabledConfig; public final WorldGenModule worldGenModule; @@ -48,7 +47,7 @@ public class ServerLevelModule { this.parentServerLevel = parentServerLevel; this.saveStructure = saveStructure; - this.dataFileHandler = new GeneratedFullDataFileHandler(parentServerLevel, saveStructure); + this.dataFileHandler = new NewGeneratedFullDataFileHandler(parentServerLevel, saveStructure); this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration); this.worldGenModule = new WorldGenModule(this.dataFileHandler, this.parentServerLevel); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index 5b274857f..f008660b1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -34,15 +34,15 @@ public class WorldGenModule implements Closeable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final GeneratedFullDataFileHandler dataFileHandler; - private final GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; + private final NewGeneratedFullDataFileHandler dataFileHandler; + private final NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; private final AtomicReference worldGenStateRef = new AtomicReference<>(); private final F3Screen.DynamicMessage worldGenF3Message; - public WorldGenModule(GeneratedFullDataFileHandler dataFileHandler, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) + public WorldGenModule(NewGeneratedFullDataFileHandler dataFileHandler, NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) { this.dataFileHandler = dataFileHandler; this.onWorldGenCompleteListener = onWorldGenCompleteListener; @@ -70,7 +70,7 @@ public class WorldGenModule implements Closeable // world gen control // //===================// - public void startWorldGen(GeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) + public void startWorldGen(NewGeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) { // create the new world generator if (!this.worldGenStateRef.compareAndSet(null, newWgs)) @@ -82,7 +82,7 @@ public class WorldGenModule implements Closeable dataFileHandler.setWorldGenerationQueue(newWgs.worldGenerationQueue); } - public void stopWorldGen(GeneratedFullDataFileHandler dataFileHandler) + public void stopWorldGen(NewGeneratedFullDataFileHandler dataFileHandler) { AbstractWorldGenState worldGenState = this.worldGenStateRef.get(); if (worldGenState == null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java index d6d038b74..b2f3cb1f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.pos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.LodUtil; import org.jetbrains.annotations.NotNull; @@ -32,6 +32,7 @@ import java.util.Objects; * @author Leetom * @version 2022-11-6 */ +@Deprecated public class DhLodPos implements Comparable { public final byte detailLevel; @@ -116,7 +117,7 @@ public class DhLodPos implements Comparable public DhLodPos getDhSectionRelativePositionForDetailLevel() throws IllegalArgumentException { return this.getDhSectionRelativePositionForDetailLevel(this.detailLevel); } /** * Returns a DhLodPos with the given detail level and an X/Z position somewhere between (0,0) and (63,63). - * This is done to access specific sections from a {@link IFullDataSource} where LOD columns are stored + * This is done to access specific sections from a {@link NewFullDataSource} where LOD columns are stored * in 64 x 64 blocks. * * @throws IllegalArgumentException if this position's detail level is lower than the output detail level diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index 819e2caa8..c6cc3a0ab 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -155,8 +155,10 @@ public class DhSectionPos //=========// /** @return the corner with the smallest X and Z coordinate */ + @Deprecated public DhLodPos getMinCornerLodPos() { return this.getMinCornerLodPos((byte) (this.detailLevel - 1)); } /** @return the corner with the smallest X and Z coordinate */ + @Deprecated public DhLodPos getMinCornerLodPos(byte returnDetailLevel) { LodUtil.assertTrue(returnDetailLevel <= this.detailLevel, "returnDetailLevel must be less than sectionDetail"); @@ -167,6 +169,16 @@ public class DhSectionPos this.z * BitShiftUtil.powerOfTwo(offset)); } + public DhSectionPos getMinCornerPos(byte returnDetailLevel) + { + LodUtil.assertTrue(returnDetailLevel <= this.detailLevel, "returnDetailLevel must be less than sectionDetail"); + + byte offset = (byte) (this.detailLevel - returnDetailLevel); + return new DhSectionPos(returnDetailLevel, + this.x * BitShiftUtil.powerOfTwo(offset), + this.z * BitShiftUtil.powerOfTwo(offset)); + } + /** * A detail level of X lower than this section's detail level will return:
* 0 -> 1
@@ -331,7 +343,7 @@ public class DhSectionPos } /** Applies the given consumer to all children of the position at the given section detail level. */ - public void forEachChildAtLevel(byte sectionDetailLevel, Consumer callback) + public void forEachChildAtDetailLevel(byte sectionDetailLevel, Consumer callback) { if (sectionDetailLevel == this.detailLevel) { @@ -341,7 +353,7 @@ public class DhSectionPos for (int i = 0; i < 4; i++) { - this.getChildByIndex(i).forEachChildAtLevel(sectionDetailLevel, callback); + this.getChildByIndex(i).forEachChildAtDetailLevel(sectionDetailLevel, callback); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 2f9a9bda9..5ed35e536 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -33,6 +33,7 @@ import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; @@ -206,7 +207,7 @@ public class LodQuadTree extends QuadTree implements AutoClose LodRenderSection newRenderSection = new LodRenderSection(this, sectionPos); rootNode.setValue(sectionPos, newRenderSection); - renderSection = newRenderSection; + renderSection = newRenderSection; // TODO this never seemed to be called, is it necessary? } @@ -435,7 +436,7 @@ public class LodQuadTree extends QuadTree implements AutoClose * Can be called whenever a render section's data needs to be refreshed.
* This should be called whenever a world generation task is completed or if the connected server has new data to show. */ - public void reloadPos(DhSectionPos pos) + public void reloadPos(@NotNull DhSectionPos pos) { if (pos == null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index d71eda424..2c4c56844 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -20,6 +20,8 @@ package com.seibel.distanthorizons.core.sql; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import org.apache.logging.log4j.Logger; import java.io.IOException; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java similarity index 95% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java index a692a5b08..36405f26b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/IBaseDTO.java @@ -17,7 +17,7 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.dto; /** * DTO = DataTable Object
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java similarity index 83% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java index a0782c41b..2f7ec8de0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DataSourceDto.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java @@ -17,10 +17,9 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.dto; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -30,8 +29,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; -/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */ -public class DataSourceDto implements IBaseDTO +/** handles storing {@link ColumnRenderSource}'s in the database. */ +public class LegacyDataSourceDTO implements IBaseDTO { public DhSectionPos pos; public int checksum; @@ -54,7 +53,7 @@ public class DataSourceDto implements IBaseDTO // constructor // //=============// - public DataSourceDto(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) + public LegacyDataSourceDTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) { this.pos = pos; this.checksum = checksum; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java new file mode 100644 index 000000000..154b47230 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java @@ -0,0 +1,256 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.sql.dto; + +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import org.jetbrains.annotations.NotNull; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.Adler32; +import java.util.zip.CheckedOutputStream; + +/** handles storing {@link NewFullDataSource}'s in the database. */ +public class NewFullDataSourceDTO implements IBaseDTO +{ + public DhSectionPos pos; + + public int levelMinY; + + /** only for the data array */ + public int dataChecksum; + + //public long[][] dataArray; + public byte[] dataByteArray; + + /** @see EDhApiWorldGenerationStep */ + public byte[] columnGenStepByteArray; + + //public FullDataPointIdMap mapping; + public byte[] mappingByteArray; + + public byte dataFormatVersion; + + public boolean applyToParent; + + public long lastModifiedUnixDateTime; + public long createdUnixDateTime; + + + + //=============// + // constructor // + //=============// + + public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource) throws IOException + { + CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints); + byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping()); + + return new NewFullDataSourceDTO( + dataSource.getSectionPos(), + checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, checkedDataPointArray.byteArray, + dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, + mappingByteArray, dataSource.applyToParent, + dataSource.levelMinY + ); + } + public NewFullDataSourceDTO( + DhSectionPos pos, + int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, byte[] dataByteArray, + long lastModifiedUnixDateTime, long createdUnixDateTime, + byte[] mappingByteArray, boolean applyToParent, + int levelMinY) + { + this.pos = pos; + this.dataChecksum = dataChecksum; + this.columnGenStepByteArray = columnGenStepByteArray; + + this.dataFormatVersion = dataFormatVersion; + + this.dataByteArray = dataByteArray; + this.mappingByteArray = mappingByteArray; + + this.applyToParent = applyToParent; + + this.lastModifiedUnixDateTime = lastModifiedUnixDateTime; + this.createdUnixDateTime = createdUnixDateTime; + + this.levelMinY = levelMinY; + } + + + + //========================// + // data source population // + //========================// + + public NewFullDataSource createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { return this.populateDataSource(NewFullDataSource.createEmpty(this.pos), levelWrapper); } + public NewFullDataSource populateDataSource(NewFullDataSource dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { + if (NewFullDataSource.DATA_FORMAT_VERSION != this.dataFormatVersion) + { + throw new IllegalStateException("There should only be one data format right now anyway."); + } + + dataSource.columnGenerationSteps = this.columnGenStepByteArray; + dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray); + + dataSource.getMapping().clear(dataSource.getSectionPos()); + dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper)); + + dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; + dataSource.createdUnixDateTime = this.createdUnixDateTime; + + dataSource.levelMinY = this.levelMinY; + + return dataSource; + } + + + + //=================// + // (de)serializing // + //=================// + + private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray) throws IOException + { + // write the outputs to a stream to prep for writing to the database + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + + // the order of these streams is important, otherwise the checksum won't be calculated + CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); + // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, + // but since this stream will be closed immediately after writing anyway, it won't be an issue + DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); + + + // write the data + int dataArrayLength = NewFullDataSource.WIDTH * NewFullDataSource.WIDTH; + for (int xz = 0; xz < dataArrayLength; xz++) + { + long[] dataColumn = dataArray[xz]; + + // write column length + int columnLength = (dataColumn != null) ? dataColumn.length : 0; + compressedOut.writeInt(columnLength); + + // write column data (will be skipped if no data was present) + for (int y = 0; y < columnLength; y++) + { + compressedOut.writeLong(dataColumn[y]); + } + } + + + // generate the checksum + compressedOut.flush(); + int checksum = (int) checkedOut.getChecksum().getValue(); + byteArrayOutputStream.close(); + + return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); + } + private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray) throws IOException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream); + + + // read the data + int dataArrayLength = NewFullDataSource.WIDTH * NewFullDataSource.WIDTH; + long[][] dataArray = new long[dataArrayLength][]; + for (int xz = 0; xz < dataArray.length; xz++) + { + // read the column length + int dataColumnLength = compressedIn.readInt(); // separate variables are used for debugging and in case validation wants to be added later + long[] dataColumn = new long[dataColumnLength]; + + // read column data (will be skipped if no data was present) + for (int y = 0; y < dataColumnLength; y++) + { + long dataPoint = compressedIn.readLong(); + dataColumn[y] = dataPoint; + } + + dataArray[xz] = dataColumn; + } + + + return dataArray; + } + + + private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping) throws IOException + { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream); + + mapping.serialize(compressedOut); + + compressedOut.flush(); + byteArrayOutputStream.close(); + + return byteArrayOutputStream.toByteArray(); + } + private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream); + + FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper); + return mapping; + } + + + + //===========// + // overrides // + //===========// + + @Override + public DhSectionPos getKey() { return this.pos; } + + + + //================// + // helper classes // + //================// + + private static class CheckedByteArray + { + public final int checksum; + public final byte[] byteArray; + + public CheckedByteArray(int checksum, byte[] byteArray) + { + this.checksum = checksum; + this.byteArray = byteArray; + } + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java similarity index 98% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 4acbe209d..2420a0478 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -17,9 +17,12 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.sql.DatabaseUpdater; +import com.seibel.distanthorizons.core.sql.DbConnectionClosedException; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java similarity index 85% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java index a6d94a053..bc979094c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java @@ -17,20 +17,21 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Map; -public abstract class AbstractDataSourceRepo extends AbstractDhRepo +public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo { - public AbstractDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + public AbstractLegacyDataSourceRepo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, DataSourceDto.class); + super(databaseType, databaseLocation, LegacyDataSourceDTO.class); } @@ -40,7 +41,7 @@ public abstract class AbstractDataSourceRepo extends AbstractDhRepo objectMap) throws ClassCastException + public LegacyDataSourceDTO convertDictionaryToDto(Map objectMap) throws ClassCastException { String posString = (String) objectMap.get("DhSectionPos"); DhSectionPos pos = DhSectionPos.deserialize(posString); @@ -58,7 +59,7 @@ public abstract class AbstractDataSourceRepo extends AbstractDhRepo. */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.sql.SQLException; -public class FullDataRepo extends AbstractDataSourceRepo +public class FullDataRepo extends AbstractLegacyDataSourceRepo { public static final String TABLE_NAME = "DhFullData"; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java new file mode 100644 index 000000000..6f0ab5ff0 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java @@ -0,0 +1,214 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.sql.repo; + +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NewFullDataSourceRepo extends AbstractDhRepo +{ + public NewFullDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + { + super(databaseType, databaseLocation, NewFullDataSourceDTO.class); + } + + + + @Override + public String getTableName() { return "FullData"; } + + @Override + public String createWhereStatement(DhSectionPos pos) + { + int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL; + return "DetailLevel = '"+detailLevel+"' AND PosX = '"+pos.getX()+"' AND PosZ = '"+pos.getZ()+"'"; + } + + + + //=======================// + // repo required methods // + //=======================// + + @Override + public NewFullDataSourceDTO convertDictionaryToDto(Map objectMap) throws ClassCastException + { + byte detailLevel = (Byte) objectMap.get("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = (Integer) objectMap.get("PosX"); + int posZ = (Integer) objectMap.get("PosZ"); + DhSectionPos pos = new DhSectionPos(sectionDetailLevel, posX, posZ); + + int minY = (Integer) objectMap.get("MinY"); + int dataChecksum = (Integer) objectMap.get("DataChecksum"); + + byte[] dataByteArray = (byte[]) objectMap.get("Data"); + byte[] columnGenStepByteArray = (byte[]) objectMap.get("ColumnGenerationStep"); + byte[] mappingByteArray = (byte[]) objectMap.get("Mapping"); + + + byte dataFormatVersion = (Byte) objectMap.get("DataFormatVersion"); + + boolean applyToParent = ((int) objectMap.get("ApplyToParent")) == 1; + + long lastModifiedUnixDateTime = (Long) objectMap.get("LastModifiedUnixDateTime"); + long createdUnixDateTime = (Long) objectMap.get("CreatedUnixDateTime"); + + NewFullDataSourceDTO dto = new NewFullDataSourceDTO( + pos, + dataChecksum, columnGenStepByteArray, dataFormatVersion, dataByteArray, + lastModifiedUnixDateTime, createdUnixDateTime, + mappingByteArray, applyToParent, + minY); + return dto; + } + + @Override + public PreparedStatement createInsertStatement(NewFullDataSourceDTO dto) throws SQLException + { + String sql = + "INSERT INTO "+this.getTableName() + " (\n" + + " DetailLevel, PosX, PosZ, \n" + + " MinY, DataChecksum, \n" + + " Data, ColumnGenerationStep, Mapping, \n" + + " DataFormatVersion, ApplyToParent, \n" + + " LastModifiedUnixDateTime, CreatedUnixDateTime) \n" + + "VALUES( \n" + + " ?, ?, ?, \n" + + " ?, ?, \n" + + " ?, ?, ?, \n" + + " ?, ?, \n" + + " ?, ? \n" + + ");"; + PreparedStatement statement = this.createPreparedStatement(sql); + + int i = 1; + statement.setObject(i++, dto.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + statement.setObject(i++, dto.pos.getX()); + statement.setObject(i++, dto.pos.getZ()); + + statement.setObject(i++, dto.levelMinY); + statement.setObject(i++, dto.dataChecksum); + + statement.setObject(i++, dto.dataByteArray); + statement.setObject(i++, dto.columnGenStepByteArray); + statement.setObject(i++, dto.mappingByteArray); + + statement.setObject(i++, dto.dataFormatVersion); + statement.setObject(i++, dto.applyToParent); + + statement.setObject(i++, dto.lastModifiedUnixDateTime); + statement.setObject(i++, dto.createdUnixDateTime); + + return statement; + } + + @Override + public PreparedStatement createUpdateStatement(NewFullDataSourceDTO dto) throws SQLException + { + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET \n" + + " MinY = ? \n" + + " ,DataChecksum = ? \n" + + + " ,Data = ? \n" + + " ,ColumnGenerationStep = ? \n" + + " ,Mapping = ? \n" + + + " ,DataFormatVersion = ? \n" + + " ,ApplyToParent = ? \n" + + + " ,LastModifiedUnixDateTime = ? \n" + + " ,CreatedUnixDateTime = ? \n" + + + "WHERE DetailLevel = ? AND PosX = ? AND PosZ = ?"; + PreparedStatement statement = this.createPreparedStatement(sql); + + int i = 1; + statement.setObject(i++, dto.levelMinY); + statement.setObject(i++, dto.dataChecksum); + + statement.setObject(i++, dto.dataByteArray); + statement.setObject(i++, dto.columnGenStepByteArray); + statement.setObject(i++, dto.mappingByteArray); + + statement.setObject(i++, dto.dataFormatVersion); + statement.setObject(i++, dto.applyToParent); + + statement.setObject(i++, dto.lastModifiedUnixDateTime); + statement.setObject(i++, dto.createdUnixDateTime); + + statement.setObject(i++, dto.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + statement.setObject(i++, dto.pos.getX()); + statement.setObject(i++, dto.pos.getZ()); + + return statement; + } + + + + // updates // + + public void setApplyToParent(DhSectionPos pos, boolean applyToParent) throws SQLException + { + int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET ApplyToParent = "+applyToParent+" \n" + + "WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ(); + + this.queryDictionaryFirst(sql); + } + + public ArrayList getPositionsToUpdate(int returnCount) + { + ArrayList list = new ArrayList<>(); + + List> resultMapList = this.queryDictionary( + "select DetailLevel, PosX, PosZ " + + "from "+this.getTableName()+" " + + "where ApplyToParent = 1 " + + "order by DetailLevel asc LIMIT "+returnCount+";"); + + for (Map resultMap : resultMapList) + { + byte detailLevel = (Byte) resultMap.get("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = (Integer) resultMap.get("PosX"); + int posZ = (Integer) resultMap.get("PosZ"); + + DhSectionPos pos = new DhSectionPos(sectionDetailLevel, posX, posZ); + list.add(pos); + } + + return list; + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java similarity index 91% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java index 7e2aee31e..fa43e81ea 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/RenderDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java @@ -17,13 +17,13 @@ * along with this program. If not, see . */ -package com.seibel.distanthorizons.core.sql; +package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.sql.SQLException; -public class RenderDataRepo extends AbstractDataSourceRepo +public class RenderDataRepo extends AbstractLegacyDataSourceRepo { public static final String TABLE_NAME = "DhRenderData"; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index 606af3eb6..9360c3ac5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -118,6 +118,9 @@ public class FullDataPointUtil { // this try-catch is present to fix an issue where the stack trace is missing // and to allow for easily attaching a debugger + + // if this was thrown that probably means the datasource has been + // re-mapped multiple times, causing the ID's to go out of their expected bounds. throw new RuntimeException(e); } } diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql new file mode 100644 index 000000000..e78cff45c --- /dev/null +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -0,0 +1,25 @@ + +--batch-- + +CREATE TABLE FullData ( + -- compound primary key + DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8 + ,PosX INT NOT NULL + ,PosZ INT NOT NULL + + ,MinY INT NOT NULL + ,DataChecksum INT NOT NULL + + ,Data BLOB NULL + ,ColumnGenerationStep BLOB NULL + ,Mapping BLOB NULL + + ,DataFormatVersion TINYINT NULL + + ,ApplyToParent BIT NULL + + ,LastModifiedUnixDateTime BIGINT NOT NULL -- in GMT 0 + ,CreatedUnixDateTime BIGINT NOT NULL -- in GMT 0 + + ,PRIMARY KEY (DetailLevel, PosX, PosZ) +); diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index 47c749378..2a2f2e905 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -1,2 +1,3 @@ 0010-sqlite-createInitialDataTables.sql +0020-sqlite-createGeneratedFullDataSourceTables.sql diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java index fbabbe52e..745014dec 100644 --- a/core/src/test/java/testItems/sql/TestCompoundKeyDto.java +++ b/core/src/test/java/testItems/sql/TestCompoundKeyDto.java @@ -20,8 +20,7 @@ package testItems.sql; import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; public class TestCompoundKeyDto implements IBaseDTO { diff --git a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java index 114e4b57d..3b7ed530e 100644 --- a/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java +++ b/core/src/test/java/testItems/sql/TestCompoundKeyRepo.java @@ -20,7 +20,7 @@ package testItems.sql; import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.sql.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java index 748c3a214..347e6d686 100644 --- a/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java +++ b/core/src/test/java/testItems/sql/TestPrimaryKeyRepo.java @@ -19,7 +19,7 @@ package testItems.sql; -import com.seibel.distanthorizons.core.sql.AbstractDhRepo; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/core/src/test/java/testItems/sql/TestSingleKeyDto.java b/core/src/test/java/testItems/sql/TestSingleKeyDto.java index 6de1352f3..90709a162 100644 --- a/core/src/test/java/testItems/sql/TestSingleKeyDto.java +++ b/core/src/test/java/testItems/sql/TestSingleKeyDto.java @@ -19,7 +19,7 @@ package testItems.sql; -import com.seibel.distanthorizons.core.sql.IBaseDTO; +import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; public class TestSingleKeyDto implements IBaseDTO { diff --git a/core/src/test/java/tests/DhRepoSqliteTest.java b/core/src/test/java/tests/DhRepoSqliteTest.java index 8d15c5c6a..066106110 100644 --- a/core/src/test/java/tests/DhRepoSqliteTest.java +++ b/core/src/test/java/tests/DhRepoSqliteTest.java @@ -21,6 +21,7 @@ package tests; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.sql.DatabaseUpdater; +import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -34,7 +35,7 @@ import java.sql.SQLException; import java.util.Map; /** - * Validates {@link com.seibel.distanthorizons.core.sql.AbstractDhRepo} is set up correctly. + * Validates {@link AbstractDhRepo} is set up correctly. */ public class DhRepoSqliteTest { From 89012711ce54eea61dd28269df360a5d2cbae656 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 14:02:54 -0600 Subject: [PATCH 005/183] rename ThreadPools -> ThreadPoolUtil --- .../distanthorizons/core/api/internal/SharedApi.java | 8 ++++---- .../render/bufferBuilding/ColumnRenderBufferBuilder.java | 6 +++--- .../core/dataObjects/transformers/ChunkToLodBuilder.java | 6 +++--- .../core/file/AbstractLegacyDataSourceHandler.java | 6 +++--- .../core/file/AbstractNewDataSourceHandler.java | 9 +++------ .../core/file/fullDatafile/NewFullDataFileHandler.java | 4 ++-- .../core/file/renderfile/RenderSourceFileHandler.java | 4 ++-- .../core/generation/WorldGenerationQueue.java | 8 ++++---- .../com/seibel/distanthorizons/core/util/ThreadUtil.java | 4 ++-- .../threading/{ThreadPools.java => ThreadPoolUtil.java} | 8 ++++---- 10 files changed, 30 insertions(+), 33 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/util/threading/{ThreadPools.java => ThreadPoolUtil.java} (96%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java index b0499375d..f72484df5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java @@ -32,7 +32,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.objects.Pair; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.world.*; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper; @@ -102,11 +102,11 @@ public class SharedApi // access the MC level at inappropriate times, which can cause exceptions if (currentWorld != null) { - ThreadPools.setupThreadPools(); + ThreadPoolUtil.setupThreadPools(); } else { - ThreadPools.shutdownThreadPools(); + ThreadPoolUtil.shutdownThreadPools(); DebugRenderer.clearRenderables(); MC_RENDER.clearTargetFrameBuffer(); @@ -285,7 +285,7 @@ public class SharedApi private static void bakeChunkLightingAndSendToLevelAsync(IChunkWrapper chunkWrapper, @Nullable ArrayList neighbourChunkList, IDhLevel dhLevel) { // lighting the chunk needs to be done on a separate thread to prevent lagging any of the event threads - ThreadPoolExecutor executor = ThreadPools.getLightPopulatorExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getLightPopulatorExecutor(); if (executor == null) { return; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 6b75be291..f133f424e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -34,7 +34,7 @@ import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.objects.Reference; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -67,8 +67,8 @@ public class ColumnRenderBufferBuilder IDhClientLevel clientLevel, Reference renderBufferRef, ColumnRenderSource renderSource, ColumnRenderSource[] adjData) { - ThreadPoolExecutor bufferBuilderExecutor = ThreadPools.getBufferBuilderExecutor(); - ThreadPoolExecutor bufferUploaderExecutor = ThreadPools.getBufferUploaderExecutor(); + ThreadPoolExecutor bufferBuilderExecutor = ThreadPoolUtil.getBufferBuilderExecutor(); + ThreadPoolExecutor bufferUploaderExecutor = ThreadPoolUtil.getBufferUploaderExecutor(); if ((bufferBuilderExecutor == null || bufferBuilderExecutor.isTerminated()) || (bufferUploaderExecutor == null || bufferUploaderExecutor.isTerminated())) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index 8d27273c7..a3a727f65 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -27,7 +27,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataS import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.pos.DhChunkPos; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.LogManager; @@ -82,7 +82,7 @@ public class ChunkToLodBuilder implements AutoCloseable // TODO why on tick? public void tick() { - int threadCount = ThreadPools.getWorkerThreadCount(); + int threadCount = ThreadPoolUtil.getWorkerThreadCount(); if (this.runningCount.get() >= threadCount) { return; @@ -102,7 +102,7 @@ public class ChunkToLodBuilder implements AutoCloseable return; } - ThreadPoolExecutor lodBuilderExecutor = ThreadPools.getChunkToLodBuilderExecutor(); + ThreadPoolExecutor lodBuilderExecutor = ThreadPoolUtil.getChunkToLodBuilderExecutor(); if (lodBuilderExecutor == null) { return; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 5b7a4f97f..80e7ae3fe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -11,7 +11,7 @@ import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -128,7 +128,7 @@ public abstract class AbstractLegacyDataSourceHandler getAsync(DhSectionPos pos) { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { return CompletableFuture.completedFuture(null); @@ -190,7 +190,7 @@ public abstract class AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { return CompletableFuture.completedFuture(null); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 217e1958b..c91c4319d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -9,18 +9,15 @@ import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.TimerUtil; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; import java.util.ArrayList; -import java.util.Enumeration; import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.locks.ReentrantLock; @@ -123,7 +120,7 @@ public abstract class AbstractNewDataSourceHandler @Override public CompletableFuture getAsync(DhSectionPos pos) { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { return CompletableFuture.completedFuture(null); @@ -171,7 +168,7 @@ public abstract class AbstractNewDataSourceHandler @Override public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) { - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { return CompletableFuture.completedFuture(null); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 6754f1d00..1fe3796d3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -32,7 +32,7 @@ import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; import com.seibel.distanthorizons.core.util.ThreadUtil; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -211,7 +211,7 @@ public class NewFullDataFileHandler if (updatePosList.size() != 0) { // stop if the file handler has been shut down - ThreadPoolExecutor executor = ThreadPools.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { this.updateQueueThreadRunningRef.set(false); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index b50dcc1d3..70ce500bf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -33,7 +33,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import java.io.IOException; @@ -126,7 +126,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler { try @@ -472,7 +472,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender granularity, targetDataDetail, generatorMode, - ThreadPools.getWorldGenExecutor(), + ThreadPoolUtil.getWorldGenExecutor(), (DhApiChunk dataPoints) -> { try @@ -572,7 +572,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender try { int waitTimeInSeconds = 3; - ThreadPoolExecutor executor = ThreadPools.getWorldGenExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getWorldGenExecutor(); if (executor != null && !executor.awaitTermination(waitTimeInSeconds, TimeUnit.SECONDS)) { LOGGER.warn("World generator thread pool shutdown didn't complete after [" + waitTimeInSeconds + "] seconds. Some world generator requests may still be running."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java index 1ac4567d0..792b2dc9f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.config.types.ConfigEntry; import com.seibel.distanthorizons.core.util.threading.DhThreadFactory; import com.seibel.distanthorizons.core.util.threading.RateLimitedThreadPoolExecutor; -import com.seibel.distanthorizons.core.util.threading.ThreadPools; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.coreapi.ModInfo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,7 +33,7 @@ import java.util.concurrent.*; /** * Handles thread pool creation. * - * @see ThreadPools + * @see ThreadPoolUtil * @see TimerUtil */ public class ThreadUtil diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPools.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java similarity index 96% rename from core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPools.java rename to core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java index 279486bf6..6924ab7b2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPools.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java @@ -32,7 +32,7 @@ import java.util.concurrent.ThreadPoolExecutor; * * @see ThreadUtil */ -public class ThreadPools +public class ThreadPoolUtil { //=========================// // standalone thread pools // @@ -143,9 +143,9 @@ public class ThreadPools // worker threads - ThreadPools.lightPopulatorThreadPool.shutdownExecutorService(); - ThreadPools.chunkToLodBuilderThreadPool.shutdownExecutorService(); - ThreadPools.bufferBuilderThreadPool.shutdownExecutorService(); + ThreadPoolUtil.lightPopulatorThreadPool.shutdownExecutorService(); + ThreadPoolUtil.chunkToLodBuilderThreadPool.shutdownExecutorService(); + ThreadPoolUtil.bufferBuilderThreadPool.shutdownExecutorService(); workerThreadSemaphore = null; From 2a39a4cdc4fdcf9a7b4d88d0be522c13c41939d4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 15:20:51 -0600 Subject: [PATCH 006/183] Add several TODO comments and minor reformatting --- .../dataObjects/fullData/FullDataPointIdMap.java | 13 +++++++++---- .../core/file/AbstractNewDataSourceHandler.java | 1 - .../seibel/distanthorizons/core/pos/DhLodPos.java | 5 +++++ .../core/util/threading/DhThreadFactory.java | 4 +--- .../threading/RateLimitedThreadPoolExecutor.java | 3 ++- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index a0d391554..4df982758 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -36,10 +36,15 @@ import java.util.HashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; /** - * WARNING: This is not THREAD-SAFE! - *

- * Used to map a numerical IDs to a Biome/BlockState pair. - * + * WARNING: This is not THREAD-SAFE!

+ * + * Used to map a numerical IDs to a Biome/BlockState pair.

+ * + * TODO the serializing of this map might be really big + * since it stringifies every block and biome name, which is quite bulky. + * It might be worth while to have a biome and block ID that then both get mapped + * to the data point ID to reduce file size. + * * @author Leetom */ public class FullDataPointIdMap diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index c91c4319d..ac003e9f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -181,7 +181,6 @@ public abstract class AbstractNewDataSourceHandler return CompletableFuture.runAsync(() -> { this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource, true); - }, executor); } catch (RejectedExecutionException ignore) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java index b2f3cb1f4..1706e9365 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java @@ -31,6 +31,11 @@ import java.util.Objects; * * @author Leetom * @version 2022-11-6 + * + * @deprecated TODO replace entirely with DhSectionPos, + * we don't need to have a full fledged position object for + * positions inside a LOD, we only need this position object + * to get to/from the LOD section. */ @Deprecated public class DhLodPos implements Comparable diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/DhThreadFactory.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/DhThreadFactory.java index 8eb41b5bc..129e628f2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/DhThreadFactory.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/DhThreadFactory.java @@ -32,12 +32,10 @@ import org.jetbrains.annotations.NotNull; /** * Just a simple ThreadFactory to name ExecutorService * threads, which is helpful when debugging. - * - * @author James Seibel */ public class DhThreadFactory implements ThreadFactory { - private static final Logger LOGGER = DhLoggerBuilder.getLogger(MethodHandles.lookup().lookupClass().getSimpleName()); + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); public final String threadName; public final int priority; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/RateLimitedThreadPoolExecutor.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/RateLimitedThreadPoolExecutor.java index 5b27bb6fb..4b49ade2c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/RateLimitedThreadPoolExecutor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/RateLimitedThreadPoolExecutor.java @@ -62,7 +62,7 @@ public class RateLimitedThreadPoolExecutor extends ThreadPoolExecutor { super(corePoolSize, corePoolSize, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), + new LinkedBlockingQueue<>(), // TODO using a PriorityBlockingQueue would be nice to allow for prioritizing tasks, but then all tasks must be Comparable threadFactory); this.runTimeRatio = runTimeRatio; @@ -75,6 +75,7 @@ public class RateLimitedThreadPoolExecutor extends ThreadPoolExecutor // overrides // //===========// + @Override protected void beforeExecute(Thread thread, Runnable runnable) { super.beforeExecute(thread, runnable); From b8e03a2144a79be7a3e0fd0c109f8832d62764bf Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 15:21:22 -0600 Subject: [PATCH 007/183] Deprecate DhApiScreenResizeEvent, replace with DhApiColorDepthTextureCreatedEvent --- .../DhApiColorDepthTextureCreatedEvent.java | 77 +++++++++++++++++++ .../DhApiScreenResizeEvent.java | 4 +- .../core/render/renderer/LodRenderer.java | 12 ++- 3 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiColorDepthTextureCreatedEvent.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiColorDepthTextureCreatedEvent.java b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiColorDepthTextureCreatedEvent.java new file mode 100644 index 000000000..c67cd547d --- /dev/null +++ b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiColorDepthTextureCreatedEvent.java @@ -0,0 +1,77 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.api.methods.events.abstractEvents; + +import com.seibel.distanthorizons.api.methods.events.interfaces.IDhApiEvent; +import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiEventParam; + +/** + * Called whenever Distant Horizons (re)creates + * the color and depth textures it renders to.
+ * + * @author James Seibel + * @version 2024-3-2 + * @since API 1.1.0 + */ +public abstract class DhApiColorDepthTextureCreatedEvent implements IDhApiEvent +{ + /** Fired before Distant Horizons creates. */ + public abstract void onResize(DhApiEventParam event); + + + //=========================// + // internal DH API methods // + //=========================// + + @Override + public final void fireEvent(DhApiEventParam event) { this.onResize(event); } + + + //==================// + // parameter object // + //==================// + + public static class EventParam + { + /** Measured in pixels */ + public final int previousWidth; + /** Measured in pixels */ + public final int previousHeight; + + /** Measured in pixels */ + public final int newWidth; + /** Measured in pixels */ + public final int newHeight; + + + public EventParam( + int previousWidth, int previousHeight, + int newWidth, int newHeight) + { + this.previousWidth = previousWidth; + this.previousHeight = previousHeight; + + this.newWidth = newWidth; + this.newHeight = newHeight; + + } + } + +} \ No newline at end of file diff --git a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java index 64407bb5c..2fdad3b66 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java @@ -23,13 +23,13 @@ import com.seibel.distanthorizons.api.methods.events.interfaces.IDhApiEvent; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiEventParam; /** - * Called before Distant Horizons starts rendering a buffer.
- * This event cannot be cancelled, use {@link DhApiBeforeRenderEvent} if you want to cancel rendering. + * TODO remove * * @author James Seibel * @version 2023-1-23 * @since API 1.1.0 */ +@Deprecated // TODO remove before stable release and write a merge request for Iris public abstract class DhApiScreenResizeEvent implements IDhApiEvent { /** Fired immediately before Distant Horizons handles the screen resize. */ diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index bd3e4eea1..40986bd19 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.core.render.renderer; -import com.seibel.distanthorizons.api.enums.rendering.EDhApiRenderPass; import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiFramebuffer; import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiShaderProgram; @@ -41,7 +40,6 @@ import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.render.glObject.buffer.QuadElementBuffer; import com.seibel.distanthorizons.core.render.glObject.texture.*; import com.seibel.distanthorizons.core.render.renderer.shaders.*; -import com.seibel.distanthorizons.core.util.RenderUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; @@ -57,8 +55,6 @@ import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import com.seibel.distanthorizons.coreapi.util.math.Vec3d; import com.seibel.distanthorizons.coreapi.util.math.Vec3f; import org.apache.logging.log4j.LogManager; -import org.joml.Matrix4f; -import org.joml.Matrix4fc; import org.lwjgl.opengl.GL32; import java.awt.*; @@ -749,6 +745,14 @@ public class LodRenderer this.cachedHeight = MC_RENDER.getTargetFrameBufferViewportHeight(); + // TODO add on texture (re)created event + // https://discord.com/channels/881614130614767666/1211290858134052894/1211431000580554752 + ApiEventInjector.INSTANCE.fireAllEvents(DhApiColorDepthTextureCreatedEvent.class, + new DhApiColorDepthTextureCreatedEvent.EventParam( + oldWidth, oldHeight, + this.cachedWidth, this.cachedHeight + )); + ApiEventInjector.INSTANCE.fireAllEvents(DhApiScreenResizeEvent.class, new DhApiScreenResizeEvent.EventParam( oldWidth, oldHeight, From 47391028d8b90f284592a7cda7102b9b87bdef2d Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 16:21:32 -0600 Subject: [PATCH 008/183] Prevent re-saving unmodified full data --- .../sources/CompleteFullDataSource.java | 2 +- .../fullData/sources/NewFullDataSource.java | 52 +++++++++++++++---- .../render/ColumnRenderSource.java | 7 ++- .../file/AbstractNewDataSourceHandler.java | 22 ++++---- .../core/file/IDataSource.java | 3 +- 5 files changed, 62 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 4ed5e4cc7..7b52a298f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -99,7 +99,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa @Deprecated @Override - public void update(NewFullDataSource dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } + public boolean update(NewFullDataSource dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 8eddbcc50..b6b9c2deb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Arrays; import java.util.concurrent.locks.ReentrantLock; /** @@ -150,8 +151,8 @@ public class NewFullDataSource implements IDataSource public SingleColumnFullDataAccessor get(int relX, int relZ) { return new SingleColumnFullDataAccessor(this.mapping, this.dataPoints, relativePosToIndex(relX, relZ)); } @Override - public void update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { this.update(inputDataSource); } - public void update(NewFullDataSource inputDataSource) + public boolean update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } + public boolean update(NewFullDataSource inputDataSource) { byte thisDetailLevel = this.pos.getDetailLevel(); byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); @@ -160,7 +161,7 @@ public class NewFullDataSource implements IDataSource // determine the mapping changes necessary for the input to map onto this datasource int[] remappedIds = this.mapping.mergeAndReturnRemappedEntityIds(inputDataSource.mapping); - boolean dataChanged = false; + boolean dataChanged; if (inputDetailLevel == thisDetailLevel) { dataChanged = this.updateFromSameDetailLevel(inputDataSource, remappedIds); @@ -171,13 +172,11 @@ public class NewFullDataSource implements IDataSource } else { - // TODO what should happen here? - // other detail levels aren't supported since it would be more difficult to maintain // and would lead to edge cases that don't necessarily need to be supported // (IE what do you do when the input is smaller than a single datapoint in the receiving data source?) // instead it's better to just percolate the updates up - //throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); + throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); } if (dataChanged && this.pos.getDetailLevel() < NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL) @@ -185,6 +184,8 @@ public class NewFullDataSource implements IDataSource // mark that this data source should be applied to its parent this.applyToParent = true; } + + return dataChanged; } public boolean updateFromSameDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) { @@ -212,13 +213,22 @@ public class NewFullDataSource implements IDataSource if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value && thisGenState <= inputGenState) { + long[] oldDataArray = this.dataPoints[index]; + + // copy over the new data this.dataPoints[index] = new long[newDataArray.length]; System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); this.remapDataColumn(index, remappedIds); - this.columnGenerationSteps[index] = inputGenState; + // we only need to see if the data was changed in one column + if (!dataChanged) + { + // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same + dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[index]); + } - dataChanged = true; // TODO contents of the arrays should be compared to prevent re-writing the same data + this.columnGenerationSteps[index] = inputGenState; + this.isEmpty = false; } } } @@ -265,13 +275,20 @@ public class NewFullDataSource implements IDataSource int recipientZ = (z / 2) + recipientOffsetZ; int recipientIndex = relativePosToIndex(recipientX, recipientZ); + long[] oldDataArray = this.dataPoints[recipientIndex]; + this.columnGenerationSteps[recipientIndex] = inputGenStep; this.dataPoints[recipientIndex] = inputDataArray; this.remapDataColumn(recipientIndex, remappedIds); - this.isEmpty = false; + // we only need to see if the data was changed in one column + if (!dataChanged) + { + // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same + dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[recipientIndex]); + } - dataChanged = true; // TODO contents of the arrays should probably be compared or something + this.isEmpty = false; } } } @@ -291,6 +308,21 @@ public class NewFullDataSource implements IDataSource dataColumn[i] = FullDataPointUtil.remap(remappedIds, dataColumn[i]); } } + private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) + { + if (oldDataArray == null || oldDataArray.length != newDataArray.length) + { + // new data was added/removed + return true; + } + else + { + // check if the new column data is different + int oldArrayHash = Arrays.hashCode(oldDataArray); + int newArrayHash = Arrays.hashCode(newDataArray); + return (newArrayHash != oldArrayHash); + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index fdebf427e..42870ff5d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -290,7 +290,7 @@ public class ColumnRenderSource implements IDataSource } @Override - public void update(NewFullDataSource inputDataSource, IDhClientLevel level) + public boolean update(NewFullDataSource inputDataSource, IDhClientLevel level) { final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputDataSource.getSectionPos() + "]. Error:"; @@ -302,7 +302,7 @@ public class ColumnRenderSource implements IDataSource if (Thread.interrupted()) { LOGGER.warn(errorMessagePrefix + "write interrupted."); - return; + return false; } @@ -340,11 +340,14 @@ public class ColumnRenderSource implements IDataSource } + if (dataChanged) { this.localVersion.incrementAndGet(); this.markNotEmpty(); } + + return dataChanged; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index ac003e9f4..8978ae0af 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -210,19 +210,21 @@ public abstract class AbstractNewDataSourceHandler // get or create the data source TDataSource dataSource = this.get(pos); - dataSource.update(inputData, this.level); // TODO only write to database and send update signal if data was changed + boolean dataModified = dataSource.update(inputData, this.level); - - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(dataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + if (dataModified) { - if (listener != null) + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(dataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - listener.OnDataSourceUpdated(dataSource); + if (listener != null) + { + listener.OnDataSourceUpdated(dataSource); + } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 5a61dd2e7..c4ae71ee1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -25,7 +25,8 @@ public interface IDataSource extends IBaseDTO Date: Sat, 2 Mar 2024 21:31:21 -0600 Subject: [PATCH 009/183] Change FullDataPointUtil to get/set sky and block lighting separately The binary format is identical the only difference is the getter/setter methods --- .../objects/data/DhApiTerrainDataPoint.java | 8 +++-- .../methods/data/DhApiTerrainDataRepo.java | 3 +- .../FullDataToRenderDataTransformer.java | 5 ++-- .../transformers/LodDataBuilder.java | 16 ++++++---- .../core/util/FullDataPointUtil.java | 30 ++++++++++++------- .../core/util/RenderDataPointUtil.java | 14 --------- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/objects/data/DhApiTerrainDataPoint.java b/api/src/main/java/com/seibel/distanthorizons/api/objects/data/DhApiTerrainDataPoint.java index d78ca58e2..0c70dfbfb 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/objects/data/DhApiTerrainDataPoint.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/objects/data/DhApiTerrainDataPoint.java @@ -40,7 +40,8 @@ public class DhApiTerrainDataPoint */ public final byte detailLevel; - public final int lightLevel; + public final int blockLightLevel; + public final int skyLightLevel; public final int topYBlockPos; public final int bottomYBlockPos; @@ -49,11 +50,12 @@ public class DhApiTerrainDataPoint - public DhApiTerrainDataPoint(byte detailLevel, int lightLevel, int topYBlockPos, int bottomYBlockPos, IDhApiBlockStateWrapper blockStateWrapper, IDhApiBiomeWrapper biomeWrapper) + public DhApiTerrainDataPoint(byte detailLevel, int blockLightLevel, int skyLightLevel, int topYBlockPos, int bottomYBlockPos, IDhApiBlockStateWrapper blockStateWrapper, IDhApiBiomeWrapper biomeWrapper) { this.detailLevel = detailLevel; - this.lightLevel = lightLevel; + this.blockLightLevel = blockLightLevel; + this.skyLightLevel = skyLightLevel; this.topYBlockPos = topYBlockPos; this.bottomYBlockPos = bottomYBlockPos; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 0cc87f208..c67342fac 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -290,7 +290,8 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo int topY = bottomY + height; return new DhApiTerrainDataPoint(detailLevel, - FullDataPointUtil.getLight(dataPoint), topY, bottomY, + FullDataPointUtil.getBlockLight(dataPoint), FullDataPointUtil.getSkyLight(dataPoint), + topY, bottomY, blockState, biomeWrapper); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 6c6451ecb..7ac3a5e9d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -178,7 +178,8 @@ public class FullDataToRenderDataTransformer int bottomY = FullDataPointUtil.getBottomY(fullData); int blockHeight = FullDataPointUtil.getHeight(fullData); int id = FullDataPointUtil.getId(fullData); - int light = FullDataPointUtil.getLight(fullData); + int blockLight = FullDataPointUtil.getBlockLight(fullData); + int skyLight = FullDataPointUtil.getSkyLight(fullData); // TODO how should corrupted data be handled? // TODO why is the full data corrupted in the first place? FullDataPointUtil hasn't been changed in a long time, could one of the full data point objects be corrupted? @@ -257,7 +258,7 @@ public class FullDataToRenderDataTransformer // add the block isVoid = false; - long columnData = RenderDataPointUtil.createDataPoint(bottomY + blockHeight, bottomY, color, light, block.getIrisBlockMaterialId()); + long columnData = RenderDataPointUtil.createDataPoint(bottomY + blockHeight, bottomY, color, skyLight, blockLight, block.getIrisBlockMaterialId()); column.set(columnOffset, columnData); columnOffset++; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 65c043fda..7b560e3d5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -137,7 +137,8 @@ public class LodDataBuilder IBlockStateWrapper blockState = AIR; int mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting - byte light = (byte) ((chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ) << 4) + chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ)); + byte blockLight = (byte) chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ); + byte skyLight = (byte) chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ); // determine the starting Y Pos @@ -171,19 +172,21 @@ public class LodDataBuilder { IBiomeWrapper newBiome = chunkWrapper.getBiome(chunkX, y, chunkZ); IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); - byte newLight = (byte) ((chunkWrapper.getBlockLight(chunkX, y + 1, chunkZ) << 4) + chunkWrapper.getSkyLight(chunkX, y + 1, chunkZ)); + byte newBlockLight = (byte) chunkWrapper.getBlockLight(chunkX, y + 1, chunkZ); + byte newSkyLight = (byte) chunkWrapper.getSkyLight(chunkX, y + 1, chunkZ); if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) { - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), light)); + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); biome = newBiome; blockState = newBlockState; mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); - light = newLight; + blockLight = newBlockLight; + skyLight = newSkyLight; lastY = y; } } - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), light)); + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); dataSource.setSingleColumn(longs.toLongArray(), chunkX + chunkOffsetX, @@ -228,7 +231,8 @@ public class LodDataBuilder id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, - (byte) (dataPoint.lightLevel) + (byte) (dataPoint.blockLightLevel), + (byte) (dataPoint.skyLightLevel) ); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index 9360c3ac5..04d65bb4f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -60,23 +60,26 @@ public class FullDataPointUtil public static final int ID_WIDTH = 32; public static final int DP_WIDTH = 12; public static final int Y_WIDTH = 12; - public static final int LIGHT_WIDTH = 8; + public static final int BLOCK_LIGHT_WIDTH = 4; + public static final int SKY_LIGHT_WIDTH = 4; public static final int ID_OFFSET = 0; public static final int DP_OFFSET = ID_OFFSET + ID_WIDTH; /** indicates the Y position where the LOD starts relative to the level's minimum height */ public static final int Y_OFFSET = DP_OFFSET + DP_WIDTH; - public static final int LIGHT_OFFSET = Y_OFFSET + Y_WIDTH; + public static final int BLOCK_LIGHT_OFFSET = Y_OFFSET + Y_WIDTH; + public static final int SKY_LIGHT_OFFSET = BLOCK_LIGHT_OFFSET + BLOCK_LIGHT_WIDTH; public static final long ID_MASK = Integer.MAX_VALUE; public static final long INVERSE_ID_MASK = ~ID_MASK; public static final int DP_MASK = (int) Math.pow(2, DP_WIDTH) - 1; public static final int Y_MASK = (int) Math.pow(2, Y_WIDTH) - 1; - public static final int LIGHT_MASK = (int) Math.pow(2, LIGHT_WIDTH) - 1; + public static final int BLOCK_LIGHT_MASK = (int) Math.pow(2, BLOCK_LIGHT_WIDTH) - 1; + public static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1; /** creates a new datapoint with the given values */ - public static long encode(int id, int depth, int y, byte lightPair) + public static long encode(int id, int depth, int y, byte blockLight, byte skyLight) { LodUtil.assertTrue(y >= 0 && y < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y[{}] out of range!", y); LodUtil.assertTrue(depth > 0 && depth < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with depth[{}] out of range!", depth); @@ -86,10 +89,16 @@ public class FullDataPointUtil data |= id & ID_MASK; data |= (long) (depth & DP_MASK) << DP_OFFSET; data |= (long) (y & Y_MASK) << Y_OFFSET; - data |= (long) lightPair << LIGHT_OFFSET; - LodUtil.assertTrue(getId(data) == id && getHeight(data) == depth && getBottomY(data) == y && getLight(data) == Byte.toUnsignedInt(lightPair), - "Trying to create datapoint with id[{}], depth[{}], y[{}], lightPair[{}] but got id[{}], depth[{}], y[{}], lightPair[{}]!", - id, depth, y, Byte.toUnsignedInt(lightPair), getId(data), getHeight(data), getBottomY(data), getLight(data)); + data |= (long) blockLight << BLOCK_LIGHT_OFFSET; + data |= (long) skyLight << SKY_LIGHT_OFFSET; + + // confirm the written data can still be retrieved + LodUtil.assertTrue( + getId(data) == id && getHeight(data) == depth && getBottomY(data) == y && getBlockLight(data) == Byte.toUnsignedInt(blockLight) && getSkyLight(data) == Byte.toUnsignedInt(skyLight), + "Trying to create datapoint with " + + "id[" + id + "], height[" + depth + "], minY[" + y + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + + "but got " + + "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); return data; } @@ -100,9 +109,10 @@ public class FullDataPointUtil public static int getHeight(long data) { return (int) ((data >> DP_OFFSET) & DP_MASK); } /** Returns the block position of the bottom vertices for this LOD relative to the level's minimum height. */ public static int getBottomY(long data) { return (int) ((data >> Y_OFFSET) & Y_MASK); } - public static int getLight(long data) { return (int) ((data >> LIGHT_OFFSET) & LIGHT_MASK); } + public static int getBlockLight(long data) { return (int) ((data >> BLOCK_LIGHT_OFFSET) & BLOCK_LIGHT_MASK); } + public static int getSkyLight(long data) { return (int) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } - public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",Light:" + getLight(data) + "]"; } + public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",BlockLight:" + getBlockLight(data) + ",SkyLight:" + getSkyLight(data) + "]"; } /** Remaps the biome/blockState ID of the given datapoint */ @Contract(pure = true) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 1a110fc2a..240a0b4aa 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -121,20 +121,6 @@ public class RenderDataPointUtil height, depth, lightSky, lightBlock, irisBlockMaterialId); } - public static long createDataPoint(int height, int depth, int color, int light, int irisBlockMaterialId) - { - LodUtil.assertTrue(light >= 0 && light < 256, "Raw Light value must be between 0 and 255!"); - - return createDataPoint( - ColorUtil.getAlpha(color), - ColorUtil.getRed(color), - ColorUtil.getGreen(color), - ColorUtil.getBlue(color), - height, depth, - light % 16, light / 16, - irisBlockMaterialId); - } - public static long createDataPoint(int alpha, int red, int green, int blue, int height, int depth, int lightSky, int lightBlock, int irisBlockMaterialId) { LodUtil.assertTrue(height >= 0 && height < MAX_WORLD_Y_SIZE, "Trying to create datapoint with height[" + height + "] out of range!"); From e15ffe10c93672cc3410308a57781607f1729a2c Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 2 Mar 2024 21:36:54 -0600 Subject: [PATCH 010/183] Improve FullDataPointUtil parameter names --- .../core/util/FullDataPointUtil.java | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index 04d65bb4f..9862120e6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -19,8 +19,6 @@ package com.seibel.distanthorizons.core.util; -// - import org.jetbrains.annotations.Contract; /** @@ -34,16 +32,16 @@ import org.jetbrains.annotations.Contract; * DataPoint Format:
* * ID: blockState id
- * Y: Height(signed)
- * DP: Depth (Depth means the length of the block!)
+ * MY: Min Y Height (unsigned, relative to the minimum level height)
+ * HI: Height (how tall this data point is in blocks)
* BL: Block light
* SL: Sky light

* * =======Bit layout=======
* BL BL BL BL SL SL SL SL <-- Top bits
- * YY YY YY YY YY YY YY YY
- * YY YY YY YY DP DP DP DP
- * DP DP DP DP DP DP DP DP
+ * MY MY MY MY MY MY MY MY
+ * MY MY MY MY HI HI HI HI
+ * HI HI HI HI HI HI HI HI
* ID ID ID ID ID ID ID ID
* ID ID ID ID ID ID ID ID
* ID ID ID ID ID ID ID ID
@@ -52,51 +50,54 @@ import org.jetbrains.annotations.Contract; * * @see RenderDataPointUtil */ +// TODO make a legacy version of this specifically for CompleteFullDataSource just in case things change in the future public class FullDataPointUtil { /** Represents the data held by an empty data point */ public static final int EMPTY_DATA_POINT = 0; public static final int ID_WIDTH = 32; - public static final int DP_WIDTH = 12; - public static final int Y_WIDTH = 12; + public static final int HEIGHT_WIDTH = 12; + public static final int MIN_Y_WIDTH = 12; public static final int BLOCK_LIGHT_WIDTH = 4; public static final int SKY_LIGHT_WIDTH = 4; + public static final int ID_OFFSET = 0; - public static final int DP_OFFSET = ID_OFFSET + ID_WIDTH; + public static final int HEIGHT_OFFSET = ID_OFFSET + ID_WIDTH; /** indicates the Y position where the LOD starts relative to the level's minimum height */ - public static final int Y_OFFSET = DP_OFFSET + DP_WIDTH; - public static final int BLOCK_LIGHT_OFFSET = Y_OFFSET + Y_WIDTH; + public static final int MIN_Y_OFFSET = HEIGHT_OFFSET + HEIGHT_WIDTH; + public static final int BLOCK_LIGHT_OFFSET = MIN_Y_OFFSET + MIN_Y_WIDTH; public static final int SKY_LIGHT_OFFSET = BLOCK_LIGHT_OFFSET + BLOCK_LIGHT_WIDTH; public static final long ID_MASK = Integer.MAX_VALUE; public static final long INVERSE_ID_MASK = ~ID_MASK; - public static final int DP_MASK = (int) Math.pow(2, DP_WIDTH) - 1; - public static final int Y_MASK = (int) Math.pow(2, Y_WIDTH) - 1; + public static final int HEIGHT_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1; + public static final int MIN_Y_MASK = (int) Math.pow(2, MIN_Y_WIDTH) - 1; public static final int BLOCK_LIGHT_MASK = (int) Math.pow(2, BLOCK_LIGHT_WIDTH) - 1; public static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1; - /** creates a new datapoint with the given values */ - public static long encode(int id, int depth, int y, byte blockLight, byte skyLight) + /** + * creates a new datapoint with the given values + * @param relMinY relative to the minimum level Y value + */ + public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) { - LodUtil.assertTrue(y >= 0 && y < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y[{}] out of range!", y); - LodUtil.assertTrue(depth > 0 && depth < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with depth[{}] out of range!", depth); - LodUtil.assertTrue(y + depth <= RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y+depth[{}] out of range!", y + depth); + LodUtil.assertTrue(relMinY >= 0 && relMinY < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y["+relMinY+"] out of range!"); + LodUtil.assertTrue(height > 0 && height < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with height["+height+"] out of range!"); + LodUtil.assertTrue(relMinY + height <= RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y+depth["+(relMinY+height)+"] out of range!"); long data = 0; data |= id & ID_MASK; - data |= (long) (depth & DP_MASK) << DP_OFFSET; - data |= (long) (y & Y_MASK) << Y_OFFSET; + data |= (long) (height & HEIGHT_MASK) << HEIGHT_OFFSET; + data |= (long) (relMinY & MIN_Y_MASK) << MIN_Y_OFFSET; data |= (long) blockLight << BLOCK_LIGHT_OFFSET; data |= (long) skyLight << SKY_LIGHT_OFFSET; - // confirm the written data can still be retrieved - LodUtil.assertTrue( - getId(data) == id && getHeight(data) == depth && getBottomY(data) == y && getBlockLight(data) == Byte.toUnsignedInt(blockLight) && getSkyLight(data) == Byte.toUnsignedInt(skyLight), + LodUtil.assertTrue(getId(data) == id && getHeight(data) == height && getBottomY(data) == relMinY && getBlockLight(data) == Byte.toUnsignedInt(blockLight) && getSkyLight(data) == Byte.toUnsignedInt(skyLight), "Trying to create datapoint with " + - "id[" + id + "], height[" + depth + "], minY[" + y + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + + "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + "but got " + "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); @@ -106,12 +107,16 @@ public class FullDataPointUtil /** Returns the BlockState/Biome pair ID used to identify this LOD's color */ public static int getId(long data) { return (int) (data & ID_MASK); } /** Returns how many blocks tall this LOD is. */ - public static int getHeight(long data) { return (int) ((data >> DP_OFFSET) & DP_MASK); } - /** Returns the block position of the bottom vertices for this LOD relative to the level's minimum height. */ - public static int getBottomY(long data) { return (int) ((data >> Y_OFFSET) & Y_MASK); } + public static int getHeight(long data) { return (int) ((data >> HEIGHT_OFFSET) & HEIGHT_MASK); } + /** + * Returns the unsigned block position of the bottom vertices for this LOD relative to the level's minimum height. + * Should be between 0 and {@link RenderDataPointUtil#MAX_WORLD_Y_SIZE} + */ + public static int getBottomY(long data) { return (int) ((data >> MIN_Y_OFFSET) & MIN_Y_MASK); } public static int getBlockLight(long data) { return (int) ((data >> BLOCK_LIGHT_OFFSET) & BLOCK_LIGHT_MASK); } public static int getSkyLight(long data) { return (int) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } + public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",BlockLight:" + getBlockLight(data) + ",SkyLight:" + getSkyLight(data) + "]"; } /** Remaps the biome/blockState ID of the given datapoint */ From dc687f70aeffdf905ee087685421823998898219 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 3 Mar 2024 14:53:10 -0600 Subject: [PATCH 011/183] Add LodUtil light constants --- .../java/com/seibel/distanthorizons/core/util/LodUtil.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java index edfdc8cf1..28bfdd83e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java @@ -118,6 +118,13 @@ public class LodUtil public static final int REGION_WIDTH_IN_CHUNKS = REGION_WIDTH / CHUNK_WIDTH; + /** maximum possible light level handled by Minecraft */ + public static final byte MAX_MC_LIGHT = 15; + /** lowest possible light level handled by Minecraft */ + public static final byte MIN_MC_LIGHT = 0; + + + /** * This regex finds any characters that are invalid for use in a windows * (and by extension mac and linux) file path From a734bb6a693ee006fb896438aedbdd225710e6c4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 3 Mar 2024 19:08:23 -0600 Subject: [PATCH 012/183] Increase AbstractNewDataSourceHandler update lock count 2x -> 4x Trying to reduce lock collisions. --- .../core/file/AbstractNewDataSourceHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 8978ae0af..c6ec0a08a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -75,9 +75,9 @@ public abstract class AbstractNewDataSourceHandler LOGGER.warn("Unable to create full data folder, file saving may fail."); } - // the lock array's length is double the number of CPU cores so the number of collisions + // the lock array's length is 4x the number of CPU cores so the number of collisions // should be relatively low without having too many extra locks - int lockCount = Runtime.getRuntime().availableProcessors() * 2; + int lockCount = Runtime.getRuntime().availableProcessors() * 4; this.updateLockArray = new ReentrantLock[lockCount]; for (int i = 0; i < lockCount; i++) { From 0f044531340fb348a45d2ada82d384895e0eec0a Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 3 Mar 2024 19:17:24 -0600 Subject: [PATCH 013/183] Make down sampling average values instead of grabbing the value closest to -inf --- .../fullData/sources/NewFullDataSource.java | 288 ++++++++++++++++-- .../transformers/LodDataBuilder.java | 10 + .../fullDatafile/NewFullDataFileHandler.java | 7 +- 3 files changed, 270 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index b6b9c2deb..69c8f17fb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -30,6 +30,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; @@ -41,10 +42,10 @@ import java.util.Arrays; import java.util.concurrent.locks.ReentrantLock; /** - * This data source contains every datapoint over its given {@link DhSectionPos}. - * + * This data source contains every datapoint over its given {@link DhSectionPos}.

+ * * TODO create a child object that extends AutoClosable - * that can be pooled so we don't have GC overhead + * that can be pooled to reduce GC overhead * * @see FullDataPointUtil * @see CompleteFullDataSource @@ -52,6 +53,8 @@ import java.util.concurrent.locks.ReentrantLock; public class NewFullDataSource implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ + private static final boolean RUN_UPDATE_DEV_VALIDATION = false; //ModInfo.IS_DEV_BUILD; /** measured in data columns */ public static final int WIDTH = 64; @@ -76,7 +79,10 @@ public class NewFullDataSource implements IDataSource */ public byte[] columnGenerationSteps; - /** stored x/z, y */ + /** + * stored x/z, y + * The y data should be sorted from bottom to top + */ public long[][] dataPoints; private boolean isEmpty; @@ -154,6 +160,15 @@ public class NewFullDataSource implements IDataSource public boolean update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } public boolean update(NewFullDataSource inputDataSource) { + // shouldn't happen, but James saw it happen once + if (inputDataSource.mapping.getMaxValidId() == 0) + { + LOGGER.warn("Invalid mapping given from input update data source at pos: ["+inputDataSource.pos+"], skipping update."); + return false; + } + + + byte thisDetailLevel = this.pos.getDetailLevel(); byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); @@ -263,38 +278,184 @@ public class NewFullDataSource implements IDataSource { for (int z = 0; z < WIDTH; z += 2) { - int inputIndex = relativePosToIndex(x, z); + long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); + byte inputGenStep = inputDataSource.columnGenerationSteps[0]; // TODO - long[] inputDataArray = inputDataSource.dataPoints[inputIndex]; - if (inputDataArray != null) + + int recipientX = (x / 2) + recipientOffsetX; + int recipientZ = (z / 2) + recipientOffsetZ; + int recipientIndex = relativePosToIndex(recipientX, recipientZ); + + long[] oldDataArray = this.dataPoints[recipientIndex]; + + this.columnGenerationSteps[recipientIndex] = inputGenStep; + this.dataPoints[recipientIndex] = mergedInputDataArray; + this.remapDataColumn(recipientIndex, remappedIds); + + // we only need to see if the data was changed in one column + if (!dataChanged) { - byte inputGenStep = inputDataSource.columnGenerationSteps[inputIndex]; - - // TODO downsample instad of grabbing the column nearest to (-inf, -inf) - int recipientX = (x / 2) + recipientOffsetX; - int recipientZ = (z / 2) + recipientOffsetZ; - int recipientIndex = relativePosToIndex(recipientX, recipientZ); - - long[] oldDataArray = this.dataPoints[recipientIndex]; - - this.columnGenerationSteps[recipientIndex] = inputGenStep; - this.dataPoints[recipientIndex] = inputDataArray; - this.remapDataColumn(recipientIndex, remappedIds); - - // we only need to see if the data was changed in one column - if (!dataChanged) - { - // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same - dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[recipientIndex]); - } - - this.isEmpty = false; + // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same + dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[recipientIndex]); } + + this.isEmpty = false; } } return dataChanged; } + + + private static long[] mergeInputTwoByTwoDataColumn(NewFullDataSource inputDataSource, int x, int z) + { + ArrayList newColumnList = new ArrayList<>(); + + int[] currentDatapointIndex = new int[] { 0, 0, 0, 0 }; + + int lastId = 0; + byte lastBlockLight = 0; + byte lastSkyLight = 0; + int height = 0; + int minY = 0; + + + for (int blockY = 0; blockY < RenderDataPointUtil.MAX_WORLD_Y_SIZE; blockY++, height++) + { + //if (x == 38 && z == 2 && blockY == 65+72) // nether portal + if (x == 32 && z == 0 && blockY == 138) // glowstone + { + int k = 0; // TODO remove, just for testing + } + + // if each column has reached the end of their data, nothing more needs to be done + if (currentDatapointIndex[0] == -1 + && currentDatapointIndex[1] == -1 + && currentDatapointIndex[2] == -1 + && currentDatapointIndex[3] == -1 + ) + { + break; + } + + + long[] datapointsForYSlice = new long[4]; + + + // scary double loop but, + // this will only ever loop 4 times, + // once for each of the 4 input columns + int colIndex = 0; + for (int inputX = x; inputX < x +2; inputX++) + { + for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) + { + long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; + if (inputDataArray == null || inputDataArray.length == 0) + { + currentDatapointIndex[colIndex] = -1; + continue; + } + + + int dataPointIndex = currentDatapointIndex[colIndex]; + if (dataPointIndex == -1) + { + // went over the end + continue; + } + long datapoint = inputDataArray[dataPointIndex]; + + int datapointMinY = FullDataPointUtil.getBottomY(datapoint); + int numbOfBlocksTall = FullDataPointUtil.getHeight(datapoint); + int datapointMaxY = (datapointMinY + numbOfBlocksTall); + + + // check if y position is inside this datapoint + if (blockY < datapointMinY) + { + // this y-slice is below this datapoint, nothing can be added + continue; + } + else if (blockY >= datapointMaxY) + { + // this y-slice is above this datapoint, + // try the next data point + + int newDatapointIndex = currentDatapointIndex[colIndex] + 1; + if (newDatapointIndex >= inputDataArray.length) + { + // went to far, no additional data present + newDatapointIndex = -1; + } + currentDatapointIndex[colIndex] = newDatapointIndex; + + + // try again with the next data point + inputZ--; + colIndex--; + continue; + } + + + + datapointsForYSlice[colIndex] = datapoint; + } + } + + + + int[] mergeIds = new int[4]; + int[] mergeBlockLights = new int[4]; + int[] mergeSkyLights = new int[4]; + for (int i = 0; i < 4; i++) + { + mergeIds[i] = FullDataPointUtil.getId(datapointsForYSlice[i]); + mergeBlockLights[i] = FullDataPointUtil.getBlockLight(datapointsForYSlice[i]); + mergeSkyLights[i] = FullDataPointUtil.getSkyLight(datapointsForYSlice[i]); + } + + + // determine the most common values for this slice + int id = determineMostValueInColumnSlice(mergeIds, inputDataSource.mapping); + byte blockLight = (byte) determineAverageValueInColumnSlice(mergeBlockLights); + byte skyLight = (byte) determineAverageValueInColumnSlice(mergeSkyLights); + + // if this slice is different then the last one, create a new one + if (id != lastId + // block and sky light might not be necessary + || blockLight != lastBlockLight + || skyLight != lastSkyLight) + { + if (height != 0) + { + newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + } + + lastId = id; + lastBlockLight = blockLight; + lastSkyLight = skyLight; + height = 0; + minY = blockY; + } + } + + // add the last slice if present + if (height != 0) + { + newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + } + + + + + long[]mergedInputDataArray = new long[newColumnList.size()]; + for (int i = 0; i < mergedInputDataArray.length; i++) + { + mergedInputDataArray[i] = newColumnList.get(i); + } + return mergedInputDataArray; + } /** * Only update the ID once it's been added to this data source. * Updating the incoming data source will cause issues if it is applied @@ -323,6 +484,72 @@ public class NewFullDataSource implements IDataSource return (newArrayHash != oldArrayHash); } } + /** @param mapping can be included to ignore air ID's, otherwise all 4 values are treated equally */ + private static int determineMostValueInColumnSlice(int[] sliceArray, @Nullable FullDataPointIdMap mapping) + { + if (RUN_UPDATE_DEV_VALIDATION) + { + LodUtil.assertTrue(sliceArray.length == 4, "Column Slice should only contain 4 values."); + } + + int value0 = sliceArray[0]; + int count0 = 0; + int value1 = sliceArray[1]; + int count1 = 0; + int value2 = sliceArray[2]; + int count2 = 0; + int value3 = sliceArray[3]; + int count3 = 0; + + // count the occurrences of each value + for (int i = 0; i < 4; i++) + { + int value = sliceArray[i]; + if (mapping != null && mapping.getBlockStateWrapper(value).isAir()) + { + // always overwrite air to prevent holes in hollow structures + continue; + } + + if (value == value0) + count0++; + else if (value == value1) + count1++; + else if (value == value2) + count2++; + else + count3++; + } + + // return the most common occurance + int maxCount = Math.max(count0, Math.max(count1, Math.max(count2, count3))); + if (maxCount == count0) + // if the max count is 1 then we'll just go with the first column + return value0; + else if (maxCount == count1) + return value1; + else if (maxCount == count2) + return value2; + else + return value3; + } + private static int determineAverageValueInColumnSlice(int[] sliceArray) + { + if (RUN_UPDATE_DEV_VALIDATION) + { + LodUtil.assertTrue(sliceArray.length == 4, "Column Slice should only contain 4 values."); + } + + + int value = 0; + for (int i = 0; i < 4; i++) + { + value += sliceArray[i]; + } + + value /= 4; + return value; + } @@ -377,10 +604,9 @@ public class NewFullDataSource implements IDataSource this.columnGenerationSteps[index] = worldGenStep.value; - // validate the incoming ID's - // shouldn't normally happen and can be disabled for release builds - if (ModInfo.IS_DEV_BUILD) + if (RUN_UPDATE_DEV_VALIDATION) { + // validate the incoming ID's int maxValidId = this.mapping.getMaxValidId(); for (int i = 0; i < longArray.length; i++) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 7b560e3d5..26d34d8bb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -188,6 +188,16 @@ public class LodDataBuilder } longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + // reverse the array so index 0 is the lowest, + // this is necessary for later logic + // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java + for(int i = 0; i < longs.size() / 2; i++) + { + long temp = longs.getLong(i); + longs.set(i, longs.getLong(longs.size() - i - 1)); + longs.set(longs.size() - i - 1, temp); + } + dataSource.setSingleColumn(longs.toLongArray(), chunkX + chunkOffsetX, chunkZ + chunkOffsetZ, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 1fe3796d3..59a3ccc3b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -53,15 +53,14 @@ public class NewFullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 20; + private static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; /** how many parent update tasks can be in the queue at once */ private static final int MAX_PARENT_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); /** indicates how long the update queue thread should wait between queuing ticks */ - private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 1_000; + private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 500; - - // TODO add a debug view + /** the list of queued positions that need to update their parents */ Set parentApplicationPositionSet = ConcurrentHashMap.newKeySet(); private final ThreadPoolExecutor updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Update Queue Processor"); private final AtomicBoolean updateQueueThreadRunningRef = new AtomicBoolean(false); From 0c5c4f3a74e9f309af30cc9f1b42c59b4cd49bac Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 4 Mar 2024 07:08:32 -0600 Subject: [PATCH 014/183] Fix multithreaded save calls to AbstractDhRepo --- .../core/sql/repo/AbstractDhRepo.java | 39 +++++++++++++++++-- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 2420a0478..1a5dfd945 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.DatabaseUpdater; import com.seibel.distanthorizons.core.sql.DbConnectionClosedException; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; @@ -29,6 +30,7 @@ import org.jetbrains.annotations.Nullable; import java.sql.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; /** * Handles interfacing with SQL databases. @@ -51,6 +53,11 @@ public abstract class AbstractDhRepo> public final Class dtoClass; + protected final ReentrantLock[] saveLockArray; + /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ + protected ReentrantLock getSaveLockForKey(TKey key) { return this.saveLockArray[Math.abs(key.hashCode()) % this.saveLockArray.length]; } + + //=============// @@ -65,6 +72,16 @@ public abstract class AbstractDhRepo> this.dtoClass = dtoClass; + // the lock array's length is 2x the number of CPU cores so the number of collisions + // should be relatively low without having too many extra locks + int lockCount = Runtime.getRuntime().availableProcessors() * 2; + this.saveLockArray = new ReentrantLock[lockCount]; + for (int i = 0; i < lockCount; i++) + { + this.saveLockArray[i] = new ReentrantLock(); + } + + try { // needed by Forge to load the Java database connection @@ -126,13 +143,27 @@ public abstract class AbstractDhRepo> public void save(TDTO dto) { - if (this.getByKey(dto.getKey()) != null) + // a lock is necessary to prevent concurrent modification between + // existsWithKey and insert/update, + // otherwise another thread might cause the insert/update to fail. + ReentrantLock saveLock = this.getSaveLockForKey(dto.getKey()); + + try { - this.update(dto); + saveLock.lock(); + + if (this.existsWithKey(dto.getKey())) + { + this.update(dto); + } + else + { + this.insert(dto); + } } - else + finally { - this.insert(dto); + saveLock.unlock(); } } private void insert(TDTO dto) From c866fbfbfd56e89cacfd20f41d25bce39888fc4b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 4 Mar 2024 07:10:20 -0600 Subject: [PATCH 015/183] Add update propagator thread pool --- .../distanthorizons/core/config/Config.java | 27 +++++++++++++++- .../ThreadPresetConfigEventHandler.java | 25 +++++++++++++++ .../file/AbstractNewDataSourceHandler.java | 31 ++++++++++++------- .../fullDatafile/NewFullDataFileHandler.java | 9 +++--- .../renderfile/RenderSourceFileHandler.java | 13 +++++--- .../core/util/threading/ThreadPoolUtil.java | 10 +++++- .../assets/distanthorizons/lang/en_us.json | 7 +++++ 7 files changed, 101 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index f81eb8ce9..a3a39aefb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -907,7 +907,7 @@ public class Config ThreadPresetConfigEventHandler.getFileHandlerDefaultThreadCount(), Runtime.getRuntime().availableProcessors()) .comment("" - + "How many threads should be used when reading in LOD data from disk? \n" + + "How many threads should be used when reading/writing LOD data to/from disk? \n" + "\n" + "Increasing this number will cause LODs to load in faster, \n" + "but may cause lag when loading a new world or when \n" @@ -920,6 +920,31 @@ public class Config .comment(THREAD_RUN_TIME_RATIO_NOTE) .build(); + public static final ConfigEntry numberOfUpdatePropagatorThreads = new ConfigEntry.Builder() + .setMinDefaultMax(1, + ThreadPresetConfigEventHandler.getUpdatePropagatorDefaultThreadCount(), + Runtime.getRuntime().availableProcessors()) + .comment("" + + "How many threads should be used when applying LOD updates? \n" + + "An LOD update is the operation of down-sampling a high detail LOD \n" + + "into a lower detail one.\n" + + "\n" + + "This config can have a much higher number of threads \n" + + "assigned and much lower run time ratio vs other thread pools \n" + + "because the amount of time any particular thread may run is relatively low.\n" + + "\n" + + "This is because LOD updating only only partially thread safe, \n" + + "so between 40% and 60% of the time a given thread may end up \n" + + "waiting on another thread to finish updating the same LOD it also wants\n" + + "to work on.\n" + + "\n" + + THREAD_NOTE) + .build(); + public static final ConfigEntry runTimeRatioForUpdatePropagatorThreads = new ConfigEntry.Builder() + .setMinDefaultMax(0.01, ThreadPresetConfigEventHandler.getUpdatePropagatorDefaultRunTimeRatio(), 1.0) + .comment(THREAD_RUN_TIME_RATIO_NOTE) + .build(); + public static final ConfigEntry numberOfLodBuilderThreads = new ConfigEntry.Builder() .setMinDefaultMax(1, ThreadPresetConfigEventHandler.getLodBuilderDefaultThreadCount(), diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 67b7e2010..fee7bb252 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -85,6 +85,28 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan }}); + public static int getUpdatePropagatorDefaultThreadCount() { return getThreadCountByPercent(0.5); } + private final ConfigEntryWithPresetOptions UpdatePropagatorThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 1); + this.put(EThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultThreadCount()); + this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.75)); + this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.75)); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); + }}); + public static double getUpdatePropagatorDefaultRunTimeRatio() { return 0.25; } + private final ConfigEntryWithPresetOptions UpdatePropagatorRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads, + new HashMap() + {{ + this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultRunTimeRatio()); + this.put(EThreadPreset.BALANCED, 0.5); + this.put(EThreadPreset.AGGRESSIVE, 1.0); + this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + }}); + + public static int getLodBuilderDefaultThreadCount() { return getThreadCountByPercent(0.1); } private final ConfigEntryWithPresetOptions lodBuilderThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads, new HashMap() @@ -122,6 +144,9 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.configList.add(this.fileHandlerThreadCount); this.configList.add(this.fileHandlerRunTime); + this.configList.add(this.UpdatePropagatorThreadCount); + this.configList.add(this.UpdatePropagatorRunTime); + this.configList.add(this.lodBuilderThreadCount); this.configList.add(this.lodBuilderRunTime); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index c6ec0a08a..99eb57815 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -168,7 +168,7 @@ public abstract class AbstractNewDataSourceHandler @Override public CompletableFuture updateDataSourceAsync(NewFullDataSource inputDataSource) { - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); if (executor == null || executor.isTerminated()) { return CompletableFuture.completedFuture(null); @@ -214,18 +214,27 @@ public abstract class AbstractNewDataSourceHandler if (dataModified) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(dataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) { - if (listener != null) - { - listener.OnDataSourceUpdated(dataSource); - } + return; } + + executor.execute(() -> + { + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(dataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + { + if (listener != null) + { + listener.OnDataSourceUpdated(dataSource); + } + } + }); } } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 59a3ccc3b..3233b55ae 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -58,7 +58,7 @@ public class NewFullDataFileHandler private static final int MAX_PARENT_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); /** indicates how long the update queue thread should wait between queuing ticks */ - private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 500; + private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 1_000; /** the list of queued positions that need to update their parents */ Set parentApplicationPositionSet = ConcurrentHashMap.newKeySet(); @@ -210,7 +210,7 @@ public class NewFullDataFileHandler if (updatePosList.size() != 0) { // stop if the file handler has been shut down - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); if (executor == null || executor.isTerminated()) { this.updateQueueThreadRunningRef.set(false); @@ -222,6 +222,9 @@ public class NewFullDataFileHandler int queueCount = 0; for (DhSectionPos pos : updatePosList) { + // James thought batching together updates + // based on the parent they were going to update would reduce update locks, + // but after testing it didn't, so we're just queing each section individually if (this.parentApplicationPositionSet.add(pos)) { queueCount++; @@ -230,8 +233,6 @@ public class NewFullDataFileHandler NewFullDataSource inputData = this.get(pos); // update the parent position with this new data this.updateDataSourceAtPos(pos.getParentPos(), inputData, true); - - // TODO add comparable interface to make this low priority }); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 70ce500bf..1724a6a5b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -126,13 +126,18 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler lines = new ArrayList<>(); lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]"); - lines.add(" Thread pool tasks: " + queueSize + " (completed: " + completedTaskSize + ")"); + lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); + lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size()); return lines.toArray(new String[0]); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java index 6924ab7b2..1d78ea34e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java @@ -46,11 +46,17 @@ public class ThreadPoolUtil @Nullable public static ThreadPoolExecutor getFileHandlerExecutor() { return fileHandlerThreadPool.executor; } + public static final DhThreadFactory UPDATE_PROPAGATOR_THREAD_FACTORY = new DhThreadFactory("LOD Update Propagator", Thread.MIN_PRIORITY); + private static RateLimitedThreadPoolExecutor updatePropagatorThreadPool; + @Nullable + public static ThreadPoolExecutor getUpdatePropagatorExecutor() { return updatePropagatorThreadPool; } + public static final DhThreadFactory WORLD_GEN_THREAD_FACTORY = new DhThreadFactory("World Gen", Thread.MIN_PRIORITY); private static ConfigThreadPool worldGenThreadPool; @Nullable public static ThreadPoolExecutor getWorldGenExecutor() { return worldGenThreadPool.executor; } + public static final String BUFFER_UPLOADER_THREAD_NAME = "Buffer Uploader"; private static ThreadPoolExecutor bufferUploaderThreadPool; @Nullable public static ThreadPoolExecutor getBufferUploaderExecutor() { return bufferUploaderThreadPool; } @@ -99,8 +105,9 @@ public class ThreadPoolUtil // standalone threads // fileHandlerThreadPool = new ConfigThreadPool(FILE_HANDLER_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, null); + updatePropagatorThreadPool = new RateLimitedThreadPoolExecutor(8, 1.0, UPDATE_PROPAGATOR_THREAD_FACTORY); worldGenThreadPool = new ConfigThreadPool(WORLD_GEN_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, null); - bufferUploaderThreadPool = ThreadUtil.makeSingleThreadPool("Buffer Uploader"); + bufferUploaderThreadPool = ThreadUtil.makeSingleThreadPool(BUFFER_UPLOADER_THREAD_NAME); @@ -138,6 +145,7 @@ public class ThreadPoolUtil { // standalone threads fileHandlerThreadPool.shutdownExecutorService(); + updatePropagatorThreadPool.shutdown(); worldGenThreadPool.shutdownExecutorService(); bufferUploaderThreadPool.shutdown(); diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index b6a02475e..f581ea4aa 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -395,6 +395,13 @@ "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForFileHandlerThreads": "Runtime % for file handler threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfUpdatePropagatorThreads": + "NO. of update propagator threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfUpdatePropagatorThreads.@tooltip": + "How many threads should be used when applying LOD updates? \nAn LOD update is the operation of down-sampling a high detail LOD \ninto a lower detail one. \n\nThis config can have a much higher number of threads \nassigned and much lower run time ratio vs other thread pools \nbecause the amount of time any particular thread may run is relatively low.", + "distanthorizons.config.client.advanced.multiThreading.runTimeRatioForUpdatePropagatorThreads": + "Runtime % for update propagator threads", + "distanthorizons.config.client.advanced.multiThreading.numberOfLodBuilderThreads": "NO. of LOD builder threads", "distanthorizons.config.client.advanced.multiThreading.numberOfLodBuilderThreads.@tooltip": From 97554ccae6bd51d37dc4b92d271c75506dd5bd9b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 4 Mar 2024 07:13:43 -0600 Subject: [PATCH 016/183] Increase Vertical Quality settings to hopefully reduce issues with floating islands --- .../api/enums/config/EVerticalQuality.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java index c20b86b70..47f241f1c 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java @@ -35,10 +35,10 @@ import com.seibel.distanthorizons.coreapi.util.MathUtil; public enum EVerticalQuality { HEIGHT_MAP(new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}), - LOW(new int[]{4, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1}), - MEDIUM(new int[]{6, 4, 3, 2, 2, 1, 1, 1, 1, 1, 1}), - HIGH(new int[]{8, 6, 4, 2, 2, 2, 2, 1, 1, 1, 1}), - EXTREME(new int[]{16, 8, 4, 2, 2, 2, 2, 1, 1, 1, 1}); + LOW(new int[]{4, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1}), + MEDIUM(new int[]{6, 4, 3, 3, 3, 3, 3, 2, 2, 2, 1}), + HIGH(new int[]{16, 8, 4, 3, 3, 3, 3, 3, 3, 3, 1}), + EXTREME(new int[]{32, 16, 8, 4, 4, 3, 3, 3, 3, 3, 1}); /** represents how many LODs can be rendered in a single vertical slice */ public final int[] maxVerticalData; From 8f6ff8c943c2c23957dbb516f3c3d23a2584822f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 4 Mar 2024 20:56:48 -0600 Subject: [PATCH 017/183] Fix data sources not saving correctly --- .../file/AbstractNewDataSourceHandler.java | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 99eb57815..db3c2c783 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -214,27 +214,18 @@ public abstract class AbstractNewDataSourceHandler if (dataModified) { - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return; - } + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(dataSource); + this.repo.save(dto); - executor.execute(() -> + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(dataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + if (listener != null) { - if (listener != null) - { - listener.OnDataSourceUpdated(dataSource); - } + listener.OnDataSourceUpdated(dataSource); } - }); + } } } catch (Exception e) From eea2155e20dbfc0525199a16c20e6c6baa7e04a3 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 4 Mar 2024 21:25:02 -0600 Subject: [PATCH 018/183] Temp fix for 0020 auto update script --- .../0020-sqlite-createGeneratedFullDataSourceTables.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql index e78cff45c..d0d833358 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -1,4 +1,6 @@ +select * from DhRenderData; -- here to prevent crashing when running the first batch + --batch-- CREATE TABLE FullData ( From 244d960ec02ec46671c293e1e346a6355e1bf192 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 4 Mar 2024 21:34:00 -0600 Subject: [PATCH 019/183] minor render data bugfix --- .../dataObjects/render/ColumnRenderSource.java | 18 ++++++++---------- .../render/columnViews/ColumnArrayView.java | 12 +++++++----- .../FullDataToRenderDataTransformer.java | 2 +- .../core/level/ClientLevelModule.java | 16 ++++++++++++---- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 42870ff5d..39024ff91 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -71,7 +71,7 @@ public class ColumnRenderSource implements IDataSource public static final int NO_DATA_FLAG_BYTE = 0x00000001; - + /** will be zero if an empty data source was created */ public int verticalDataCount; public final DhSectionPos sectionPos; public final int yOffset; @@ -290,12 +290,12 @@ public class ColumnRenderSource implements IDataSource } @Override - public boolean update(NewFullDataSource inputDataSource, IDhClientLevel level) + public boolean update(NewFullDataSource inputFullDataSource, IDhClientLevel level) { - final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputDataSource.getSectionPos() + "]. Error:"; + final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputFullDataSource.getSectionPos() + "]. Error:"; boolean dataChanged = false; - if (inputDataSource.getSectionPos().getDetailLevel() == this.sectionPos.getDetailLevel()) + if (inputFullDataSource.getSectionPos().getDetailLevel() == this.sectionPos.getDetailLevel()) { try { @@ -307,8 +307,8 @@ public class ColumnRenderSource implements IDataSource - DhBlockPos2D centerBlockPos = inputDataSource.getSectionPos().getCenterBlockPos(); - int halfBlockWidth = inputDataSource.getSectionPos().getBlockWidth() / 2; + DhBlockPos2D centerBlockPos = inputFullDataSource.getSectionPos().getCenterBlockPos(); + int halfBlockWidth = inputFullDataSource.getSectionPos().getBlockWidth() / 2; DhBlockPos2D minBlockPos = new DhBlockPos2D(centerBlockPos.x - halfBlockWidth, centerBlockPos.z - halfBlockWidth); for (int x = 0; x < NewFullDataSource.WIDTH; x++) @@ -318,8 +318,8 @@ public class ColumnRenderSource implements IDataSource ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = inputDataSource.get(x, z); - EDhApiWorldGenerationStep worldGenStep = inputDataSource.getWorldGenStepAtRelativePos(x, z); + SingleColumnFullDataAccessor fullArrayView = inputFullDataSource.get(x, z); + EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z); if (fullArrayView != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { FullDataToRenderDataTransformer.convertColumnData(level, @@ -339,8 +339,6 @@ public class ColumnRenderSource implements IDataSource } } - - if (dataChanged) { this.localVersion.incrementAndGet(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java index dad170788..17b1e9be2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java @@ -29,6 +29,7 @@ public final class ColumnArrayView implements IColumnDataView public final long[] data; public final int size; public final int offset; // offset in longs + /** can be 0 if this column was created for an empty data source */ public final int vertSize; // vertical size in longs @@ -55,7 +56,7 @@ public final class ColumnArrayView implements IColumnDataView public int verticalSize() { return vertSize; } @Override - public int dataCount() { return size / vertSize; } + public int dataCount() { return (this.vertSize != 0) ? (this.size / this.vertSize) : 0; } @Override public ColumnArrayView subView(int dataIndexStart, int dataCount) @@ -130,17 +131,18 @@ public final class ColumnArrayView implements IColumnDataView public void changeVerticalSizeFrom(IColumnDataView source) { - if (dataCount() != source.dataCount()) + if (this.dataCount() != source.dataCount()) { throw new IllegalArgumentException("Cannot copy and resize to views with different dataCounts"); } - if (vertSize >= source.verticalSize()) + + if (this.vertSize >= source.verticalSize()) { - copyFrom(source); + this.copyFrom(source); } else { - for (int i = 0; i < dataCount(); i++) + for (int i = 0; i < this.dataCount(); i++) { RenderDataPointUtil.mergeMultiData(source.subView(i, 1), subView(i, 1)); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 7ac3a5e9d..b9293258c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -63,7 +63,7 @@ public class FullDataToRenderDataTransformer { return null; } - else if (MC.getWrappedClientLevel() == null) + else if (level == null) { // if the client is no longer loaded in the world, render sources cannot be created return null; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 23a358d7e..c1f968156 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -49,8 +50,12 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); + private final IDhClientLevel parentClientLevel; + + public final NewFullDataFileHandler fullDataSourceProvider; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); + public final F3Screen.NestedMessage f3Message; @@ -64,8 +69,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle this.parentClientLevel = parentClientLevel; this.f3Message = new F3Screen.NestedMessage(this::f3Log); - NewFullDataFileHandler fileHandler = this.parentClientLevel.getFullDataProvider(); - fileHandler.dateSourceUpdateListeners.add(this); + this.fullDataSourceProvider = this.parentClientLevel.getFullDataProvider(); + this.fullDataSourceProvider.dateSourceUpdateListeners.add(this); } @@ -137,7 +142,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle /** @return if the {@link ClientRenderState} was successfully swapped */ public boolean startRenderer(IClientLevelWrapper clientLevelWrapper) { - ClientRenderState ClientRenderState = new ClientRenderState(parentClientLevel, clientLevelWrapper, parentClientLevel.getFullDataProvider(), parentClientLevel.getSaveStructure()); + ClientRenderState ClientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState)) { LOGGER.warn("Failed to start renderer due to concurrency"); @@ -208,6 +213,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle @Override public void OnDataSourceUpdated(NewFullDataSource updatedFullDataSource) { + // if rendering also update the render sources ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { @@ -240,6 +246,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle } } + this.fullDataSourceProvider.dateSourceUpdateListeners.remove(this); + this.f3Message.close(); } @@ -252,7 +260,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle /** Returns what should be displayed in Minecraft's F3 debug menu */ protected String[] f3Log() { - String dimName = parentClientLevel.getClientLevelWrapper().getDimensionType().getDimensionName(); + String dimName = this.parentClientLevel.getClientLevelWrapper().getDimensionType().getDimensionName(); ClientRenderState renderState = this.ClientRenderStateRef.get(); if (renderState == null) { From e6c985a189a5b995197def41c4c9077b896ff278 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 9 Mar 2024 09:06:39 -0600 Subject: [PATCH 020/183] Add hash code and equals to NewFullDataSource --- .../fullData/sources/NewFullDataSource.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 69c8f17fb..317a30ebb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -33,7 +33,6 @@ import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; -import com.seibel.distanthorizons.coreapi.ModInfo; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -73,6 +72,8 @@ public class NewFullDataSource implements IDataSource public int levelMinY; + private int cachedHashCode = 0; + /** * stores how far each column has been generated should start with {@link EDhApiWorldGenerationStep#EMPTY} * @see EDhApiWorldGenerationStep @@ -200,6 +201,12 @@ public class NewFullDataSource implements IDataSource this.applyToParent = true; } + if (dataChanged) + { + // update the hash code + this.generateHashCode(); + } + return dataChanged; } public boolean updateFromSameDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) @@ -623,7 +630,45 @@ public class NewFullDataSource implements IDataSource @Override public String toString() { return this.pos.toString(); } + @Override + public int hashCode() + { + if (this.cachedHashCode == 0) + { + this.generateHashCode(); + } + return this.cachedHashCode; + } + private void generateHashCode() + { + int result = this.pos.hashCode(); + result = 31 * result + Arrays.deepHashCode(this.dataPoints); + result = 17 * result + Arrays.hashCode(this.columnGenerationSteps); + + this.cachedHashCode = result; + } + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof NewFullDataSource)) + { + return false; + } + NewFullDataSource other = (NewFullDataSource) obj; + + if (!other.pos.equals(this.pos)) + { + return false; + } + else + { + // the positions are the same, use the hash as a quick method + // to determine if the data inside is the same. + // Note: this isn't perfect, but should work well enough for our use case. + return other.hashCode() == this.hashCode(); + } + } //============// // deprecated // From 599340c4e88362af29f4046d9afe448cfb868781 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 9 Mar 2024 17:05:54 -0600 Subject: [PATCH 021/183] Reduce data source update locking and add some data update batching --- .../methods/data/DhApiTerrainDataRepo.java | 2 +- .../file/AbstractNewDataSourceHandler.java | 107 +++++--- .../DelayedFullDataSourceSaveCache.java | 121 +++++++++ .../fullDatafile/IFullDataSourceProvider.java | 4 +- .../fullDatafile/NewFullDataFileHandler.java | 234 ++++++++++-------- .../NewGeneratedFullDataFileHandler.java | 27 +- .../core/level/AbstractDhLevel.java | 52 +++- .../core/level/ClientLevelModule.java | 3 +- .../core/level/DhClientLevel.java | 3 +- .../core/level/DhClientServerLevel.java | 3 +- .../core/level/DhServerLevel.java | 9 +- .../distanthorizons/core/level/IDhLevel.java | 10 +- .../threading/PositionalLockProvider.java | 187 ++++++++++++++ .../core/util/threading/ThreadPoolUtil.java | 8 +- 14 files changed, 603 insertions(+), 167 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/threading/PositionalLockProvider.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index c67342fac..056f263de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -461,7 +461,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // this will throw a cast exception if the chunk object array isn't correct IChunkWrapper chunk = SingletonInjector.INSTANCE.get(IWrapperFactory.class).createChunkWrapper(chunkObjectArray); - dhLevel.updateChunkAsync(chunk); + SharedApi.INSTANCE.applyChunkUpdate(chunk, dhLevel.getLevelWrapper(), true); return DhApiResult.createSuccess(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index db3c2c783..ed2bc7746 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -8,18 +8,16 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.TimerUtil; +import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Timer; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; public abstract class AbstractNewDataSourceHandler @@ -29,9 +27,6 @@ public abstract class AbstractNewDataSourceHandler implements ISourceProvider { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); - /** How long a data source must remain un-modified before being written to disk. */ - private static final int SAVE_DELAY_IN_MS = 4_000; /** * The highest numerical detail level possible. @@ -48,7 +43,15 @@ public abstract class AbstractNewDataSourceHandler public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - protected final ReentrantLock[] updateLockArray; + protected final PositionalLockProvider updateLockProvider = new PositionalLockProvider(); + /** + * generally just used for debugging, + * keeps track of which positions are currently locked. + */ + public final Set lockedPosSet = ConcurrentHashMap.newKeySet(); + public final ConcurrentHashMap queuedUpdateCountsByPos = new ConcurrentHashMap<>(); + + protected final ReentrantLock closeLock = new ReentrantLock(); protected volatile boolean isShutdown = false; @@ -75,15 +78,6 @@ public abstract class AbstractNewDataSourceHandler LOGGER.warn("Unable to create full data folder, file saving may fail."); } - // the lock array's length is 4x the number of CPU cores so the number of collisions - // should be relatively low without having too many extra locks - int lockCount = Runtime.getRuntime().availableProcessors() * 4; - this.updateLockArray = new ReentrantLock[lockCount]; - for (int i = 0; i < lockCount; i++) - { - this.updateLockArray[i] = new ReentrantLock(); - } - this.repo = this.createRepo(); } @@ -178,44 +172,61 @@ public abstract class AbstractNewDataSourceHandler try { // run file handling on a separate thread + this.markUpdateStart(inputDataSource.getSectionPos()); return CompletableFuture.runAsync(() -> { - this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource, true); + try + { + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource, true); + } + catch (Exception e) + { + LOGGER.error("Unexpected error in async data source update, error: "+e.getMessage(), e); + } + finally + { + this.markUpdateEnd(inputDataSource.getSectionPos()); + } }, executor); } catch (RejectedExecutionException ignore) { // can happen if the executor was shutdown while this task was queued + this.markUpdateEnd(inputDataSource.getSectionPos()); return CompletableFuture.completedFuture(null); } } - /** - * @param pos the position to update - * @param lockOnPosition Can be disabled by inheriting children to allow for their own locking logic. - * This is important if the child has its own position specific logic that shouldn't be done concurrently. - */ - protected void updateDataSourceAtPos(DhSectionPos pos, NewFullDataSource inputData, boolean lockOnPosition) + + /** @param updatePos the position to update */ + protected void updateDataSourceAtPos(DhSectionPos updatePos, NewFullDataSource inputData, boolean lockOnUpdatePos) { + boolean methodLocked = false; // a lock is necessary to prevent two threads from writing to the same position at once, // if that happens only the second update will apply and the LOD will end up with hole(s) - ReentrantLock updateLock = this.getUpdateLockForPos(pos); + ReentrantLock updateLock = this.updateLockProvider.getLock(updatePos); try { - if (lockOnPosition) + if (lockOnUpdatePos) { + methodLocked = true; updateLock.lock(); + this.lockedPosSet.add(updatePos); + } + else + { + methodLocked = false; } // get or create the data source - TDataSource dataSource = this.get(pos); - boolean dataModified = dataSource.update(inputData, this.level); + TDataSource recipientDataSource = this.get(updatePos); + boolean dataModified = recipientDataSource.update(inputData, this.level); if (dataModified) { // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(dataSource); + TDTO dto = this.createDtoFromDataSource(recipientDataSource); this.repo.save(dto); @@ -223,20 +234,21 @@ public abstract class AbstractNewDataSourceHandler { if (listener != null) { - listener.OnDataSourceUpdated(dataSource); + listener.OnDataSourceUpdated(recipientDataSource); } } } } catch (Exception e) { - LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); + LOGGER.error("Error updating pos ["+updatePos+"], error: "+e.getMessage(), e); } finally { - if (lockOnPosition) + if (methodLocked) { updateLock.unlock(); + this.lockedPosSet.remove(updatePos); } } } @@ -247,8 +259,31 @@ public abstract class AbstractNewDataSourceHandler // helper methods // //================// - /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ - protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } + /** used for debugging to track which positions are queued for updating */ + private void markUpdateStart(DhSectionPos dataSourcePos) + { + this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) -> + { + if (atomicCount == null) + { + atomicCount = new AtomicInteger(0); + } + atomicCount.incrementAndGet(); + return atomicCount; + }); + } + /** used for debugging to track which positions are queued for updating */ + private void markUpdateEnd(DhSectionPos dataSourcePos) + { + this.queuedUpdateCountsByPos.compute(dataSourcePos, (pos, atomicCount) -> + { + if (atomicCount != null && atomicCount.decrementAndGet() <= 0) + { + atomicCount = null; + } + return atomicCount; + }); + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java new file mode 100644 index 000000000..733af7001 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java @@ -0,0 +1,121 @@ +package com.seibel.distanthorizons.core.file.fullDatafile; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.TimerUtil; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; + +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Used to batch together multiple data source updates that all + * affect the same position. + */ +public class DelayedFullDataSourceSaveCache +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + private static final Timer DELAY_UPDATE_TIMER = TimerUtil.CreateTimer("Delayed Full Datasource Save Timer"); + + + public final ConcurrentHashMap dataSourceByPosition = new ConcurrentHashMap<>(); + private final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); + + private final ISaveDataSourceFunc onSaveTimeoutFunc; + private final int saveDelayInMs; + + + + //=============// + // constructor // + //=============// + + public DelayedFullDataSourceSaveCache(@NotNull ISaveDataSourceFunc onSaveTimeoutFunc, int saveDelayInMs) + { + this.onSaveTimeoutFunc = onSaveTimeoutFunc; + this.saveDelayInMs = saveDelayInMs; + } + + + + //==============// + // update queue // + //==============// + + public void queueDataSourceForUpdateAndSave(NewFullDataSource inputDataSource) + { + DhSectionPos dataSourcePos = inputDataSource.getSectionPos(); + this.dataSourceByPosition.compute(dataSourcePos, (inputPos, temporaryDataSource) -> + { + if (temporaryDataSource == null) + { + temporaryDataSource = NewFullDataSource.createEmpty(inputPos); + } + temporaryDataSource.update(inputDataSource); + + + TimerTask timerTask = new TimerTask() + { + @Override + public void run() + { + DelayedFullDataSourceSaveCache.this.saveTimerTasksBySectionPos.remove(dataSourcePos); + + try + { + NewFullDataSource dataSourceToSave = DelayedFullDataSourceSaveCache.this.dataSourceByPosition.remove(dataSourcePos); + if (dataSourceToSave != null) + { + DelayedFullDataSourceSaveCache.this.onSaveTimeoutFunc.save(dataSourceToSave); + } + } + catch (Exception e) // this can throw errors (not exceptions) when installed in Iris' dev environment for some reason due to an issue with LZ4's compression library + { + LOGGER.error("Failed to save updated data for section ["+dataSourcePos+"], error: ["+e.getMessage()+"]", e); + } + } + }; + try + { + DELAY_UPDATE_TIMER.schedule(timerTask, this.saveDelayInMs); + } + catch (IllegalStateException ignore) + { + // James isn't sure why this is possible since this logic is inside a lock, + // maybe the timer is just async enough that there can be problems? + LOGGER.warn("Attempted to queue an already canceled task. Pos: ["+dataSourcePos+"], task already queued for pos: ["+this.saveTimerTasksBySectionPos.containsKey(dataSourcePos)+"]"); + } + + + // cancel the old save timer if present + // (this is equivalent to restarting the timer) + TimerTask oldTask = this.saveTimerTasksBySectionPos.put(dataSourcePos, timerTask); + if (oldTask != null) + { + oldTask.cancel(); + } + + return temporaryDataSource; + }); + } + + public int getUnsavedCount() { return this.dataSourceByPosition.size(); } + + + + //================// + // helper classes // + //================// + + @FunctionalInterface + public interface ISaveDataSourceFunc + { + /** called after the timeout expires */ + void save(NewFullDataSource inputDataSource); + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index 1c6e40ccd..889462ca6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -45,7 +45,7 @@ public interface IFullDataSourceProvider extends ISourceProvider updateDataSourceAsync(NewFullDataSource chunkData); - @Deprecated - int getUnsavedDataSourceCount(); + /** @return -1 if this provider never has unsaved data sources */ + default int getUnsavedDataSourceCount() { return -1; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 3233b55ae..38d075e89 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -40,11 +40,9 @@ import java.awt.*; import java.io.File; import java.io.IOException; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; public class NewFullDataFileHandler @@ -55,15 +53,20 @@ public class NewFullDataFileHandler private static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; /** how many parent update tasks can be in the queue at once */ - private static final int MAX_PARENT_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); + private static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); /** indicates how long the update queue thread should wait between queuing ticks */ - private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 1_000; + private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250; - /** the list of queued positions that need to update their parents */ - Set parentApplicationPositionSet = ConcurrentHashMap.newKeySet(); - private final ThreadPoolExecutor updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Update Queue Processor"); - private final AtomicBoolean updateQueueThreadRunningRef = new AtomicBoolean(false); + + public final Set parentUpdatingPosSet = ConcurrentHashMap.newKeySet(); + + // TODO only run thread if modifications happened recently + /** + * This isn't in {@link AbstractNewDataSourceHandler} since we don't need parent updating logic + * for render data, only full data. + */ + private final ThreadPoolExecutor updateQueueProcessor; @@ -76,7 +79,11 @@ public class NewFullDataFileHandler { super(level, saveStructure, saveDirOverride); - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus); + + String dimensionName = level.getLevelWrapper().getDimensionType().getDimensionName(); + this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Parent Update Queue ["+dimensionName+"]"); + this.updateQueueProcessor.execute(() -> this.runUpdateQueue()); } @@ -127,10 +134,6 @@ public class NewFullDataFileHandler @Override protected NewFullDataSource makeEmptyDataSource(DhSectionPos pos) { return NewFullDataSource.createEmpty(pos); } - @Deprecated - @Override - public int getUnsavedDataSourceCount() { return 0; } - //================// @@ -145,115 +148,126 @@ public class NewFullDataFileHandler // that aren't already in the database } - @Override - protected void updateDataSourceAtPos(DhSectionPos pos, NewFullDataSource inputData, boolean lockOnPosition) + + private void runUpdateQueue() { - ReentrantLock updateLock = this.getUpdateLockForPos(pos); - - try - { - if (lockOnPosition) - { - updateLock.lock(); - } - - super.updateDataSourceAtPos(pos, inputData, false); - this.tryQueueParentUpdates(); - - this.parentApplicationPositionSet.remove(inputData.getSectionPos()); - if (pos.getDetailLevel() != inputData.getSectionPos().getDetailLevel()) - { - // mark that the update has completed - ((NewFullDataSourceRepo) this.repo).setApplyToParent(inputData.getSectionPos(), false); // TODO remove casting - } - } - catch (Exception e) - { - LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); - } - finally - { - if (lockOnPosition) - { - updateLock.unlock(); - } - } - } - /** Queues some of the parent updates listed in the database. */ - private void tryQueueParentUpdates() - { - // the update thread is already running, - // we don't need multiple running - if (this.updateQueueThreadRunningRef.getAndSet(true)) - { - return; - } - - - this.updateQueueProcessor.execute(() -> + while (!Thread.interrupted()) { try { - ArrayList updatePosList = null; + Thread.sleep(UPDATE_QUEUE_THREAD_DELAY_IN_MS); - while ( // continue queuing update positions as long as there are positions to queue - (updatePosList == null || updatePosList.size() != 0) - // only add more items to the queue if half or more of the previous tasks have been completed - && this.parentApplicationPositionSet.size() < (MAX_PARENT_UPDATE_TASK_COUNT / 2)) + ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); + if (executor == null || executor.isTerminated()) + { + continue; + } + + + + // only add more items to the queue if half or more of the previous tasks have been completed + if (executor.getQueue().size() < (MAX_UPDATE_TASK_COUNT) + && this.parentUpdatingPosSet.size() < (MAX_UPDATE_TASK_COUNT)) { - // prevent hitting the database more often than is necessary - Thread.sleep(UPDATE_QUEUE_THREAD_DELAY_IN_MS); - - // get the positions that need to be applied to their parents - updatePosList = ((NewFullDataSourceRepo) this.repo).getPositionsToUpdate(MAX_PARENT_UPDATE_TASK_COUNT); - if (updatePosList.size() != 0) + ArrayList parentUpdatePosList = ((NewFullDataSourceRepo) this.repo).getPositionsToUpdate(MAX_UPDATE_TASK_COUNT); + + HashMap> updatePosByParentPos = new HashMap<>(); + for (DhSectionPos pos : parentUpdatePosList) { - // stop if the file handler has been shut down - ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); - if (executor == null || executor.isTerminated()) + updatePosByParentPos.compute(pos.getParentPos(), (parentPos, updatePosSet) -> { - this.updateQueueThreadRunningRef.set(false); - return; - } - - - // queue each update - int queueCount = 0; - for (DhSectionPos pos : updatePosList) - { - // James thought batching together updates - // based on the parent they were going to update would reduce update locks, - // but after testing it didn't, so we're just queing each section individually - if (this.parentApplicationPositionSet.add(pos)) + if (updatePosSet == null) { - queueCount++; - executor.execute(() -> - { - NewFullDataSource inputData = this.get(pos); - // update the parent position with this new data - this.updateDataSourceAtPos(pos.getParentPos(), inputData, true); - }); + updatePosSet = new HashSet<>(); } + updatePosSet.add(pos); + return updatePosSet; + }); + } + + + + // queue each update + for (DhSectionPos parentUpdatePos : updatePosByParentPos.keySet()) + { + // stop if there are already a bunch of updates queued + if (this.parentUpdatingPosSet.size() > MAX_UPDATE_TASK_COUNT + && this.parentUpdatingPosSet.add(parentUpdatePos)) + { + break; } - // can be used for debugging - if (queueCount != 0) + try { - LOGGER.trace("Queued [" + queueCount + "] out of ["+updatePosList.size()+"] parent updates."); + executor.execute(() -> + { + ReentrantLock parentWriteLock = this.updateLockProvider.getLock(parentUpdatePos); + boolean parentLocked = false; + try + { + //LOGGER.info("updating parent: "+parentUpdatePos); + + // Locking the parent before the children should prevent deadlocks. + // TryLock is used instead of lock so this thread can handle a different update. + if (parentWriteLock.tryLock()) + { + parentLocked = true; + this.lockedPosSet.add(parentUpdatePos); + + // apply each child pos to the parent + for (DhSectionPos childPos : updatePosByParentPos.get(parentUpdatePos)) + { + ReentrantLock childReadLock = this.updateLockProvider.getLock(childPos); + try + { + childReadLock.lock(); + this.lockedPosSet.add(childPos); + + NewFullDataSource dataSource = this.get(childPos); + this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); + ((NewFullDataSourceRepo) this.repo).setApplyToParent(childPos, false); + } + catch (Exception e) + { + LOGGER.error("issue in update for parent pos: " + parentUpdatePos); + } + finally + { + childReadLock.unlock(); + this.lockedPosSet.remove(childPos); + } + } + } + } + finally + { + if (parentLocked) + { + parentWriteLock.unlock(); + this.lockedPosSet.remove(parentUpdatePos); + } + + this.parentUpdatingPosSet.remove(parentUpdatePos); + } + }); + } + catch (Exception e) + { + this.parentUpdatingPosSet.remove(parentUpdatePos); + throw e; } } } } + catch (InterruptedException ignored) { Thread.currentThread().interrupt(); } catch (Exception e) { LOGGER.error("Unexpected error in the parent update queue thread. Error: " + e.getMessage(), e); } - finally - { - this.updateQueueThreadRunningRef.set(false); - } - }); + } + + LOGGER.info("Update thread ["+Thread.currentThread().getName()+"] terminated."); } @@ -265,12 +279,20 @@ public class NewFullDataFileHandler @Override public void debugRender(DebugRenderer renderer) { - if (Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus.get()) - { - this.parentApplicationPositionSet - .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); - } + this.lockedPosSet + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 74f, 0.15f, Color.PINK)); }); + + this.queuedUpdateCountsByPos + .forEach((pos, updateCountRef) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f + (updateCountRef.get() * 16f), 0.20f, Color.WHITE)); }); + this.parentUpdatingPosSet + .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.MAGENTA)); }); } + @Override + public void close() + { + super.close(); + this.updateQueueProcessor.shutdownNow(); + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 5b38ed4e8..6adcf1360 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; @@ -28,9 +29,12 @@ import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.util.LodUtil; import org.apache.logging.log4j.Logger; +import java.awt.*; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -38,7 +42,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; -public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler +public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -46,10 +50,12 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); + protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 5_000); // TODO name better // this is just the list of section pos that have had their world generation // calculated and queued this session. + @Deprecated private final Set genHandledPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); @@ -126,6 +132,9 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler }); } + @Override + public int getUnsavedDataSourceCount() { return this.delayedFullDataSourceSaveCache.getUnsavedCount(); } + //=================// @@ -279,6 +288,16 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler }); } + @Override + public void debugRender(DebugRenderer renderer) + { + super.debugRender(renderer); + + this.delayedFullDataSourceSaveCache.dataSourceByPosition + .forEach((pos, dataSource) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 80f, 0.20f, Color.green.darker())); }); + } + + //================// @@ -305,10 +324,14 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler { return (chunkSizedFullDataSource) -> { - NewGeneratedFullDataFileHandler.this.level.updateDataSources(chunkSizedFullDataSource); + NewGeneratedFullDataFileHandler.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(chunkSizedFullDataSource); }; } } + private void onDataSourceSave(NewFullDataSource fullDataSource) + { NewGeneratedFullDataFileHandler.this.updateDataSourceAsync(fullDataSource); } + + /** used by external event listeners */ @FunctionalInterface diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java index 24e76ba78..d4a9d8a27 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java @@ -22,13 +22,23 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkModifiedEvent; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.transformers.ChunkToLodBuilder; +import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache; +import com.seibel.distanthorizons.core.pos.DhChunkPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector; +import java.util.HashSet; +import java.util.concurrent.ConcurrentHashMap; + public abstract class AbstractDhLevel implements IDhLevel { public final ChunkToLodBuilder chunkToLodBuilder; + protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 2_000); + /** contains the {@link DhChunkPos} for each {@link DhSectionPos} that are queued to save via {@link AbstractDhLevel#delayedFullDataSourceSaveCache} */ + protected final ConcurrentHashMap> updatedChunkPosSetBySectionPos = new ConcurrentHashMap<>(); + //=============// @@ -44,21 +54,51 @@ public abstract class AbstractDhLevel implements IDhLevel //=================// @Override - public void updateChunkAsync(IChunkWrapper chunk) + public int getUnsavedDataSourceCount() { return this.delayedFullDataSourceSaveCache.getUnsavedCount(); } + + @Override + public void updateChunkAsync(IChunkWrapper chunkWrapper) { - NewFullDataSource dataSource = NewFullDataSource.createFromChunk(chunk); + NewFullDataSource dataSource = NewFullDataSource.createFromChunk(chunkWrapper); if (dataSource == null) { // This can happen if, among other reasons, a chunk save is superseded by a later event return; } - this.updateDataSources(dataSource); - ApiEventInjector.INSTANCE.fireAllEvents( - DhApiChunkModifiedEvent.class, - new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunk.getChunkPos().x, chunk.getChunkPos().z)); + + this.updatedChunkPosSetBySectionPos.compute(dataSource.getSectionPos(), (dataSourcePos, chunkPosSet) -> + { + if (chunkPosSet == null) + { + chunkPosSet = new HashSet<>(); + } + chunkPosSet.add(chunkWrapper.getChunkPos()); + return chunkPosSet; + }); + + // batch updates to reduce overhead when flying around or breaking/placing a lot of blocks in an area + this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(dataSource); } + private void onDataSourceSave(NewFullDataSource fullDataSource) + { + this.updateDataSourcesAsync(fullDataSource).thenRun(() -> + { + HashSet updatedChunkPosSet = this.updatedChunkPosSetBySectionPos.remove(fullDataSource.getSectionPos()); + if (updatedChunkPosSet != null) + { + for (DhChunkPos chunkPos : updatedChunkPosSet) + { + ApiEventInjector.INSTANCE.fireAllEvents( + DhApiChunkModifiedEvent.class, + new DhApiChunkModifiedEvent.EventParam(this.getLevelWrapper(), chunkPos.x, chunkPos.z)); + } + } + }); + } + + @Override public void close() { this.chunkToLodBuilder.close(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index c1f968156..e49129a71 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -44,6 +44,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapp import org.apache.logging.log4j.Logger; import java.io.Closeable; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc @@ -209,7 +210,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // data handling // //===============// - public void updateDataSources(NewFullDataSource data) { this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } + public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } @Override public void OnDataSourceUpdated(NewFullDataSource updatedFullDataSource) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 500588d13..439b0228d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import java.io.File; +import java.util.concurrent.CompletableFuture; /** The level used when connected to a server */ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel @@ -118,7 +119,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public ILevelWrapper getLevelWrapper() { return levelWrapper; } @Override - public void updateDataSources(NewFullDataSource data) { this.clientside.updateDataSources(data); } + public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.clientside.updateDataSourcesAsync(data); } @Override public int getMinY() { return levelWrapper.getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 8a063f933..f2cf7a8e8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -43,6 +43,7 @@ import org.apache.logging.log4j.Logger; import java.awt.*; import java.util.Iterator; +import java.util.concurrent.CompletableFuture; /** The level used on a singleplayer world */ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLevel, IDhServerLevel @@ -186,7 +187,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public boolean hasSkyLight() { return this.serverLevelWrapper.hasSkyLight(); } @Override - public void updateDataSources(NewFullDataSource data) { this.clientside.updateDataSources(data); } + public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.clientside.updateDataSourcesAsync(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index e8cdd66de..ec62cf12a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -30,6 +30,8 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import org.apache.logging.log4j.Logger; +import java.util.concurrent.CompletableFuture; + public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -50,12 +52,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public void serverTick() { this.chunkToLodBuilder.tick(); } @Override - public void updateDataSources(NewFullDataSource data) - { - DhSectionPos pos = data.getSectionPos(); - pos = pos.convertNewToDetailLevel(CompleteFullDataSource.SECTION_SIZE_OFFSET); - this.getFullDataProvider().updateDataSourceAsync(data); - } + public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.getFullDataProvider().updateDataSourceAsync(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index 02331409b..f8baafd4a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -25,6 +25,8 @@ import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import java.util.concurrent.CompletableFuture; + public interface IDhLevel extends AutoCloseable { int getMinY(); @@ -43,6 +45,12 @@ public interface IDhLevel extends AutoCloseable boolean hasSkyLight(); - void updateDataSources(NewFullDataSource data); + CompletableFuture updateDataSourcesAsync(NewFullDataSource data); + + /** + * this number is generally related to how many data sources have been updated + * due to chunk modifications or loads. + */ + int getUnsavedDataSourceCount(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PositionalLockProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PositionalLockProvider.java new file mode 100644 index 000000000..515c8c04a --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/PositionalLockProvider.java @@ -0,0 +1,187 @@ +package com.seibel.distanthorizons.core.util.threading; + +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.ThreadUtil; +import org.apache.logging.log4j.Logger; + +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Handles creating and destroying {@link ReentrantLock}'s for + * a given {@link DhSectionPos}. + * This is necessary since we need an unlimited number of locks + * when handling data updating, but we don't want to infinitely create locks. + * This provider will create/destroy locks as necessary given the current requirements by the file handlers. + */ +public class PositionalLockProvider +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + private static final ThreadPoolExecutor LOCK_CLEANUP_THREAD = ThreadUtil.makeSingleThreadPool("Positional Lock Cleanup"); + private static final int CLEANUP_THREAD_MAX_FREQUENCY_IN_MS = 1000; + + /** How long a lock can be unused before it is eligible for deletion */ + private static final long UNUSED_LOCK_TIMEOUT_IN_MS = 10_000; // 10 seconds + private static final int MAX_NUMBER_OF_LOCKS = 100; + + + private final ConcurrentHashMap lockByPos = new ConcurrentHashMap<>(); + + private final AtomicBoolean lockRemovalThreadRunning = new AtomicBoolean(false); + + + + //=============// + // constructor // + //=============// + + public PositionalLockProvider() {} + + + + //========// + // getter // + //========// + + public ReentrantLock getLock(DhSectionPos pos) + { + return this.lockByPos.compute(pos, (ignorePos, lock) -> + { + if (lock == null) + { + lock = new ExpiringLock(); + } + + if (this.lockByPos.size() > MAX_NUMBER_OF_LOCKS + && this.lockRemovalThreadRunning.getAndSet(true)) + { + LOCK_CLEANUP_THREAD.execute(this::removeExpiredLocks); + } + + return lock; + }); + } + private void removeExpiredLocks() + { + try + { + // we don't need to trigger this every time the lock count is over the limit + Thread.sleep(CLEANUP_THREAD_MAX_FREQUENCY_IN_MS); + + // walk over every lock and check which ones need to be removed + Iterator keySet = this.lockByPos.keySet().iterator(); + while (keySet.hasNext()) + { + try + { + long currentTime = System.currentTimeMillis(); + + DhSectionPos pos = keySet.next(); + ExpiringLock lock = this.lockByPos.get(pos); + + // don't try removing a lock that's currently in use + if (lock.tryLockWithoutUpdatingExpirationTime()) + { + if (currentTime > lock.expirationTimeInMs) + { + this.lockByPos.remove(pos); + //LOGGER.info("removed lock: "+pos); + } + lock.unlock(); + } + + } + catch (NoSuchElementException ignore) { } + } + } + catch (Exception e) + { + LOGGER.error("PositionLockProvider unexpected error when removing expired locks. Error: "+e.getMessage(), e); + } + finally + { + this.lockRemovalThreadRunning.set(false); + //LOGGER.info("update lock count: "+this.lockByPos.size()); + } + } + + + + //================// + // helper classes // + //================// + + /** a lock that tracks when it was last used */ + private static class ExpiringLock extends ReentrantLock + { + /** + * Indicates the system time in Milliseconds when this lock has expired.
+ * Should update whenever the lock is used. + */ + public long expirationTimeInMs; + + + + //=============// + // constructor // + //=============// + + public ExpiringLock() { this.resetExpirationTime(); } + + + + //===========// + // overrides // + //===========// + + @Override + public void lock() + { + this.resetExpirationTime(); + super.lock(); + } + @Override + public boolean tryLock() + { + this.resetExpirationTime(); + return super.tryLock(); + } + @Override + public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException + { + this.resetExpirationTime(); + return super.tryLock(timeout, unit); + } + + @Override + public void unlock() + { + this.resetExpirationTime(); + super.unlock(); + } + + + + //================// + // helper methods // + //================// + + /** should be called whenever this lock is needed */ + private void resetExpirationTime() + { + this.expirationTimeInMs = System.currentTimeMillis() + UNUSED_LOCK_TIMEOUT_IN_MS; + } + + public boolean tryLockWithoutUpdatingExpirationTime() { return super.tryLock(); } + + + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java index 1d78ea34e..3adacc81c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java @@ -47,9 +47,9 @@ public class ThreadPoolUtil public static ThreadPoolExecutor getFileHandlerExecutor() { return fileHandlerThreadPool.executor; } public static final DhThreadFactory UPDATE_PROPAGATOR_THREAD_FACTORY = new DhThreadFactory("LOD Update Propagator", Thread.MIN_PRIORITY); - private static RateLimitedThreadPoolExecutor updatePropagatorThreadPool; + private static ConfigThreadPool updatePropagatorThreadPool; @Nullable - public static ThreadPoolExecutor getUpdatePropagatorExecutor() { return updatePropagatorThreadPool; } + public static ThreadPoolExecutor getUpdatePropagatorExecutor() { return updatePropagatorThreadPool.executor; } public static final DhThreadFactory WORLD_GEN_THREAD_FACTORY = new DhThreadFactory("World Gen", Thread.MIN_PRIORITY); private static ConfigThreadPool worldGenThreadPool; @@ -105,7 +105,7 @@ public class ThreadPoolUtil // standalone threads // fileHandlerThreadPool = new ConfigThreadPool(FILE_HANDLER_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, null); - updatePropagatorThreadPool = new RateLimitedThreadPoolExecutor(8, 1.0, UPDATE_PROPAGATOR_THREAD_FACTORY); + updatePropagatorThreadPool = new ConfigThreadPool(UPDATE_PROPAGATOR_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads, null); worldGenThreadPool = new ConfigThreadPool(WORLD_GEN_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, null); bufferUploaderThreadPool = ThreadUtil.makeSingleThreadPool(BUFFER_UPLOADER_THREAD_NAME); @@ -145,7 +145,7 @@ public class ThreadPoolUtil { // standalone threads fileHandlerThreadPool.shutdownExecutorService(); - updatePropagatorThreadPool.shutdown(); + updatePropagatorThreadPool.shutdownExecutorService(); worldGenThreadPool.shutdownExecutorService(); bufferUploaderThreadPool.shutdown(); From 20394068b29b4ff3fac5629c0920860317a1a5eb Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 9 Mar 2024 17:06:17 -0600 Subject: [PATCH 022/183] Add additional F3 lines for file updating --- .../core/file/renderfile/RenderSourceFileHandler.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 1724a6a5b..1be730830 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataS import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -138,6 +139,10 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler Date: Sun, 10 Mar 2024 20:44:12 -0500 Subject: [PATCH 023/183] Fix world gen for extreme render distances --- .../fullDatafile/IFullDataSourceProvider.java | 40 +++- .../fullDatafile/NewFullDataFileHandler.java | 2 +- .../NewGeneratedFullDataFileHandler.java | 189 ++++++++++-------- .../renderfile/RenderSourceFileHandler.java | 24 ++- .../generation/IWorldGenerationQueue.java | 4 + .../core/generation/WorldGenerationQueue.java | 25 ++- .../core/level/ClientLevelModule.java | 2 +- .../core/level/WorldGenModule.java | 3 +- .../core/pos/DhSectionPos.java | 6 + .../core/render/LodQuadTree.java | 113 +++++++++-- .../core/render/LodRenderSection.java | 56 ++++++ .../core/sql/repo/NewFullDataSourceRepo.java | 20 ++ 12 files changed, 355 insertions(+), 129 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index 889462ca6..f625f50b0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -24,7 +24,9 @@ import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; +import org.jetbrains.annotations.Nullable; +import java.util.ArrayList; import java.util.concurrent.CompletableFuture; /** @@ -36,16 +38,40 @@ public interface IFullDataSourceProvider extends ISourceProvider getAsync(DhSectionPos pos); NewFullDataSource get(DhSectionPos pos); - /** - * If this provider has the ability to create (world gen) or get (networking) - * missing data sources this method will queue the given position - * for generation or retrieval. - */ - void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos); - CompletableFuture updateDataSourceAsync(NewFullDataSource chunkData); /** @return -1 if this provider never has unsaved data sources */ default int getUnsavedDataSourceCount() { return -1; } + + + // retrieval (world gen) // + + /** + * If true this {@link IFullDataSourceProvider} can generate or retrieve + * {@link NewFullDataSource}'s that aren't currently in the database. + */ + default boolean canRetrieveMissingDataSources() { return false; } + + /** @return null if it was unable to generate any positions, an empty array if all positions were generated */ + @Nullable + default ArrayList getPositionsToRetrieve(DhSectionPos pos) { return null; } + /** + * Returns how many positions could potentially be generated for this position assuming the position is empty. + * Used when estimating the total number of retrieval requests. + */ + default int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) { return -1; } + + /** @return true if the position was queued, false if not */ + default boolean queuePositionForRetrieval(DhSectionPos genPos) { return false; } + /** + * @return false if the provider isn't accepting new requests, + * this can be due to having a full queue or some other + * limiting factor. + */ + default boolean canQueueRetrieval() { return false; } + + /** Can be used to display how many total retrieval requests might be available. */ + default void setTotalRetrievalPositionCount(int newCount) { } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 38d075e89..5cb74f015 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -127,7 +127,7 @@ public class NewFullDataFileHandler @Override protected NewFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { - // TODO maybe just set children update flags to true? + // TODO maybe just set children update flags to true? return NewFullDataSource.createEmpty(pos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 6adcf1360..81652fa0f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -31,13 +31,15 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; +import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; +import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import org.apache.logging.log4j.Logger; import java.awt.*; import java.util.*; import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; @@ -46,18 +48,14 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); + public static final int MAX_WORLD_GEN_REQUESTS_PER_THREAD = 20; + + private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 5_000); - // TODO name better - // this is just the list of section pos that have had their world generation - // calculated and queued this session. - @Deprecated - private final Set genHandledPosSet = Collections.newSetFromMap(new ConcurrentHashMap<>()); - //=============// @@ -68,29 +66,6 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl - //===========// - // overrides // - //===========// - - @Override - public NewFullDataSource get(DhSectionPos pos) { return this.get(pos, true); } - public NewFullDataSource get(DhSectionPos pos, boolean runWorldGenCheck) - { - NewFullDataSource dataSource = super.get(pos); - - if (runWorldGenCheck) - { - this.tryQueueSection(pos); - } - - return dataSource; - } - - @Override - public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) { this.tryQueueSection(pos); } - - - //==================// // generation queue // //==================// @@ -106,11 +81,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl LOGGER.info("Set world gen queue for level ["+this.level+"]."); } - public void clearGenerationQueue() - { - this.worldGenQueueRef.set(null); - this.genHandledPosSet.clear(); - } + public void clearGenerationQueue() { this.worldGenQueueRef.set(null); } /** Can be used to remove positions that are outside the player's render distance. */ public void removeGenRequestIf(Function removeIf) @@ -122,14 +93,6 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl { worldGenQueue.removeGenRequestIf(removeIf); } - - this.genHandledPosSet.forEach((pos) -> - { - if (removeIf.apply(pos)) - { - this.genHandledPosSet.remove(pos); - } - }); } @Override @@ -191,61 +154,92 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl - //================// - // helper methods // - //================// + //===================================// + // world gen (data source retrieval) // + //===================================// - /** does nothing if this section or one of it's parents has already been queued */ - public void tryQueueSection(DhSectionPos pos) + @Override + public boolean canRetrieveMissingDataSources() { return true; } + + @Override + public void setTotalRetrievalPositionCount(int newCount) { - IWorldGenerationQueue tempWorldGenQueue = this.worldGenQueueRef.get(); - if (tempWorldGenQueue == null) + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) { - return; + worldGenQueue.setEstimatedTotalTaskCount(newCount); } - - if (this.genHandledPosSet.contains(pos)) + } + + @Override + public boolean canQueueRetrieval() + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) { - return; + // we can't queue anything if the world generator isn't set up yet + return false; } - - AtomicBoolean positionAlreadyHandled = new AtomicBoolean(false); - pos.forEachPosUpToDetailLevel(NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL, (parentPos) -> + ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); + if (updateExecutor == null || updateExecutor.getQueue().size() >= MAX_UPDATE_TASK_COUNT) { - if (!positionAlreadyHandled.get()) - { - if (this.genHandledPosSet.contains(parentPos)) - { - positionAlreadyHandled.set(true); - } - } - }); - - if (positionAlreadyHandled.get()) - { - return; + // don't queue additional world gen requests if the updater is behind + return false; } - this.genHandledPosSet.add(pos); + // don't queue additional world gen requests beyond the max allotted + int maxQueueCount = MAX_WORLD_GEN_REQUESTS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads.get(); + return worldGenQueue.getWaitingTaskCount() < maxQueueCount; + } + + @Override + public boolean queuePositionForRetrieval(DhSectionPos genPos) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return false; + } - // get the un-generated pos list - byte minGeneratorSectionDetailLevel = (byte) (tempWorldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + GenTask genTask = new GenTask(genPos); + CompletableFuture worldGenFuture = worldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask); + worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex)); + return true; + } + + @Override + public ArrayList getPositionsToRetrieve(DhSectionPos pos) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return null; + } + + + // TODO based on the column generation step, only check children that are un-generated + ArrayList generationList = new ArrayList<>(); + byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); pos.forEachChildAtDetailLevel(minGeneratorSectionDetailLevel, (genPos) -> { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue == null) + if (!this.repo.existsWithKey(genPos)) { - return; + // nothing exists for this position, it needs generation + generationList.add(genPos); } - - if (this.repo.existsWithKey(genPos)) + else { - // TODO only pull in the generation steps - NewFullDataSource potentialDataSource = this.get(genPos, false); + byte[] columnGenerationSteps = ((NewFullDataSourceRepo)this.repo).getColumnGenerationStepForPos(genPos); + if (columnGenerationSteps == null) + { + // shouldn't happen, but just in case + return; + } + EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT; checkWorldGenLoop: @@ -254,7 +248,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl for (int z = 0; z < NewFullDataSource.WIDTH; z++) { int index = NewFullDataSource.relativePosToIndex(x, z); - byte genStepValue = potentialDataSource.columnGenerationSteps[index]; + byte genStepValue = columnGenerationSteps[index]; if (genStepValue < currentMinWorldGenStep.value) { @@ -275,19 +269,38 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl if (currentMinWorldGenStep != EDhApiWorldGenerationStep.EMPTY) { - // no world gen needed + // no world gen needed for this position return; } + + generationList.add(genPos); } - - - // queue each new gen task - GenTask genTask = new GenTask(genPos); - CompletableFuture worldGenFuture = tempWorldGenQueue.submitGenTask(genPos, (byte) (genPos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL), genTask); - worldGenFuture.whenComplete((genTaskResult, ex) -> this.onWorldGenTaskComplete(genTaskResult, ex)); }); + + return generationList; } + @Override + public int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) + { + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue == null) + { + return -1; + } + + int minGeneratorSectionDetailLevel = worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + int detailLevelDiff = pos.getDetailLevel() - minGeneratorSectionDetailLevel; + + return BitShiftUtil.powerOfTwo(detailLevelDiff); + } + + + + //=======// + // debug // + //=======// + @Override public void debugRender(DebugRenderer renderer) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 1be730830..c2b4a06c6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -47,6 +47,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) + { + // TODO once the legacy data provider has been replaced this can be removed + this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); + return CompletableFuture.completedFuture(null); + } + + //=========// // F3 menu // @@ -148,12 +158,6 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) - { - this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); - return CompletableFuture.completedFuture(null); - } //=====================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index 52caedbef..cdea7be81 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -45,6 +45,10 @@ public interface IWorldGenerationQueue extends Closeable int getWaitingTaskCount(); int getInProgressTaskCount(); + /** used for rendering to the F3 menu */ + int getEstimatedTotalTaskCount(); + void setEstimatedTotalTaskCount(int newEstimate); + CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning); void close(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 27cce7d41..cd8c87f08 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -67,13 +67,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender /** largest numerical detail level allowed */ public final byte lowestDataDetail; - @Override - public byte lowestDataDetail() { return this.lowestDataDetail; } - /** smallest numerical detail level allowed */ public final byte highestDataDetail; - @Override - public byte highestDataDetail() { return this.highestDataDetail; } /** If not null this generator is in the process of shutting down */ @@ -96,6 +91,9 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender private final HashMap alreadyGeneratedPosHashSet = new HashMap<>(MAX_ALREADY_GENERATED_COUNT); private final Queue alreadyGeneratedPosQueue = new LinkedList<>(); + /** just used for rendering to the F3 menu */ + private int estimatedTotalTaskCount = 0; + //==============// @@ -498,13 +496,24 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender - //=========// - // getters // - //=========// + //===================// + // getters / setters // + //===================// public int getWaitingTaskCount() { return this.waitingTasks.size(); } public int getInProgressTaskCount() { return this.inProgressGenTasksByLodPos.size(); } + @Override + public byte lowestDataDetail() { return this.lowestDataDetail; } + @Override + public byte highestDataDetail() { return this.highestDataDetail; } + + @Override + public int getEstimatedTotalTaskCount() { return this.estimatedTotalTaskCount; } + @Override + public void setEstimatedTotalTaskCount(int newEstimate) { this.estimatedTotalTaskCount = newEstimate; } + + //==========// // shutdown // diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index e49129a71..c1648dab7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -316,7 +316,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get() * LodUtil.CHUNK_WIDTH * 2, // initial position is (0,0) just in case the player hasn't loaded in yet, the tree will be moved once the level starts ticking 0, 0, - this.renderSourceFileHandler); + this.renderSourceFileHandler.fullDataSourceProvider, this.renderSourceFileHandler); RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree); this.renderer = new LodRenderer(renderBufferHandler); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index f008660b1..0d7f5bf98 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -53,8 +53,9 @@ public class WorldGenModule implements Closeable { int waitingCount = worldGenState.worldGenerationQueue.getWaitingTaskCount(); int inProgressCount = worldGenState.worldGenerationQueue.getInProgressTaskCount(); + int totalCountEstimate = worldGenState.worldGenerationQueue.getEstimatedTotalTaskCount(); - return "World Gen Tasks: "+waitingCount+", (in progress: "+inProgressCount+")"; + return "World Gen Tasks: "+waitingCount+"/"+totalCountEstimate+", (in progress: "+inProgressCount+")"; } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index c6cc3a0ab..d64bfe83c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -224,6 +224,12 @@ public class DhSectionPos return (centerBlockPos * BitShiftUtil.powerOfTwo(this.detailLevel)) + positionOffset; } + public int getManhattanBlockDistance(DhBlockPos2D blockPos) + { + return Math.abs(this.getCenterBlockPosX() - blockPos.x) + + Math.abs(this.getCenterBlockPosZ() - blockPos.z); + } + //==================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 5ed35e536..a2c145f24 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -23,20 +23,25 @@ import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import java.util.ArrayList; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; /** @@ -48,8 +53,13 @@ public class LodQuadTree extends QuadTree implements AutoClose public static final byte TREE_LOWEST_DETAIL_LEVEL = ColumnRenderSource.SECTION_SIZE_OFFSET; private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + /** there should only ever be one {@link LodQuadTree} so having the thread static should be fine */ + private static final ThreadPoolExecutor FULL_DATA_RETRIEVAL_QUEUE_THREAD = ThreadUtil.makeSingleThreadPool("QuadTree Full Data Retrieval Queue Populator"); + private static final int WORLD_GEN_QUEUE_UPDATE_DELAY_IN_MS = 1_000; + public final int blockRenderDistanceDiameter; + private final IFullDataSourceProvider fullDataSourceProvider; private final IRenderSourceProvider renderSourceProvider; /** @@ -60,6 +70,7 @@ public class LodQuadTree extends QuadTree implements AutoClose private final IDhClientLevel level; //FIXME: Proper hierarchy to remove this reference! private final ConfigChangeListener horizontalScaleChangeListener; private final ReentrantLock treeReadWriteLock = new ReentrantLock(); + private final AtomicBoolean fullDataRetrievalQueueRunning = new AtomicBoolean(false); /** the smallest numerical detail level number that can be rendered */ private byte maxRenderDetailLevel; @@ -80,12 +91,14 @@ public class LodQuadTree extends QuadTree implements AutoClose public LodQuadTree( IDhClientLevel level, int viewDiameterInBlocks, int initialPlayerBlockX, int initialPlayerBlockZ, - IRenderSourceProvider provider) + IFullDataSourceProvider fullDataSourceProvider, + IRenderSourceProvider renderSourceProvider) { super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL); this.level = level; - this.renderSourceProvider = provider; + this.fullDataSourceProvider = fullDataSourceProvider; + this.renderSourceProvider = renderSourceProvider; this.blockRenderDistanceDiameter = viewDiameterInBlocks; this.horizontalScaleChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, (newHorizontalScale) -> this.onHorizontalQualityChange()); @@ -163,6 +176,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // walk through each root node + ArrayList nodesNeedingRetrieval = new ArrayList<>(); Iterator rootPosIterator = this.rootNodePosIterator(); while (rootPosIterator.hasNext()) { @@ -174,14 +188,23 @@ public class LodQuadTree extends QuadTree implements AutoClose } QuadNode rootNode = this.getNode(rootPos); - this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false); + this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingRetrieval); + } + + + // full data retrieval (world gen) + if (!this.fullDataRetrievalQueueRunning.get()) + { + this.fullDataRetrievalQueueRunning.set(true); + FULL_DATA_RETRIEVAL_QUEUE_THREAD.execute(() -> this.queueFullDataRetrievalTasks(playerPos, nodesNeedingRetrieval)); } } /** @return whether the current position is able to render (note: not if it IS rendering, just if it is ABLE to.) */ private boolean recursivelyUpdateRenderSectionNode( DhBlockPos2D playerPos, QuadNode rootNode, QuadNode quadNode, DhSectionPos sectionPos, - boolean parentRenderSectionIsEnabled) + boolean parentRenderSectionIsEnabled, + ArrayList nodesNeedingRetrieval) { //===============================// // node and render section setup // @@ -237,7 +260,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled, nodesNeedingRetrieval); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } @@ -259,7 +282,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled, nodesNeedingRetrieval); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } if (!allChildrenSectionsAreLoaded) @@ -276,6 +299,9 @@ public class LodQuadTree extends QuadTree implements AutoClose // TODO this should only equal the expected detail level, the (expectedDetailLevel-1) is a temporary fix to prevent corners from being cut out else if (sectionPos.getDetailLevel() == expectedDetailLevel || sectionPos.getDetailLevel() == expectedDetailLevel - 1) { + // this is the detail level we want to render // + + /* Can be uncommented to easily debug a single render section. */ /* Don't forget the disableRendering() at the bottom though. */ //if (sectionPos.getDetailLevel() == 10 @@ -285,9 +311,9 @@ public class LodQuadTree extends QuadTree implements AutoClose // sectionPos.getZ() == -4 // )) { - // this is the detail level we want to render // // prepare this section for rendering - renderSection.loadRenderSource(this.renderSourceProvider, this.level); // TODO this should fire for the lowest detail level first, wait for it to finish then fire the next highest to prevent waiting forever for 2 million chunk section to finish sampling everything + // TODO this should fire for the lowest detail level first to improve loading speed + renderSection.loadRenderSource(this.renderSourceProvider, this.level); // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections if (!parentRenderSectionIsEnabled && renderSection.canRenderNow()) @@ -308,6 +334,11 @@ public class LodQuadTree extends QuadTree implements AutoClose }); } } + + if (!renderSection.isFullyGenerated()) + { + nodesNeedingRetrieval.add(renderSection); + } } //else //{ @@ -400,7 +431,7 @@ public class LodQuadTree extends QuadTree implements AutoClose */ public void clearRenderDataCache() { - if (this.treeReadWriteLock.tryLock()) + if (this.treeReadWriteLock.tryLock()) // TODO make async, can lock render thread { try { @@ -451,14 +482,70 @@ public class LodQuadTree extends QuadTree implements AutoClose + //=================================// + // full data retrieval (world gen) // + //=================================// + + private void queueFullDataRetrievalTasks(DhBlockPos2D playerPos, ArrayList nodesNeedingRetrieval) + { + try + { + // add a slight delay since we don't need to check the world gen queue every tick + Thread.sleep(WORLD_GEN_QUEUE_UPDATE_DELAY_IN_MS); + + // sort the nodes from nearest to farthest + nodesNeedingRetrieval.sort((a, b) -> + { + int aDist = a.pos.getManhattanBlockDistance(playerPos); + int bDist = b.pos.getManhattanBlockDistance(playerPos); + return Integer.compare(aDist, bDist); + }); + + // add retrieval tasks to the queue + for (int i = 0; i < nodesNeedingRetrieval.size(); i++) + { + LodRenderSection renderSection = nodesNeedingRetrieval.get(i); + if (!this.fullDataSourceProvider.canQueueRetrieval()) + { + break; + } + + renderSection.tryQueuingMissingLodRetrieval(this.fullDataSourceProvider); + } + + // calculate an estimate for the max number of tasks for the queue + int totalWorldGenCount = 0; + for (int i = 0; i < nodesNeedingRetrieval.size(); i++) + { + LodRenderSection renderSection = nodesNeedingRetrieval.get(i); + if (!renderSection.missingPositionsCalculated()) + { + // may be higher than the actual amount + totalWorldGenCount += this.fullDataSourceProvider.getMaxPossibleRetrievalPositionCountForPos(renderSection.pos); + } + else + { + totalWorldGenCount += renderSection.ungeneratedPositionCount(); + } + } + this.fullDataSourceProvider.setTotalRetrievalPositionCount(totalWorldGenCount); + } + catch (Exception e) + { + LOGGER.error("Unexpected error: "+e.getMessage(), e); + } + finally + { + this.fullDataRetrievalQueueRunning.set(false); + } + } + + //==================// // config listeners // //==================// - private void onHorizontalQualityChange() - { - this.clearRenderDataCache(); - } + private void onHorizontalQualityChange() { this.clearRenderDataCache(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 629fffab0..0a72522de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.distanthorizons.core.enums.EDhDirection; +import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -36,6 +37,7 @@ import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import org.apache.logging.log4j.Logger; import java.awt.*; +import java.util.ArrayList; import java.util.Objects; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; @@ -84,6 +86,10 @@ public class LodRenderSection implements IDebugRenderable private final QuadTree parentQuadTree; + private boolean missingPositionsCalculated = false; + /** should be an empty array if no positions need to be generated */ + private ArrayList missingGenerationPos = null; + //=============// @@ -407,6 +413,56 @@ public class LodRenderSection implements IDebugRenderable + //=================================// + // full data retrieval (world gen) // + //=================================// + + public boolean isFullyGenerated() { return this.missingPositionsCalculated && this.missingGenerationPos.size() == 0; } + public boolean missingPositionsCalculated() { return this.missingPositionsCalculated; } + public int ungeneratedPositionCount() { return (this.missingGenerationPos != null) ? this.missingGenerationPos.size() : 0; } + + public void tryQueuingMissingLodRetrieval(IFullDataSourceProvider fullDataSourceProvider) + { + if (fullDataSourceProvider.canRetrieveMissingDataSources() && fullDataSourceProvider.canQueueRetrieval()) + { + // calculate the missing positions if not already done + if (!this.missingPositionsCalculated) + { + this.missingGenerationPos = fullDataSourceProvider.getPositionsToRetrieve(this.pos); + if (this.missingGenerationPos != null) + { + this.missingPositionsCalculated = true; + } + } + + // if the missing positions were found, queue them + if (this.missingGenerationPos != null) + { + // queue from last to first to prevent shifting the array unnecessarily + for (int i = this.missingGenerationPos.size() - 1; i >= 0; i--) + { + if (!fullDataSourceProvider.canQueueRetrieval()) + { + // the data source provider isn't accepting any more jobs + break; + } + + DhSectionPos pos = this.missingGenerationPos.remove(i); + boolean positionQueued = fullDataSourceProvider.queuePositionForRetrieval(pos); + if (!positionQueued) + { + // shouldn't normally happen, but just in case + this.missingGenerationPos.add(pos); + break; + } + } + } + } + } + + + + //==============// // base methods // //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java index 6f0ab5ff0..f5d27003c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java @@ -209,6 +209,26 @@ public class NewFullDataSourceRepo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( + "select ColumnGenerationStep " + + "from "+this.getTableName()+" " + + "WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ()); + + if (resultMap != null) + { + return (byte[]) resultMap.get("ColumnGenerationStep"); + } + else + { + return null; + } + } + } From 8bdc4fe77961592c81896b3dcd1900c30cb9412e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:05:13 -0500 Subject: [PATCH 024/183] Add missing changed for 28de6f93 --- .../fullDatafile/NewFullDataFileHandler.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 5cb74f015..d7e7baffd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -51,12 +51,12 @@ public class NewFullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; + protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; /** how many parent update tasks can be in the queue at once */ - private static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); + protected static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); /** indicates how long the update queue thread should wait between queuing ticks */ - private static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250; + protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250; public final Set parentUpdatingPosSet = ConcurrentHashMap.newKeySet(); @@ -140,15 +140,6 @@ public class NewFullDataFileHandler // parent updates // //================// - @Override - public void queuePositionForGenerationOrRetrievalIfNecessary(DhSectionPos pos) - { - // Do nothing. - // This file handler doesn't have the ability to generate or retrieve data sources - // that aren't already in the database - } - - private void runUpdateQueue() { while (!Thread.interrupted()) @@ -166,8 +157,8 @@ public class NewFullDataFileHandler // only add more items to the queue if half or more of the previous tasks have been completed - if (executor.getQueue().size() < (MAX_UPDATE_TASK_COUNT) - && this.parentUpdatingPosSet.size() < (MAX_UPDATE_TASK_COUNT)) + if (executor.getQueue().size() < MAX_UPDATE_TASK_COUNT + && this.parentUpdatingPosSet.size() < MAX_UPDATE_TASK_COUNT) { // get the positions that need to be applied to their parents ArrayList parentUpdatePosList = ((NewFullDataSourceRepo) this.repo).getPositionsToUpdate(MAX_UPDATE_TASK_COUNT); From 02a144573263e7100c6348a487d6fece3f1c94ce Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:14:59 -0500 Subject: [PATCH 025/183] Fix render data not regenerating on config change --- .../distanthorizons/core/sql/dto/NewFullDataSourceDTO.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java index 154b47230..cba8a4d7f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java @@ -129,6 +129,8 @@ public class NewFullDataSourceDTO implements IBaseDTO dataSource.levelMinY = this.levelMinY; + dataSource.markNotEmpty(); + return dataSource; } From 46ba2630f117fef1c2fe41acc6dfba0f01b44b21 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:15:37 -0500 Subject: [PATCH 026/183] Fix LodQuadTree clear render cache not working at extreme distances --- .../com/seibel/distanthorizons/core/render/LodQuadTree.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index a2c145f24..ed6ed0232 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -437,6 +437,9 @@ public class LodQuadTree extends QuadTree implements AutoClose { LOGGER.info("Clearing render cache..."); + // delete the cache first so the nodes won't accidentally try re-loading the old data + this.renderSourceProvider.deleteRenderCache(); + // clear the tree Iterator> nodeIterator = this.nodeIterator(); while (nodeIterator.hasNext()) @@ -449,7 +452,6 @@ public class LodQuadTree extends QuadTree implements AutoClose } } - this.renderSourceProvider.deleteRenderCache(); LOGGER.info("Render cache invalidated, please wait a moment for everything to reload..."); } catch (Exception e) From b49988af9e3344ce7bf2740ddd598362b1464386 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:15:45 -0500 Subject: [PATCH 027/183] Remove an unused method added by Iris --- .../seibel/distanthorizons/core/render/LodRenderSection.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 0a72522de..dc557d1db 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -303,9 +303,6 @@ public class LodRenderSection implements IDebugRenderable } } - // probably used by Iris - public void disposeBufferForRecreate() { this.disposeActiveBuffer = true; } - /** * Try and swap in new render buffer for this section. Note that before this call, there should be no other From 5f8eceee8de67467464e6dc569ebad25f3a2aecb Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:16:03 -0500 Subject: [PATCH 028/183] Remove unused override in RenderSourceFileHandler --- .../fullData/sources/NewFullDataSource.java | 2 ++ .../core/file/renderfile/RenderSourceFileHandler.java | 11 ----------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 317a30ebb..3510d789b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -670,6 +670,8 @@ public class NewFullDataSource implements IDataSource } } + + //============// // deprecated // //============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index c2b4a06c6..d7ae22c10 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -67,17 +67,6 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler Date: Sun, 10 Mar 2024 21:30:59 -0500 Subject: [PATCH 029/183] Deprecate EHorizontalQuality.UNLIMITED --- .../distanthorizons/api/enums/config/EHorizontalQuality.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java index 3fdcddd83..b180498d5 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java @@ -45,6 +45,9 @@ public enum EHorizontalQuality HIGH(2.2f, 24), EXTREME(2.4f, 64), + /** @deprecated this setting is unmaintainable at high render distances. */ + @Deprecated + @DisallowSelectingViaConfigGui UNLIMITED(-1, -1); From 64fce77a0a4026d3d11b73efd54acd57aced9eac Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:38:41 -0500 Subject: [PATCH 030/183] Fix repo castring for AbstractNewDataSourceHandler --- .../core/file/AbstractNewDataSourceHandler.java | 7 ++++--- .../core/file/fullDatafile/NewFullDataFileHandler.java | 8 ++++---- .../fullDatafile/NewGeneratedFullDataFileHandler.java | 3 +-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index ed2bc7746..a2b1adf49 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -22,7 +22,8 @@ import java.util.concurrent.locks.ReentrantLock; public abstract class AbstractNewDataSourceHandler , - TDTO extends IBaseDTO, + TDTO extends IBaseDTO, + TRepo extends AbstractDhRepo, TDhLevel extends IDhLevel> implements ISourceProvider { @@ -58,7 +59,7 @@ public abstract class AbstractNewDataSourceHandler protected final TDhLevel level; protected final File saveDir; - public final AbstractDhRepo repo; + public final TRepo repo; public final ArrayList> dateSourceUpdateListeners = new ArrayList<>(); @@ -89,7 +90,7 @@ public abstract class AbstractNewDataSourceHandler //==================// /** When this is called the parent folders should be created */ - protected abstract AbstractDhRepo createRepo(); + protected abstract TRepo createRepo(); protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException; protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index d7e7baffd..ad3e708aa 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -46,7 +46,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.locks.ReentrantLock; public class NewFullDataFileHandler - extends AbstractNewDataSourceHandler + extends AbstractNewDataSourceHandler implements IFullDataSourceProvider, IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -93,7 +93,7 @@ public class NewFullDataFileHandler //====================// @Override - protected AbstractDhRepo createRepo() + protected NewFullDataSourceRepo createRepo() { try { @@ -161,7 +161,7 @@ public class NewFullDataFileHandler && this.parentUpdatingPosSet.size() < MAX_UPDATE_TASK_COUNT) { // get the positions that need to be applied to their parents - ArrayList parentUpdatePosList = ((NewFullDataSourceRepo) this.repo).getPositionsToUpdate(MAX_UPDATE_TASK_COUNT); + ArrayList parentUpdatePosList = this.repo.getPositionsToUpdate(MAX_UPDATE_TASK_COUNT); HashMap> updatePosByParentPos = new HashMap<>(); for (DhSectionPos pos : parentUpdatePosList) @@ -217,7 +217,7 @@ public class NewFullDataFileHandler NewFullDataSource dataSource = this.get(childPos); this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); - ((NewFullDataSourceRepo) this.repo).setApplyToParent(childPos, false); + this.repo.setApplyToParent(childPos, false); } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 81652fa0f..78e373202 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -31,7 +31,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; @@ -233,7 +232,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl } else { - byte[] columnGenerationSteps = ((NewFullDataSourceRepo)this.repo).getColumnGenerationStepForPos(genPos); + byte[] columnGenerationSteps = this.repo.getColumnGenerationStepForPos(genPos); if (columnGenerationSteps == null) { // shouldn't happen, but just in case From fd1944eb26c44a3fc192a7a6073609a44a32a084 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:46:52 -0500 Subject: [PATCH 031/183] Remove unnecessary legacy debug rendering --- .../fullDatafile/LegacyFullDataFileHandler.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java index 95a1e06c4..ecac4899f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java @@ -41,7 +41,6 @@ import java.sql.SQLException; public class LegacyFullDataFileHandler extends AbstractLegacyDataSourceHandler - implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -55,8 +54,6 @@ public class LegacyFullDataFileHandler public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); - - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); } @@ -117,18 +114,4 @@ public class LegacyFullDataFileHandler } - - //===========// - // overrides // - //===========// - - @Override - public void debugRender(DebugRenderer renderer) - { - this.saveTimerTasksBySectionPos.keySet() - .forEach((pos) -> { renderer.renderBox(new DebugRenderer.Box(pos, -32f, 128f, 0.15f, Color.cyan)); }); - - } - - } From dd7d7733f0812ad606ecd9c4f9ea4ef0cf9d3435 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:47:27 -0500 Subject: [PATCH 032/183] Rename and remove debug wireframe configs --- .../seibel/distanthorizons/core/config/Config.java | 14 ++------------ .../file/fullDatafile/NewFullDataFileHandler.java | 2 +- .../core/generation/WorldGenerationQueue.java | 8 +++----- .../assets/distanthorizons/lang/en_us.json | 8 ++------ 4 files changed, 8 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index a3a39aefb..799c1a7c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -1247,19 +1247,9 @@ public class Config .comment("Render LOD section status?") .build(); - public static ConfigEntry showFullDataFileStatus = new ConfigEntry.Builder() + public static ConfigEntry showFullDataUpdateStatus = new ConfigEntry.Builder() .set(false) - .comment("Render full data file status?") - .build(); - - public static ConfigEntry showFullDataFileSampling = new ConfigEntry.Builder() - .set(false) - .comment("Render full data file sampling progress?") - .build(); - - public static ConfigEntry showRenderDataFileStatus = new ConfigEntry.Builder() - .set(false) - .comment("Render render data file status?") + .comment("Render full data update/lock status?") .build(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index ad3e708aa..913c94541 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -79,7 +79,7 @@ public class NewFullDataFileHandler { super(level, saveStructure, saveDirOverride); - DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataFileStatus); + DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); String dimensionName = level.getLevelWrapper().getDimensionType().getDimensionName(); this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Parent Update Queue ["+dimensionName+"]"); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index cd8c87f08..f1184a609 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -575,8 +575,6 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender LodUtil.assertTrue(this.generatorClosingFuture != null); - - LOGGER.info("Awaiting world generator thread pool termination..."); try { @@ -593,9 +591,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender } - this.generator.close(); - + DebugRenderer.unregister(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); try @@ -606,8 +603,9 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender { LOGGER.warn("Failed to close generation queue: ", e); } + + LOGGER.info("Finished closing " + WorldGenerationQueue.class.getSimpleName()); - DebugRenderer.unregister(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue); } diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index f581ea4aa..2f1867f4b 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -558,12 +558,8 @@ "Show World Gen Queue", "distanthorizons.config.client.advanced.debugging.debugWireframe.showRenderSectionStatus": "Show Render Section Status", - "distanthorizons.config.client.advanced.debugging.debugWireframe.showFullDataFileStatus": - "Show Full Data file Status", - "distanthorizons.config.client.advanced.debugging.debugWireframe.showFullDataFileSampling": - "Show Full Data file Sampling", - "distanthorizons.config.client.advanced.debugging.debugWireframe.showRenderDataFileStatus": - "Show Render Data file Status", + "distanthorizons.config.client.advanced.debugging.debugWireframe.showFullDataUpdateStatus": + "Show Full Data Update Status", "distanthorizons.config.client.advanced.debugging.openGl": From 34fdae1c78379195de4e76e6ac43bd17715516a8 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 10 Mar 2024 21:50:13 -0500 Subject: [PATCH 033/183] remove debug code in NewFullDataSource --- .../dataObjects/fullData/sources/NewFullDataSource.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 3510d789b..dbbb42834 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -312,8 +312,6 @@ public class NewFullDataSource implements IDataSource return dataChanged; } - - private static long[] mergeInputTwoByTwoDataColumn(NewFullDataSource inputDataSource, int x, int z) { ArrayList newColumnList = new ArrayList<>(); @@ -329,12 +327,6 @@ public class NewFullDataSource implements IDataSource for (int blockY = 0; blockY < RenderDataPointUtil.MAX_WORLD_Y_SIZE; blockY++, height++) { - //if (x == 38 && z == 2 && blockY == 65+72) // nether portal - if (x == 32 && z == 0 && blockY == 138) // glowstone - { - int k = 0; // TODO remove, just for testing - } - // if each column has reached the end of their data, nothing more needs to be done if (currentDatapointIndex[0] == -1 && currentDatapointIndex[1] == -1 From 30076f1b607208d894daba803a9913d184a0f618 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 07:21:56 -0500 Subject: [PATCH 034/183] Minor comments and refactoring --- .../core/file/AbstractLegacyDataSourceHandler.java | 1 + .../core/file/AbstractNewDataSourceHandler.java | 9 ++++++--- .../core/file/fullDatafile/IFullDataSourceProvider.java | 4 ++-- .../seibel/distanthorizons/core/sql/DatabaseUpdater.java | 1 + .../core/sql/repo/AbstractLegacyDataSourceRepo.java | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 80e7ae3fe..b24598f34 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -31,6 +31,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; +@Deprecated public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> implements ISourceProvider { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index a2b1adf49..e516ac628 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -125,7 +125,7 @@ public abstract class AbstractNewDataSourceHandler } /** * Should only be used in internal file handler methods where we are already running on a file handler thread. - * Can return null if there was a problem. + * Shouldn't return null. * @see AbstractNewDataSourceHandler#getAsync(DhSectionPos) */ public TDataSource get(DhSectionPos pos) @@ -136,7 +136,7 @@ public abstract class AbstractNewDataSourceHandler TDTO dto = this.repo.getByKey(pos); if (dto != null) { - // load from file + // load from database dataSource = this.createDataSourceFromDto(dto); } else @@ -198,7 +198,10 @@ public abstract class AbstractNewDataSourceHandler } } - /** @param updatePos the position to update */ + /** + * After this method returns the inputData will be written to file. + * @param updatePos the position to update + */ protected void updateDataSourceAtPos(DhSectionPos updatePos, NewFullDataSource inputData, boolean lockOnUpdatePos) { boolean methodLocked = false; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index f625f50b0..cf1a7bdbb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataS import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; +import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; @@ -31,7 +31,7 @@ import java.util.concurrent.CompletableFuture; /** * Handles reading, writing, and updating {@link NewFullDataSource}'s.
- * Should be backed by a database handled by a {@link FullDataRepo}. + * Should be backed by a database handled by a {@link LegacyFullDataRepo}. */ public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index 2c4c56844..c6ce1ee99 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -112,6 +112,7 @@ public class DatabaseUpdater statement.setQueryTimeout(AbstractDhRepo.TIMEOUT_SECONDS); + // TODO this should be a complete transaction, but doesn't appear to be int[] numberOfRowsModifiedArray = statement.executeBatch(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java index bc979094c..32d67d19b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java @@ -144,7 +144,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo resultMap = this.queryDictionaryFirst("select MAX(DataDetailLevel) as maxDetailLevel from DhFullData;"); + Map resultMap = this.queryDictionaryFirst("select MAX(DataDetailLevel) as maxDetailLevel from "+this.getTableName()+";"); int maxDetailLevel; if (resultMap == null || resultMap.get("maxDetailLevel") == null) { From c5787d0ff22db13bb7fd3e1a32ffb6f405326f44 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 20:24:57 -0500 Subject: [PATCH 035/183] Add Legacy data source migration --- .../fullData/sources/NewFullDataSource.java | 111 ++++++++++++++++-- .../LegacyFullDataFileHandler.java | 47 +++++--- .../fullDatafile/NewFullDataFileHandler.java | 96 ++++++++++++++- .../core/sql/repo/FullDataRepo.java | 43 ------- .../core/sql/repo/LegacyFullDataRepo.java | 91 ++++++++++++++ .../distanthorizons/core/util/ThreadUtil.java | 4 +- ...te-createGeneratedFullDataSourceTables.sql | 13 +- 7 files changed, 331 insertions(+), 74 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataRepo.java create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index dbbb42834..43b83cbf3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -25,6 +25,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.generation.DhLightingEngine; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -32,6 +33,7 @@ import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; +import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import org.apache.logging.log4j.Logger; @@ -54,6 +56,7 @@ public class NewFullDataSource implements IDataSource private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ private static final boolean RUN_UPDATE_DEV_VALIDATION = false; //ModInfo.IS_DEV_BUILD; + private static final boolean RUN_V1_MIGRATION_VALIDATION = false; /** measured in data columns */ public static final int WIDTH = 64; @@ -128,25 +131,117 @@ public class NewFullDataSource implements IDataSource public static NewFullDataSource createFromCompleteDataSource(CompleteFullDataSource legacyData) { + if (CompleteFullDataSource.WIDTH != WIDTH) + { + throw new UnsupportedOperationException( + "Unable to convert CompleteFullDataSource into NewFullDataSource. " + + "Data sources have different data point widths and no converter is present. " + + "CompleteFullDataSource width ["+CompleteFullDataSource.WIDTH+"], NewFullDataSource width ["+WIDTH+"]."); + } + + + // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; long[][] dataPoints = new long[WIDTH * WIDTH][]; for (int x = 0; x < WIDTH; x++) { for (int z = 0; z < WIDTH; z++) { - int index = relativePosToIndex(x, z); - SingleColumnFullDataAccessor accessor = legacyData.get(x, z); - if (accessor.doesColumnExist()) { + int index = relativePosToIndex(x, z); dataPoints[index] = accessor.getRaw(); - columnGenerationSteps[index] = legacyData.getWorldGenStep().value; + + // reverse the array so index 0 is the lowest, + // this is necessary for later logic + // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java + long[] dataColumn = dataPoints[index]; + for(int i = 0; i < dataColumn.length / 2; i++) + { + long temp = dataColumn[i]; + dataColumn[i] = dataColumn[dataColumn.length - i - 1]; + dataColumn[dataColumn.length - i - 1] = temp; + } + + + // convert the data point format + boolean columnHasNonAirBlock = false; + for (int i = 0; i < dataColumn.length; i++) + { + long dataPoint = dataColumn[i]; + + int id = FullDataPointUtil.getId(dataPoint); + int height = FullDataPointUtil.getHeight(dataPoint); + int bottomY = FullDataPointUtil.getBottomY(dataPoint); + byte blockLight = (byte) FullDataPointUtil.getBlockLight(dataPoint); + byte skyLight = (byte) FullDataPointUtil.getSkyLight(dataPoint); + + long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, skyLight, blockLight); + dataColumn[i] = newDataPoint; + + + // check if this datapoint is air + if (!columnHasNonAirBlock) + { + IBlockStateWrapper blockState = legacyData.getMapping().getBlockStateWrapper(id); + if (!blockState.isAir()) + { + columnHasNonAirBlock = true; + } + } + } + + // the old data sources didn't have a generation step written down + // if the column has any data points, assume it's fully generated, otherwise assume it's empty + columnGenerationSteps[index] = (columnHasNonAirBlock ? EDhApiWorldGenerationStep.LIGHT.value : EDhApiWorldGenerationStep.EMPTY.value); } } } - return NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps); + NewFullDataSource newFullDataSource = NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps); + + + // should only be used if debugging, this is a very expensive operation + if (RUN_V1_MIGRATION_VALIDATION) + { + for (int x = 0; x < WIDTH; x++) + { + for (int z = 0; z < WIDTH; z++) + { + SingleColumnFullDataAccessor legacyAccessor = legacyData.get(x, z); + if (legacyAccessor.doesColumnExist()) + { + SingleColumnFullDataAccessor newAccessor = newFullDataSource.get(x, z); + + if (newAccessor == null) + { + LodUtil.assertNotReach("Accessor column mismatch"); + } + else if (legacyAccessor.getRaw().length != newAccessor.getRaw().length) + { + LodUtil.assertNotReach("Accessor column length mismatch"); + } + else + { + long[] legacyRaw = legacyAccessor.getRaw(); + long[] newRaw = newAccessor.getRaw(); + + for (int i = 0; i < legacyRaw.length; i++) + { + if (legacyRaw[i] != newRaw[i]) + { + LodUtil.assertNotReach("Data mismatch"); + } + } + } + + } + } + } + } + + return newFullDataSource; } @@ -286,7 +381,8 @@ public class NewFullDataSource implements IDataSource for (int z = 0; z < WIDTH; z += 2) { long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); - byte inputGenStep = inputDataSource.columnGenerationSteps[0]; // TODO + // TODO + byte inputGenStep = inputDataSource.columnGenerationSteps[0]; int recipientX = (x / 2) + recipientOffsetX; @@ -349,6 +445,7 @@ public class NewFullDataSource implements IDataSource { for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) { + // TODO throw an assertion if the column isn't in order or just fix it... long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; if (inputDataArray == null || inputDataArray.length == 0) { @@ -556,7 +653,7 @@ public class NewFullDataSource implements IDataSource // helper methods // //================// - // TODO make private, any external logic should go through a method, not interact with the arrays directly + // TODO make private, any external logic should go through a method, not interact with the arrays directly public static int relativePosToIndex(int relX, int relZ) throws IndexOutOfBoundsException { if (relX < 0 || relZ < 0 || diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java index ecac4899f..12c9b6800 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java @@ -19,25 +19,22 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.sql.repo.FullDataRepo; +import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo; import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; -import java.awt.*; import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.ArrayList; public class LegacyFullDataFileHandler extends AbstractLegacyDataSourceHandler @@ -50,7 +47,6 @@ public class LegacyFullDataFileHandler // constructor // //=============// - public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); @@ -67,7 +63,7 @@ public class LegacyFullDataFileHandler { try { - return new FullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + return new LegacyFullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); } catch (SQLException e) { @@ -87,18 +83,12 @@ public class LegacyFullDataFileHandler /** Creates a new data source using any DTOs already present in the database. */ @Deprecated @Override - protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) - { - throw new UnsupportedOperationException("Deprecated"); - } + protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { return null; } @Deprecated @Override - protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) - { - throw new UnsupportedOperationException("Deprecated"); - } + protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { return null; } @@ -109,8 +99,33 @@ public class LegacyFullDataFileHandler @Deprecated @Override public void writeDataSourceToFile(CompleteFullDataSource fullDataSource) throws IOException + { throw new UnsupportedOperationException("Deprecated"); } + + + + //===========// + // migration // + //===========// + + public int getDataSourceMigrationCount() + { return ((LegacyFullDataRepo) this.repo).getMigrationCount(); } + + public ArrayList getDataSourcesToMigrate(int limit) { - throw new UnsupportedOperationException("Deprecated"); + ArrayList dataSourceList = new ArrayList<>(); + + ArrayList migrationPosList = ((LegacyFullDataRepo) this.repo).getPositionsToMigrate(limit); + for (int i = 0; i < migrationPosList.size(); i++) + { + DhSectionPos pos = migrationPosList.get(i); + CompleteFullDataSource dataSource = this.get(pos); + if (dataSource != null) + { + dataSourceList.add(dataSource); + } + } + + return dataSourceList; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 913c94541..98d5dc876 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; @@ -29,7 +30,6 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.AbstractDhRepo; import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; @@ -42,7 +42,9 @@ import java.io.IOException; import java.sql.SQLException; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; public class NewFullDataFileHandler @@ -51,6 +53,10 @@ public class NewFullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + /** how many data sources should be pulled down for migration at once */ + private static final int MIGRATION_BATCH_COUNT = 20; + private static final String MIGRATION_THREAD_NAME_PREFIX = "Full Data Migration Thread: "; + protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; /** how many parent update tasks can be in the queue at once */ protected static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); @@ -59,6 +65,18 @@ public class NewFullDataFileHandler protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250; + protected final ThreadPoolExecutor migrationThreadPool; + /** + * Interrupting the migration thread pool doesn't work well and may corrupt the database + * vs gracefully shutting down the thread ourselves. + */ + protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); + protected final LegacyFullDataFileHandler legacyFileHandler; + + /** + * Tracks which positions are currently being updated + * to prevent duplicate concurrent updates. + */ public final Set parentUpdatingPosSet = ConcurrentHashMap.newKeySet(); // TODO only run thread if modifications happened recently @@ -78,10 +96,18 @@ public class NewFullDataFileHandler public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); + this.legacyFileHandler = new LegacyFullDataFileHandler(level, saveStructure, saveDirOverride); DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); String dimensionName = level.getLevelWrapper().getDimensionType().getDimensionName(); + + // start migrating any legacy data sources present in the background + int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); + LOGGER.info("Found ["+totalCount+"] data sources that need migration."); + this.migrationThreadPool = ThreadUtil.makeRateLimitedThreadPool(1, MIGRATION_THREAD_NAME_PREFIX +"["+dimensionName+"]", Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads.get(), Thread.MIN_PRIORITY, (Semaphore)null); + this.migrationThreadPool.execute(() -> this.convertLegacyDataSources()); + this.updateQueueProcessor = ThreadUtil.makeSingleThreadPool("Parent Update Queue ["+dimensionName+"]"); this.updateQueueProcessor.execute(() -> this.runUpdateQueue()); } @@ -156,13 +182,14 @@ public class NewFullDataFileHandler - // only add more items to the queue if half or more of the previous tasks have been completed + // queue parent updates if (executor.getQueue().size() < MAX_UPDATE_TASK_COUNT && this.parentUpdatingPosSet.size() < MAX_UPDATE_TASK_COUNT) { // get the positions that need to be applied to their parents ArrayList parentUpdatePosList = this.repo.getPositionsToUpdate(MAX_UPDATE_TASK_COUNT); + // combine updates together based on their parent HashMap> updatePosByParentPos = new HashMap<>(); for (DhSectionPos pos : parentUpdatePosList) { @@ -177,9 +204,7 @@ public class NewFullDataFileHandler }); } - - - // queue each update + // queue the updates for (DhSectionPos parentUpdatePos : updatePosByParentPos.keySet()) { // stop if there are already a bunch of updates queued @@ -250,6 +275,7 @@ public class NewFullDataFileHandler } } } + } catch (InterruptedException ignored) { Thread.currentThread().interrupt(); } catch (Exception e) @@ -263,6 +289,63 @@ public class NewFullDataFileHandler + //=======================// + // data source migration // + //=======================// + + private void convertLegacyDataSources() + { + String dimensionName = this.level.getLevelWrapper().getDimensionType().getDimensionName(); + LOGGER.info("Attempting to migrate data sources for: ["+dimensionName+"]-["+this.saveDir+"]..."); + + int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); + LOGGER.info("Found ["+totalCount+"] data sources that need migration."); + + + ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + if (!legacyDataSourceList.isEmpty()) + { + // keep going until every data source has been migrated + int progressCount = 0; + while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) + { + LOGGER.info("Migrating ["+dimensionName+"] - [" + progressCount + "/" + totalCount + "]..."); + + for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) + { + // convert the legacy data source to the new format + CompleteFullDataSource legacyDataSource = legacyDataSourceList.get(i); + NewFullDataSource newDataSource = NewFullDataSource.createFromCompleteDataSource(legacyDataSource); + newDataSource.applyToParent = true; + + this.updateDataSourceAtPos(newDataSource.getSectionPos(), newDataSource, true); + + // the legacy data source can now be deleted + this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos()); + } + + legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + progressCount += legacyDataSourceList.size(); + } + + + if (this.migrationThreadRunning.get()) + { + LOGGER.info("migration complete for: ["+dimensionName+"]-["+this.saveDir+"]."); + } + else + { + LOGGER.info("migration stopped for: ["+dimensionName+"]-["+this.saveDir+"]."); + } + } + else + { + LOGGER.info("No migration necessary."); + } + } + + + //===========// // overrides // //===========// @@ -284,6 +367,9 @@ public class NewFullDataFileHandler { super.close(); this.updateQueueProcessor.shutdownNow(); + + this.migrationThreadRunning.set(false); + this.migrationThreadPool.shutdown(); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataRepo.java deleted file mode 100644 index 97932ed5e..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataRepo.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.sql.repo; - -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.sql.SQLException; - -public class FullDataRepo extends AbstractLegacyDataSourceRepo -{ - public static final String TABLE_NAME = "DhFullData"; - - - public FullDataRepo(String databaseType, String databaseLocation) throws SQLException - { - super(databaseType, databaseLocation); - } - - - @Override - public String getTableName() { return TABLE_NAME; } - - @Override - public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java new file mode 100644 index 000000000..4715edd09 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java @@ -0,0 +1,91 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.sql.repo; + +import com.seibel.distanthorizons.core.pos.DhSectionPos; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo +{ + public static final String TABLE_NAME = "Legacy_FullData_V1"; + + + public LegacyFullDataRepo(String databaseType, String databaseLocation) throws SQLException + { + super(databaseType, databaseLocation); + } + + + @Override + public String getTableName() { return TABLE_NAME; } + + @Override + public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } + + + + //===========// + // migration // + //===========// + + /** Returns how many positions need to be migrated over to the new version */ + public int getMigrationCount() + { + Map resultMap = this.queryDictionaryFirst( + "select COUNT(*) as itemCount from "+this.getTableName()); + + if (resultMap == null) + { + return 0; + } + else + { + int count = (int) resultMap.get("itemCount"); + return count; + } + } + + /** Returns the new "returnCount" positions that need to be migrated */ + public ArrayList getPositionsToMigrate(int returnCount) + { + ArrayList list = new ArrayList<>(); + + List> resultMapList = this.queryDictionary( + "select DhSectionPos " + + "from "+this.getTableName()+" " + + "LIMIT "+returnCount+";"); + + for (Map resultMap : resultMapList) + { + // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] + DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos")); + list.add(sectionPos); + } + + return list; + } + + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java index 792b2dc9f..7c5339d7c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ThreadUtil.java @@ -80,9 +80,9 @@ public class ThreadUtil /** should only be used if there isn't a config controlling the run time ratio of this thread pool */ - public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, Double runTimeRatio, int relativePriority, Semaphore activeThreadCountSemaphore) + public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, String name, Double runTimeRatio, int threadPriority, Semaphore activeThreadCountSemaphore) { - return new RateLimitedThreadPoolExecutor(poolSize, runTimeRatio, new DhThreadFactory(name, Thread.NORM_PRIORITY + relativePriority), activeThreadCountSemaphore); + return new RateLimitedThreadPoolExecutor(poolSize, runTimeRatio, new DhThreadFactory(name, threadPriority), activeThreadCountSemaphore); } public static RateLimitedThreadPoolExecutor makeRateLimitedThreadPool(int poolSize, Double runTimeRatio, DhThreadFactory threadFactory, Semaphore activeThreadCountSemaphore) { diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql index d0d833358..d0b3f5b10 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -1,5 +1,16 @@ -select * from DhRenderData; -- here to prevent crashing when running the first batch +ALTER TABLE `DhFullData` RENAME TO `Legacy_FullData_V1`; + +--batch-- + +-- we only want to convert the level 0 LOD data, the rest can be generated later +delete from Legacy_FullData_V1 +where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0; + +--batch-- + +-- shrink the database file for the removed legacy detail levels +VACUUM; --batch-- From e83f7bd62ee39ceddbadf2c6013d39c7e604eb60 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 21:00:50 -0500 Subject: [PATCH 036/183] Add FullDataPointUtilV1 for use with CompleteFullDataSource --- .../fullData/sources/NewFullDataSource.java | 14 +-- .../core/util/FullDataPointUtil.java | 8 +- .../core/util/FullDataPointUtilV1.java | 108 ++++++++++++++++++ 3 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 43b83cbf3..53091e49c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -25,11 +25,11 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColum import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; -import com.seibel.distanthorizons.core.generation.DhLightingEngine; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; @@ -171,13 +171,13 @@ public class NewFullDataSource implements IDataSource { long dataPoint = dataColumn[i]; - int id = FullDataPointUtil.getId(dataPoint); - int height = FullDataPointUtil.getHeight(dataPoint); - int bottomY = FullDataPointUtil.getBottomY(dataPoint); - byte blockLight = (byte) FullDataPointUtil.getBlockLight(dataPoint); - byte skyLight = (byte) FullDataPointUtil.getSkyLight(dataPoint); + int id = FullDataPointUtilV1.getId(dataPoint); + int height = FullDataPointUtilV1.getHeight(dataPoint); + int bottomY = FullDataPointUtilV1.getBottomY(dataPoint); + byte blockLight = (byte) FullDataPointUtilV1.getBlockLight(dataPoint); + byte skyLight = (byte) FullDataPointUtilV1.getSkyLight(dataPoint); - long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, skyLight, blockLight); + long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, blockLight, skyLight); dataColumn[i] = newDataPoint; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index 9862120e6..1e1ebfc49 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -25,9 +25,7 @@ import org.jetbrains.annotations.Contract; * A helper class that is used to access the data from a long * formatted as a full data point.
* A full data point contains the most information and is the - * base truth used when creating render data.

- * - * To access data from a long formatted as a render data point see: {@link RenderDataPointUtil}

+ * source of truth used when creating render data.

* * DataPoint Format:
* @@ -47,10 +45,10 @@ import org.jetbrains.annotations.Contract; * ID ID ID ID ID ID ID ID
* ID ID ID ID ID ID ID ID <-- Bottom bits
* - * + * * @see RenderDataPointUtil + * @see FullDataPointUtilV1 */ -// TODO make a legacy version of this specifically for CompleteFullDataSource just in case things change in the future public class FullDataPointUtil { /** Represents the data held by an empty data point */ diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java new file mode 100644 index 000000000..aa40ee8a3 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java @@ -0,0 +1,108 @@ +package com.seibel.distanthorizons.core.util; + +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; + +/** + * Only for Legacy support
+ * Used by DH versions 2.0.0 and 2.0.1.

+ * + * Specifically used by the data sources:
+ * - {@link CompleteFullDataSource} aka CompleteFullDataSource
+ * - (Deleted) HighDetailIncompleteFullDataSource
+ * - (Deleted) LowDetailIncompleteFullDataSource

+ * + * DataPoint Format:
+ * + * ID: blockState id
+ * MY: Min Y Height (unsigned, relative to the minimum level height)
+ * HI: Height (how tall this data point is in blocks)
+ * BL: Block light
+ * SL: Sky light

+ * + * =======Bit layout=======
+ * SL SL SL SL BL BL BL BL <-- Top bits
+ * MY MY MY MY MY MY MY MY
+ * MY MY MY MY HI HI HI HI
+ * HI HI HI HI HI HI HI HI
+ * ID ID ID ID ID ID ID ID
+ * ID ID ID ID ID ID ID ID
+ * ID ID ID ID ID ID ID ID
+ * ID ID ID ID ID ID ID ID <-- Bottom bits
+ * + * + * @see CompleteFullDataSource + * @see FullDataPointUtil + */ +public class FullDataPointUtilV1 +{ + /** Represents the data held by an empty data point */ + public static final int EMPTY_DATA_POINT = 0; + + public static final int ID_WIDTH = 32; + public static final int HEIGHT_WIDTH = 12; + public static final int MIN_Y_WIDTH = 12; + public static final int SKY_LIGHT_WIDTH = 4; + public static final int BLOCK_LIGHT_WIDTH = 4; + + public static final int ID_OFFSET = 0; + public static final int HEIGHT_OFFSET = ID_OFFSET + ID_WIDTH; + /** indicates the Y position where the LOD starts relative to the level's minimum height */ + public static final int MIN_Y_OFFSET = HEIGHT_OFFSET + HEIGHT_WIDTH; + public static final int SKY_LIGHT_OFFSET = MIN_Y_OFFSET + MIN_Y_WIDTH; + public static final int BLOCK_LIGHT_OFFSET = SKY_LIGHT_OFFSET + SKY_LIGHT_WIDTH; + + + public static final long ID_MASK = Integer.MAX_VALUE; + public static final long INVERSE_ID_MASK = ~ID_MASK; + public static final int HEIGHT_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1; + public static final int MIN_Y_MASK = (int) Math.pow(2, MIN_Y_WIDTH) - 1; + public static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1; + public static final int BLOCK_LIGHT_MASK = (int) Math.pow(2, BLOCK_LIGHT_WIDTH) - 1; + + + /** + * creates a new datapoint with the given values + * @param relMinY relative to the minimum level Y value + * + * @deprecated Should not be used anymore, just here as a reference for how the data points were constructed. + */ + @Deprecated + public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) + { + LodUtil.assertTrue(relMinY >= 0 && relMinY < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y["+relMinY+"] out of range!"); + LodUtil.assertTrue(height > 0 && height < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with height["+height+"] out of range!"); + LodUtil.assertTrue(relMinY + height <= RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y+depth["+(relMinY+height)+"] out of range!"); + + long data = 0; + data |= id & ID_MASK; + data |= (long) (height & HEIGHT_MASK) << HEIGHT_OFFSET; + data |= (long) (relMinY & MIN_Y_MASK) << MIN_Y_OFFSET; + data |= (long) blockLight << BLOCK_LIGHT_OFFSET; + data |= (long) skyLight << SKY_LIGHT_OFFSET; + + LodUtil.assertTrue(getId(data) == id && getHeight(data) == height && getBottomY(data) == relMinY && getBlockLight(data) == Byte.toUnsignedInt(blockLight) && getSkyLight(data) == Byte.toUnsignedInt(skyLight), + "Trying to create datapoint with " + + "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + + "but got " + + "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); + + return data; + } + + + /** Returns the BlockState/Biome pair ID used to identify this LOD's color */ + public static int getId(long data) { return (int) (data & ID_MASK); } + /** Returns how many blocks tall this LOD is. */ + public static int getHeight(long data) { return (int) ((data >> HEIGHT_OFFSET) & HEIGHT_MASK); } + /** + * Returns the unsigned block position of the bottom vertices for this LOD relative to the level's minimum height. + * Should be between 0 and {@link RenderDataPointUtil#MAX_WORLD_Y_SIZE} + */ + public static int getBottomY(long data) { return (int) ((data >> MIN_Y_OFFSET) & MIN_Y_MASK); } + public static int getBlockLight(long data) { return (int) ((data >> BLOCK_LIGHT_OFFSET) & BLOCK_LIGHT_MASK); } + public static int getSkyLight(long data) { return (int) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } + + + public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",BlockLight:" + getBlockLight(data) + ",SkyLight:" + getSkyLight(data) + "]"; } + +} From 968bc9addc54ef5e13faa6cd30be9c008dc0e4fe Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 21:05:45 -0500 Subject: [PATCH 037/183] Fix incorrect parent updating lock logic --- .../core/file/fullDatafile/NewFullDataFileHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 98d5dc876..7a0c772b8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -209,7 +209,7 @@ public class NewFullDataFileHandler { // stop if there are already a bunch of updates queued if (this.parentUpdatingPosSet.size() > MAX_UPDATE_TASK_COUNT - && this.parentUpdatingPosSet.add(parentUpdatePos)) + || !this.parentUpdatingPosSet.add(parentUpdatePos)) { break; } From 024176f97c929849cf4644447048f8f1f00a914c Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 21:15:15 -0500 Subject: [PATCH 038/183] Fix migrated downsampling light looking weird --- .../fullData/sources/NewFullDataSource.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 53091e49c..588c0776b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -177,18 +177,21 @@ public class NewFullDataSource implements IDataSource byte blockLight = (byte) FullDataPointUtilV1.getBlockLight(dataPoint); byte skyLight = (byte) FullDataPointUtilV1.getSkyLight(dataPoint); + IBlockStateWrapper blockState = legacyData.getMapping().getBlockStateWrapper(id); + if (blockState.isAir()) + { + // air shouldn't have any light, otherwise down sampling will look weird + blockLight = 0; + } + long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, blockLight, skyLight); dataColumn[i] = newDataPoint; // check if this datapoint is air - if (!columnHasNonAirBlock) + if (!columnHasNonAirBlock && !blockState.isAir()) { - IBlockStateWrapper blockState = legacyData.getMapping().getBlockStateWrapper(id); - if (!blockState.isAir()) - { - columnHasNonAirBlock = true; - } + columnHasNonAirBlock = true; } } From 6f931c66bf8bb4c78bd3a53de83230ea8ecb76bc Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 21:42:53 -0500 Subject: [PATCH 039/183] Add migration fail capture --- .../LegacyFullDataFileHandler.java | 5 ++-- .../fullDatafile/NewFullDataFileHandler.java | 26 ++++++++++++++----- .../core/sql/repo/LegacyFullDataRepo.java | 13 +++++++++- ...te-createGeneratedFullDataSourceTables.sql | 6 ++++- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java index 12c9b6800..c95eeade1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java @@ -107,8 +107,7 @@ public class LegacyFullDataFileHandler // migration // //===========// - public int getDataSourceMigrationCount() - { return ((LegacyFullDataRepo) this.repo).getMigrationCount(); } + public int getDataSourceMigrationCount() { return ((LegacyFullDataRepo) this.repo).getMigrationCount(); } public ArrayList getDataSourcesToMigrate(int limit) { @@ -128,5 +127,7 @@ public class LegacyFullDataFileHandler return dataSourceList; } + public void markMigrationFailed(DhSectionPos pos) { ((LegacyFullDataRepo) this.repo).markMigrationFailed(pos); } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 7a0c772b8..e7d6134f8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -313,15 +313,25 @@ public class NewFullDataFileHandler for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) { - // convert the legacy data source to the new format CompleteFullDataSource legacyDataSource = legacyDataSourceList.get(i); - NewFullDataSource newDataSource = NewFullDataSource.createFromCompleteDataSource(legacyDataSource); - newDataSource.applyToParent = true; - this.updateDataSourceAtPos(newDataSource.getSectionPos(), newDataSource, true); - - // the legacy data source can now be deleted - this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos()); + try + { + // convert the legacy data source to the new format + NewFullDataSource newDataSource = NewFullDataSource.createFromCompleteDataSource(legacyDataSource); + newDataSource.applyToParent = true; + + this.updateDataSourceAtPos(newDataSource.getSectionPos(), newDataSource, true); + + // the legacy data source can now be deleted + this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos()); + } + catch (Exception e) + { + DhSectionPos migrationPos = legacyDataSource.getSectionPos(); + LOGGER.error("Unexpected issue migrating data source at pos "+migrationPos+". Error: "+e.getMessage(), e); + this.legacyFileHandler.markMigrationFailed(migrationPos); + } } legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); @@ -342,6 +352,8 @@ public class NewFullDataFileHandler { LOGGER.info("No migration necessary."); } + + this.migrationThreadRunning.set(false); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java index 4715edd09..2d191da4e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java @@ -53,7 +53,7 @@ public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo public int getMigrationCount() { Map resultMap = this.queryDictionaryFirst( - "select COUNT(*) as itemCount from "+this.getTableName()); + "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); if (resultMap == null) { @@ -74,6 +74,7 @@ public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo List> resultMapList = this.queryDictionary( "select DhSectionPos " + "from "+this.getTableName()+" " + + "WHERE MigrationFailed <> 1 " + "LIMIT "+returnCount+";"); for (Map resultMap : resultMapList) @@ -86,6 +87,16 @@ public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo return list; } + public void markMigrationFailed(DhSectionPos pos) + { + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET MigrationFailed = 1 \n" + + "WHERE DhSectionPos = '"+pos.serialize()+"'"; + + this.queryDictionaryFirst(sql); + } + } diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql index d0b3f5b10..34986c361 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -1,5 +1,9 @@ -ALTER TABLE `DhFullData` RENAME TO `Legacy_FullData_V1`; +ALTER TABLE DhFullData RENAME TO Legacy_FullData_V1; + +--batch-- + +ALTER TABLE Legacy_FullData_V1 ADD COLUMN MigrationFailed BIT NOT NULL DEFAULT 0; --batch-- From 996621887c9ccf62d753f8432d8b519fcd249de1 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 21:43:26 -0500 Subject: [PATCH 040/183] Prevent world gen while migration is running --- .../core/file/fullDatafile/NewFullDataFileHandler.java | 10 ++++++++++ .../fullDatafile/NewGeneratedFullDataFileHandler.java | 6 ++++++ .../core/generation/IWorldGenerationQueue.java | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index e7d6134f8..2203bb990 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -160,6 +160,16 @@ public class NewFullDataFileHandler @Override protected NewFullDataSource makeEmptyDataSource(DhSectionPos pos) { return NewFullDataSource.createEmpty(pos); } + @Override + public boolean canQueueRetrieval() + { + // Retrieval shouldn't happen while an unknown number of + // legacy data sources are present. + // If retrieval was allowed we might run into concurrency issues. + return !this.migrationThreadRunning.get(); + } + + //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java index 78e373202..2f08a75bd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java @@ -173,6 +173,12 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl @Override public boolean canQueueRetrieval() { + if (!super.canQueueRetrieval()) + { + return false; + } + + IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue == null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java index cdea7be81..a7a834929 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java @@ -28,7 +28,7 @@ import java.io.Closeable; import java.util.concurrent.CompletableFuture; import java.util.function.Function; -// TODO does this need a interface? +// TODO This doesn't need an interface, remove the interface @Deprecated public interface IWorldGenerationQueue extends Closeable { From c3f99835dbdfdb1a2e3abe19c90b428a6106d059 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 12 Mar 2024 22:00:36 -0500 Subject: [PATCH 041/183] Multithread full data migration --- .../fullDatafile/NewFullDataFileHandler.java | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 2203bb990..a5f36c2d5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -41,9 +41,7 @@ import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; @@ -53,10 +51,6 @@ public class NewFullDataFileHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - /** how many data sources should be pulled down for migration at once */ - private static final int MIGRATION_BATCH_COUNT = 20; - private static final String MIGRATION_THREAD_NAME_PREFIX = "Full Data Migration Thread: "; - protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; /** how many parent update tasks can be in the queue at once */ protected static final int MAX_UPDATE_TASK_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads.get(); @@ -64,6 +58,12 @@ public class NewFullDataFileHandler /** indicates how long the update queue thread should wait between queuing ticks */ protected static final int UPDATE_QUEUE_THREAD_DELAY_IN_MS = 250; + /** how many data sources should be pulled down for migration at once */ + private static final int MIGRATION_BATCH_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD; + private static final String MIGRATION_THREAD_NAME_PREFIX = "Full Data Migration Thread: "; + /** 1 minute */ + private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 60 * 1_000; + protected final ThreadPoolExecutor migrationThreadPool; /** @@ -321,20 +321,27 @@ public class NewFullDataFileHandler { LOGGER.info("Migrating ["+dimensionName+"] - [" + progressCount + "/" + totalCount + "]..."); + ArrayList> updateFutureList = new ArrayList<>(); for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) { CompleteFullDataSource legacyDataSource = legacyDataSourceList.get(i); try { - // convert the legacy data source to the new format + // convert the legacy data source to the new format, + // this is a relatively cheap operation NewFullDataSource newDataSource = NewFullDataSource.createFromCompleteDataSource(legacyDataSource); newDataSource.applyToParent = true; - this.updateDataSourceAtPos(newDataSource.getSectionPos(), newDataSource, true); - - // the legacy data source can now be deleted - this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos()); + // the actual update process can be moderately expensive due to having to update + // the render data along with the full data, so running it async on the update threads gains us a good bit of speed + CompletableFuture future = this.updateDataSourceAsync(newDataSource); + updateFutureList.add(future); + future.thenRun(() -> + { + // after the update finishes the legacy data source can be safely deleted + this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos()); + }); } catch (Exception e) { @@ -344,6 +351,18 @@ public class NewFullDataFileHandler } } + + try + { + // wait for each thread to finish updating + CompletableFuture combinedFutures = CompletableFuture.allOf(updateFutureList.toArray(new CompletableFuture[0])); + combinedFutures.get(MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS); + } + catch (InterruptedException | ExecutionException | TimeoutException e) + { + LOGGER.warn("Migration update timed out. Migration will re-try the same positions again. Error:"+e.getMessage(), e); + } + legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); progressCount += legacyDataSourceList.size(); } From 4741e25349470f634a1f2d6c47b0c38029cce01f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 14 Mar 2024 21:35:52 -0500 Subject: [PATCH 042/183] Add NewFullDataSourceDTO.createUnitTestDataSource() --- .../core/sql/dto/NewFullDataSourceDTO.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java index cba8a4d7f..a9c58e6ae 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java @@ -111,7 +111,18 @@ public class NewFullDataSourceDTO implements IBaseDTO public NewFullDataSource createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { return this.populateDataSource(NewFullDataSource.createEmpty(this.pos), levelWrapper); } - public NewFullDataSource populateDataSource(NewFullDataSource dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + + public NewFullDataSource populateDataSource(NewFullDataSource dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } + + /** + * May be missing one or more data fields.
+ * Designed to be used without access to Minecraft or any supporting objects. + */ + public NewFullDataSource createUnitTestDataSource() throws IOException, InterruptedException + { return this.internalPopulateDataSource(NewFullDataSource.createEmpty(this.pos), null, true); } + + private NewFullDataSource internalPopulateDataSource(NewFullDataSource dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException { if (NewFullDataSource.DATA_FORMAT_VERSION != this.dataFormatVersion) { @@ -122,7 +133,16 @@ public class NewFullDataSourceDTO implements IBaseDTO dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray); dataSource.getMapping().clear(dataSource.getSectionPos()); - dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper)); + // should only be null when used in a unit test + if (!unitTest) + { + if (levelWrapper == null) + { + throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); + } + + dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper)); + } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; dataSource.createdUnixDateTime = this.createdUnixDateTime; @@ -160,7 +180,7 @@ public class NewFullDataSourceDTO implements IBaseDTO // write column length int columnLength = (dataColumn != null) ? dataColumn.length : 0; - compressedOut.writeInt(columnLength); + compressedOut.writeInt(columnLength); // TODO // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) From d5074feda2c6a54c7a3f9e34dacd6024758a9075 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 14 Mar 2024 21:40:36 -0500 Subject: [PATCH 043/183] Add aggregate getters to NewFullDataSourceRepo for unit testing --- .../core/sql/repo/NewFullDataSourceRepo.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java index f5d27003c..dcd37a9b4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java @@ -231,4 +231,60 @@ public class NewFullDataSourceRepo extends AbstractDhRepo getAllPositions() + { + ArrayList list = new ArrayList<>(); + + List> resultMapList = this.queryDictionary( + "select DetailLevel, PosX, PosZ " + + "from "+this.getTableName()+"; "); + + for (Map resultMap : resultMapList) + { + byte detailLevel = (Byte) resultMap.get("DetailLevel"); + byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); + int posX = (Integer) resultMap.get("PosX"); + int posZ = (Integer) resultMap.get("PosZ"); + + DhSectionPos pos = new DhSectionPos(sectionDetailLevel, posX, posZ); + list.add(pos); + } + + return list; + } + + /** + * @return the size of the full data at the given position + * (doesn't include the size of the mapping or any other column) + */ + public int getDataSizeInBytes(DhSectionPos pos) + { + int detailLevel = pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; + + Map resultMap = this.queryDictionaryFirst( + "select LENGTH(Data) as dataSize " + + "from "+this.getTableName()+" " + + "WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ()); + + int dataLength = (resultMap != null) ? (int) resultMap.get("dataSize") : 0; + return dataLength; + } + + /** @return the total size in bytes of the full data for this entire database */ + public int getTotalDataSizeInBytes() + { + Map resultMap = this.queryDictionaryFirst( + "select SUM(LENGTH(Data)) as dataSize " + + "from "+this.getTableName()+"; "); + + int dataLength = (resultMap != null) ? (int) resultMap.get("dataSize") : 0; + return dataLength; + } + + } From 150d929a45e5cfcaee8dced46caab992718bec5a Mon Sep 17 00:00:00 2001 From: cola98765 Date: Fri, 15 Mar 2024 08:58:26 +0000 Subject: [PATCH 044/183] Update EVerticalQuality.java --- .../distanthorizons/api/enums/config/EVerticalQuality.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java index c20b86b70..6c50e9bbc 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java @@ -38,7 +38,9 @@ public enum EVerticalQuality LOW(new int[]{4, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1}), MEDIUM(new int[]{6, 4, 3, 2, 2, 1, 1, 1, 1, 1, 1}), HIGH(new int[]{8, 6, 4, 2, 2, 2, 2, 1, 1, 1, 1}), - EXTREME(new int[]{16, 8, 4, 2, 2, 2, 2, 1, 1, 1, 1}); + EXTREME(new int[]{16, 8, 4, 2, 2, 2, 2, 1, 1, 1, 1}), + WHAT(new int[]{64, 16, 8, 4, 2, 2, 2, 2, 1, 1, 1}), + WHY(new int[]{120, 16, 8, 4, 2, 2, 2, 2, 1, 1, 1}); /** represents how many LODs can be rendered in a single vertical slice */ public final int[] maxVerticalData; From 9e0046ba834b77c8abfa2558542a031cfff9e8df Mon Sep 17 00:00:00 2001 From: cola98765 Date: Fri, 15 Mar 2024 09:07:24 +0000 Subject: [PATCH 045/183] Update en_us.json --- .../src/main/resources/assets/distanthorizons/lang/en_us.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index b6a02475e..12103f61d 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -651,6 +651,10 @@ "High", "distanthorizons.config.enum.EVerticalQuality.EXTREME": "Extreme", + "distanthorizons.config.enum.EVerticalQuality.WHAT": + "What? (64)", + "distanthorizons.config.enum.EVerticalQuality.WHY": + "Why? (120)", "distanthorizons.config.enum.EHorizontalQuality.LOWEST": "Lowest", "distanthorizons.config.enum.EHorizontalQuality.LOW": From 01c1dbb146765031ca02e2c13e002d9a9bec08c0 Mon Sep 17 00:00:00 2001 From: cola98765 Date: Fri, 15 Mar 2024 09:13:24 +0000 Subject: [PATCH 046/183] Update en_us.json --- .../src/main/resources/assets/distanthorizons/lang/en_us.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index b6a02475e..e4fc3dbd1 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -871,9 +871,9 @@ "Rare", "distanthorizons.config.enum.ELodShading.MINECRAFT": - "Minecraft", + "Auto", "distanthorizons.config.enum.ELodShading.OLD_LIGHTING": - "Old Lighting", + "Force", "distanthorizons.config.enum.ELodShading.NONE": "None", From 6413e17e4b80a48a58adea30df83cc7ef7598cad Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 17:25:15 -0500 Subject: [PATCH 047/183] Add multiple compression options and unit tests --- .../config/EDhApiDataCompressionMode.java | 100 ++++ .../distanthorizons/core/config/Config.java | 34 ++ .../file/AbstractLegacyDataSourceHandler.java | 3 +- .../fullDatafile/NewFullDataFileHandler.java | 5 +- .../core/sql/dto/LegacyDataSourceDTO.java | 3 +- .../core/sql/dto/NewFullDataSourceDTO.java | 38 +- .../core/sql/repo/NewFullDataSourceRepo.java | 47 +- .../dataStreams/DhDataInputStream.java | 30 +- .../dataStreams/DhDataOutputStream.java | 27 +- .../assets/distanthorizons/lang/en_us.json | 16 +- ...te-createGeneratedFullDataSourceTables.sql | 1 + core/src/test/java/tests/CompressionTest.java | 526 ++++++++++-------- 12 files changed, 551 insertions(+), 279 deletions(-) create mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java new file mode 100644 index 000000000..ce277088c --- /dev/null +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiDataCompressionMode.java @@ -0,0 +1,100 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.api.enums.config; + +/** + * UNCOMPRESSED
+ * LZ4
+ * ZSTD
+ * XZ

+ * + * Note: speed and compression ratios are examples + * and should only be used for estimated comparisons. + * + * @version 2024-3-16 + * @since API 1.1.0 + */ +public enum EDhApiDataCompressionMode +{ + // Reminder: + // when adding items up the API minor version + // when removing items up the API major version + + /** + * Should only be used internally and for unit testing.

+ * + * Read Speed: 1.64 MS / DTO
+ * Write Speed: 12.44 MS / DTO
+ * Compression ratio: 1.0
+ */ + @DisallowSelectingViaConfigGui + UNCOMPRESSED(0), + + /** + * Extremely fast (often faster than uncompressed), but generally poor compression.

+ * + * Read Speed: 1.85 MS / DTO
+ * Write Speed: 9.46 MS / DTO
+ * Compression ratio: 0.3638
+ */ + LZ4(1), + + /** + * Decent speed and good compression.

+ * + * Read Speed: 11.78 MS / DTO
+ * Write Speed: 16.76 MS / DTO
+ * Compression ratio: 0.2199
+ */ + Z_STD(2), + + /** + * Extremely slow, but very good compression.

+ * + * Read Speed: 12.25 MS / DTO
+ * Write Speed: 490.07 MS / DTO
+ * Compression ratio: 0.1242
+ */ + LZMA2(3); + + + + /** More stable than using the ordinal of the enum */ + public final byte value; + + EDhApiDataCompressionMode(int value) { this.value = (byte) value; } + + + public static EDhApiDataCompressionMode getFromValue(byte value) + { + EDhApiDataCompressionMode[] enumList = EDhApiDataCompressionMode.values(); + for (int i = 0; i < enumList.length; i++) + { + if (enumList[i].value == value) + { + return enumList[i]; + } + } + + throw new IllegalArgumentException("No compression mode with the value ["+value+"]"); + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 799c1a7c7..4cdaf7096 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -801,6 +801,40 @@ public class Config + "") .build(); + public static ConfigEntry dataCompression = new ConfigEntry.Builder() + .set(EDhApiDataCompressionMode.LZMA2) + .comment("" + + "What algorithm should be used to compress new LOD data? \n" + + "This setting will only affect new or updated LOD data, \n" + + "any data already generated when this setting is changed will be\n" + + "unaffected until it needs to be re-written to the database.\n" + + "\n" + + EDhApiDataCompressionMode.UNCOMPRESSED + " \n" + + "Should only be used for testing, is worse in every way vs ["+EDhApiDataCompressionMode.LZ4+"].\n" + + "Expected Compression Ratio: 1.0\n" + + "Estimated average DTO read speed: 1.64 milliseconds\n" + + "Estimated average DTO write speed: 12.44 milliseconds\n" + + "\n" + + EDhApiDataCompressionMode.LZ4 + " \n" + + "A good option if you're CPU limited and have plenty of hard drive space.\n" + + "Expected Compression Ratio: 0.36\n" + + "Estimated average DTO read speed: 1.85 ms\n" + + "Estimated average DTO write speed: 9.46 ms\n" + + "\n" + + EDhApiDataCompressionMode.Z_STD + " \n" + + "A good middle ground between speed and compression.\n" + + "Expected Compression Ratio: 0.21\n" + + "Estimated average DTO read speed: 11.78 ms\n" + + "Estimated average DTO write speed: 16.77 ms\n" + + "\n" + + EDhApiDataCompressionMode.LZMA2 + " \n" + + "Slow but very good compression.\n" + + "Expected Compression Ratio: 0.14\n" + + "Estimated average DTO read speed: 11.89 ms\n" + + "Estimated average DTO write speed: 192.01 ms\n" + + "") + .build(); + } public static class Multiplayer diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index b24598f34..7a3c65dc1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -1,5 +1,6 @@ package com.seibel.distanthorizons.core.file; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; @@ -340,7 +341,7 @@ public abstract class AbstractLegacyDataSourceHandler public DhDataInputStream getInputStream() throws IOException { InputStream inputStream = new ByteArrayInputStream(this.dataArray); - DhDataInputStream compressedStream = new DhDataInputStream(inputStream); + DhDataInputStream compressedStream = new DhDataInputStream(inputStream, EDhApiDataCompressionMode.LZ4); // LZ4 was used by DH before 2.1.0 and as such must be used until the render data format is changed to record the compressor return compressedStream; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java index a9c58e6ae..d61f0e2a2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java @@ -19,7 +19,9 @@ package com.seibel.distanthorizons.core.sql.dto; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -44,16 +46,15 @@ public class NewFullDataSourceDTO implements IBaseDTO /** only for the data array */ public int dataChecksum; - //public long[][] dataArray; public byte[] dataByteArray; /** @see EDhApiWorldGenerationStep */ public byte[] columnGenStepByteArray; - //public FullDataPointIdMap mapping; public byte[] mappingByteArray; public byte dataFormatVersion; + public EDhApiDataCompressionMode compressionModeEnum; public boolean applyToParent; @@ -66,14 +67,14 @@ public class NewFullDataSourceDTO implements IBaseDTO // constructor // //=============// - public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource) throws IOException + public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException { - CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints); - byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping()); + CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); + byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping(), compressionModeEnum); return new NewFullDataSourceDTO( dataSource.getSectionPos(), - checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY @@ -81,7 +82,7 @@ public class NewFullDataSourceDTO implements IBaseDTO } public NewFullDataSourceDTO( DhSectionPos pos, - int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, byte[] dataByteArray, + int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] dataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, byte[] mappingByteArray, boolean applyToParent, int levelMinY) @@ -91,6 +92,7 @@ public class NewFullDataSourceDTO implements IBaseDTO this.columnGenStepByteArray = columnGenStepByteArray; this.dataFormatVersion = dataFormatVersion; + this.compressionModeEnum = compressionModeEnum; this.dataByteArray = dataByteArray; this.mappingByteArray = mappingByteArray; @@ -130,7 +132,7 @@ public class NewFullDataSourceDTO implements IBaseDTO } dataSource.columnGenerationSteps = this.columnGenStepByteArray; - dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray, this.compressionModeEnum); dataSource.getMapping().clear(dataSource.getSectionPos()); // should only be null when used in a unit test @@ -141,7 +143,7 @@ public class NewFullDataSourceDTO implements IBaseDTO throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); } - dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper)); + dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum)); } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; @@ -160,7 +162,7 @@ public class NewFullDataSourceDTO implements IBaseDTO // (de)serializing // //=================// - private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray) throws IOException + private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -169,7 +171,7 @@ public class NewFullDataSourceDTO implements IBaseDTO CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, // but since this stream will be closed immediately after writing anyway, it won't be an issue - DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut); + DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut, compressionModeEnum); // write the data @@ -180,7 +182,7 @@ public class NewFullDataSourceDTO implements IBaseDTO // write column length int columnLength = (dataColumn != null) ? dataColumn.length : 0; - compressedOut.writeInt(columnLength); // TODO + compressedOut.writeInt(columnLength); /// TODO // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) @@ -197,10 +199,10 @@ public class NewFullDataSourceDTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray) throws IOException + private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); - DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); // read the data @@ -227,10 +229,10 @@ public class NewFullDataSourceDTO implements IBaseDTO } - private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping) throws IOException + private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream); + DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); mapping.serialize(compressedOut); @@ -239,10 +241,10 @@ public class NewFullDataSourceDTO implements IBaseDTO return byteArrayOutputStream.toByteArray(); } - private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); - DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper); return mapping; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java index dcd37a9b4..410c5a3e4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.sql.repo; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; @@ -71,6 +72,8 @@ public class NewFullDataSourceRepo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( "select SUM(LENGTH(Data)) as dataSize " + "from "+this.getTableName()+"; "); - - int dataLength = (resultMap != null) ? (int) resultMap.get("dataSize") : 0; - return dataLength; + + if (resultMap != null && resultMap.get("dataSize") != null) + { + Number resultNumber = (Number) resultMap.get("dataSize"); + long dataLength = resultNumber.longValue(); + return dataLength; + + } + else + { + return 0; + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java index 81ae64895..c78c3a486 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java @@ -19,12 +19,12 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; +import com.github.luben.zstd.ZstdInputStream; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import net.jpountz.lz4.LZ4FrameInputStream; +import org.tukaani.xz.XZInputStream; -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; /** * Combines multiple different streams together for ease of use @@ -38,9 +38,27 @@ import java.io.InputStream; */ public class DhDataInputStream extends DataInputStream { - public DhDataInputStream(InputStream stream) throws IOException + public DhDataInputStream(InputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException + { + super(warpStream(new BufferedInputStream(stream), compressionMode)); + } + private static InputStream warpStream(InputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException { - super(new LZ4FrameInputStream(new BufferedInputStream(stream))); + switch (compressionMode) + { + case UNCOMPRESSED: + return stream; + case LZ4: + return new LZ4FrameInputStream(stream); + case Z_STD: + return new ZstdInputStream(stream); + case LZMA2: + // Note: all LZMA/XZ compressors can be decompressed using this same InputStream + return new XZInputStream(stream); + + default: + throw new IllegalArgumentException("No compressor defined for ["+compressionMode+"]"); + } } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java index 650aab083..ac184e7fd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java @@ -19,7 +19,11 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; +import com.github.luben.zstd.ZstdOutputStream; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import net.jpountz.lz4.LZ4FrameOutputStream; +import org.tukaani.xz.LZMA2Options; +import org.tukaani.xz.XZOutputStream; import java.io.*; @@ -30,9 +34,28 @@ import java.io.*; */ public class DhDataOutputStream extends DataOutputStream { - public DhDataOutputStream(OutputStream stream) throws IOException + public DhDataOutputStream(OutputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException + { + super(warpStream(new BufferedOutputStream(stream), compressionMode)); + } + private static OutputStream warpStream(OutputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException { - super(new LZ4FrameOutputStream(new BufferedOutputStream(stream))); + switch (compressionMode) + { + case UNCOMPRESSED: + return stream; + case LZ4: + return new LZ4FrameOutputStream(stream); + case Z_STD: + return new ZstdOutputStream(stream); + case LZMA2: + // in James' testing preset 4 has the best balance between compression ratio and speed + // 5 is slightly more compressed 0.128 vs 0.139, but is roughly 60% slower + return new XZOutputStream(stream, new LZMA2Options(4)); + + default: + throw new IllegalArgumentException("No compressor defined for ["+compressionMode+"]"); + } } @Override diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 2f1867f4b..becb94c31 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -341,13 +341,18 @@ "distanthorizons.config.client.advanced.lodBuilding.minTimeBetweenChunkUpdatesInSeconds": "Minimum Time Between Chunk Updates In Seconds", + "distanthorizons.config.client.advanced.lodBuilding.minTimeBetweenChunkUpdatesInSeconds.@tooltip": "Determines how long must pass between LOD chunk updates before another. \nupdate can occur\n\nIncreasing this value will reduce CPU load but may may cause \nLODs to become outdated more frequently or for longer.", "distanthorizons.config.client.advanced.lodBuilding.onlyUseDhLightingEngine": "Only Use DH Lighting Engine", "distanthorizons.config.client.advanced.lodBuilding.onlyUseDhLightingEngine.@tooltip": "If false LODs will be lit by Minecraft's lighting engine when possible \nand fall back to the DH lighting engine only when necessary. \n\nIf true LODs will only be lit using Distant Horizons' lighting engine. \n\nGenerally it is best to leave this disabled and should only be enabled \nif there are lighting issues or for debugging.", - + "distanthorizons.config.client.advanced.lodBuilding.dataCompression": + "Data Compression", + "distanthorizons.config.client.advanced.lodBuilding.dataCompression.@tooltip": + "What algorithm should be used to compress new LOD data? \nThis setting will only affect new or updated LOD data, \nany data already generated when this setting is changed will be \nunaffected until it needs to be re-written to the database. \n\nFastest: LZ4 \nHighest Compression: LZMA2", + "distanthorizons.config.client.advanced.multiplayer": "Multiplayer", @@ -755,6 +760,15 @@ "distanthorizons.config.enum.EDhApiDistantGeneratorMode.FULL": "Full", + "distanthorizons.config.enum.EDhApiDataCompressionMode.UNCOMPRESSED": + "Uncompressed", + "distanthorizons.config.enum.EDhApiDataCompressionMode.LZ4": + "LZ4", + "distanthorizons.config.enum.EDhApiDataCompressionMode.Z_STD": + "Zstd", + "distanthorizons.config.enum.EDhApiDataCompressionMode.LZMA2": + "LZMA2", + "distanthorizons.config.enum.ELightGenerationMode.DISTANT_HORIZONS": "Distant Horizons", "distanthorizons.config.enum.ELightGenerationMode.MINECRAFT": diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql index 34986c361..374783d3e 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -32,6 +32,7 @@ CREATE TABLE FullData ( ,Mapping BLOB NULL ,DataFormatVersion TINYINT NULL + ,CompressionMode TINYINT NULL ,ApplyToParent BIT NULL diff --git a/core/src/test/java/tests/CompressionTest.java b/core/src/test/java/tests/CompressionTest.java index 8eb54eda2..228c6bc92 100644 --- a/core/src/test/java/tests/CompressionTest.java +++ b/core/src/test/java/tests/CompressionTest.java @@ -19,152 +19,212 @@ package tests; -import net.jpountz.lz4.*; +import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; +import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; import org.junit.Assert; -import org.junit.Test; import java.io.*; -import java.nio.file.Files; import java.text.CharacterIterator; import java.text.StringCharacterIterator; import java.util.ArrayList; -//import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; -//import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; -//import com.github.luben.zstd.ZstdInputStream; -//import com.github.luben.zstd.ZstdOutputStream; - /** - * Results (2023-5-20):
- * 200 files

- * - * uncompressed

- * - * render data - ratio 1.0 (shocker :P)
- * read time in - 784 ms, avg 3 ms/file
- * write time in - 803 ms, avg 4 ms/file

- * - * full data - ratio 1.0
- * read time in - 2,213 ms, avg 11 ms/file
- * write time in - 1,753 ms, avg 8 ms/file


- * - * - * XZ

- * - * render data - ratio 0.1044
- * read time in - 2,413 ms, avg 12 ms/file
- * write time in - 28,441 ms, avg 142 ms/file

- * - * full data - ratio 0.1123
- * read time in - 5,888 ms, avg 29 ms/file
- * write time in - 79,675 ms, avg 398 ms/file


- * - * - * LZ4

- * - * render data - ratio 0.2933
- * read time in - 846 ms, avg 4 ms/file
- * write time in - 1,040 ms, avg 5 ms/file

- * - * full data - ratio 0.3275
- * read time in - 1,964 ms, avg 9 ms/file
- * write time in - 1,584 ms, avg 7 ms/file


- * - * - * Z Standard

- * - * render data - ratio 0.1791
- * read time in - 5,170 ms, avg 25 ms/file
- * write time in - 5,294 ms, avg 26 ms/file

- * - * full data - ratio 0.2060
- * read time in - 14,754 ms, avg 73 ms/file
- * write time in - 14,057 ms, avg 70 ms/file


- * - * - * * Note: * In order to test the compressors that aren't currently in use:
- * 1. Generate DH data and point the {@link CompressionTest#TEST_DIR} variable to the "Distant_Horizons" folder. - * 2. Add the following to build.gradle's dependencies block:
+ * 1. Generate DH data (64 DH render distance is suggest) + * 2. Point the {@link CompressionTest#TEST_DIR} variable to the world's "data" folder. + * 3. (Optional) Add the following to build.gradle's dependencies block:
* - * shadowMe("org.tukaani:xz:1.9") - * shadowMe("org.apache.commons:commons-compress:1.21") - * shadowMe("com.github.luben:zstd-jni:1.5.5-3") + * forgeShadowMe('org.apache.commons:commons-compress:1.26.1') + * forgeShadowMe('org.itadaki:bzip2:0.9.1') + * forgeShadowMe('lzma:lzma:0.0.1') *
- * 3. Uncomment the tests in this file
- * 4. Run the tests like normal + * 4. (Optional) Uncomment the tests in this file
+ * 5. Run tests like normal */ public class CompressionTest { - public static String TEST_DIR = "C:\\DistantHorizonsWorkspace\\distantHorizons\\fabric\\run\\saves\\Arcapelago\\data\\Distant_Horizons"; - public static String RENDER_DATA_PATH = TEST_DIR + "\\renderCache"; - public static String FULL_DATA_PATH = TEST_DIR + "\\data"; + public static String TEST_DIR = "C:\\DistantHorizonsWorkspace\\distant-horizons\\run\\saves\\Arcapelago\\data"; + public static String DB_FILE_NAME_PREFIX = "DistantHorizons"; + public static String UNCOMPRESSED_DB_FILE_NAME = "DistantHorizons.sqlite"; - /** limits the number of files tested so I don't have to wait 10 minutes for the slower compressors */ - public static int MAX_NUMBER_OF_FILES_TO_TEST = 200; + /** -1 will test all of them */ + public static int MAX_DTO_TEST_COUNT = -1; - // @Test + //@Test public void NoCompression() { String compressorName = "Uncompressed"; - - CreateInputStreamFunc createInputStreamFunc = (inputStream) -> inputStream; - CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> outputStream; - - - System.out.println(compressorName + " testing render data"); - this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); - System.out.println(compressorName + " testing full data"); - this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); + this.testCompressor(compressorName, EDhApiDataCompressionMode.UNCOMPRESSED); } - // @Test - public void Lz4() + // collapse the following commented out code when looking at tests + + //@Test + //public void GZIP() // DNF + //{ + // String compressorName = "GZIP"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new GZIPInputStream(inputStream); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new GZIPOutputStream(outputStream); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + //public void BZip2() // DNF + //{ + // String compressorName = "bzip2"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new BZip2InputStream(inputStream, true); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new BZip2OutputStream(outputStream); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + //public void blockLz4() // DNF + //{ + // String compressorName = "Block LZ4"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new BlockLZ4CompressorInputStream(inputStream); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new BlockLZ4CompressorOutputStream(outputStream); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + //public void lzma() // DNF, doesn't support flushing + //{ + // String compressorName = "lzma"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZMA2InputStream(inputStream, LZMA2Options.DICT_SIZE_MIN); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZMAOutputStream(outputStream, new LZMA2Options(), -1); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + //public void deflate() // DNF + //{ + // String compressorName = "deflate"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new DeflateCompressorInputStream(inputStream); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new DeflateCompressorOutputStream(outputStream); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + //public void snappy() // DNF + //{ + // String compressorName = "snappy"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new SnappyCompressorInputStream(inputStream); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new SnappyCompressorOutputStream(outputStream, Long.MAX_VALUE); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + + //@Test + //public void Zstd() + //{ + // String compressorName = "Zstd"; + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new ZstdInputStream(inputStream); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new ZstdOutputStream(outputStream); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + ////@Test + //public void ZstdDictionary() throws SQLException // isn't any better than normal Zstd + //{ + // String compressorName = "ZstdDictionary"; + // + // BufferPool pool = RecyclingBufferPool.INSTANCE; + // + // + // // create the dictionary + // byte[] dictionary; + // { + // String uncompressedDatabaseFilePath = TEST_DIR + "/" + UNCOMPRESSED_DB_FILE_NAME; + // NewFullDataSourceRepo uncompressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", uncompressedDatabaseFilePath); + // ArrayList positionList = uncompressedRepo.getAllPositions(); + // + // // sample size of 10 MB or less + // // dictionary size of 64 KB (1 MB and 10 MB's both seemed to perform worse) + // ZstdDictTrainer dictTrainer = new ZstdDictTrainer(10 * 1024 * 1024, 64 * 1024); + // + // for (int i = 0; i < positionList.size(); i++) + // { + // DhSectionPos pos = positionList.get(i); + // NewFullDataSourceDTO uncompressedDto = uncompressedRepo.getByKey(pos); + // + // dictTrainer.addSample(uncompressedDto.dataByteArray); + // } + // + // dictionary = dictTrainer.trainSamples(); + // } + // + // + // + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> + // { + // ZstdInputStream stream = new ZstdInputStream(inputStream, pool); + // stream.setDict(dictionary); + // return stream; + // }; + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> + // { + // ZstdOutputStream stream = new ZstdOutputStream(outputStream, pool); + // stream.setDict(dictionary); + // return stream; + // }; + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + //public void Lz4FastCompression() // DNF + //{ + // String compressorName = "LZ4FastCompression"; + // + // LZ4FastDecompressor fastCompressor = LZ4Factory.fastestInstance().fastDecompressor(); + // + // DhDataInputStream.CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZ4BlockInputStream(inputStream, fastCompressor); + // DhDataOutputStream.CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZ4BlockOutputStream(outputStream); + // + // this.testCompressor(compressorName, createInputStreamFunc, createOutputStreamFunc); + //} + + //@Test + public void Lz4() // fast, poor compression { String compressorName = "LZ4"; - - CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new LZ4FrameInputStream(inputStream); - CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new LZ4FrameOutputStream(outputStream); - - - System.out.println(compressorName + " testing render data"); - this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); - System.out.println(compressorName + " testing full data"); - this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); + this.testCompressor(compressorName, EDhApiDataCompressionMode.LZ4); + } + + //@Test + public void Zstd() // middle of the road + { + String compressorName = "Zstd"; + this.testCompressor(compressorName, EDhApiDataCompressionMode.Z_STD); + } + + //@Test + public void LZMA2() // very slow, very good compression though + { + String compressorName = "LZMA"; + this.testCompressor(compressorName, EDhApiDataCompressionMode.LZMA2); } - -// @Test -// public void Zstandard() -// { -// String compressorName = "Z_std"; -// -// CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new ZstdInputStream(inputStream); -// CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new ZstdOutputStream(outputStream); -// -// -// System.out.println(compressorName+" testing render data"); -// this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); -// System.out.println(compressorName+" testing full data"); -// this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); -// } - -// @Test -// public void Xz() -// { -// String compressorName = "XZ"; -// -// CreateInputStreamFunc createInputStreamFunc = (inputStream) -> new XZCompressorInputStream(inputStream); -// CreateOutputStreamFunc createOutputStreamFunc = (outputStream) -> new XZCompressorOutputStream(outputStream); -// -// -// System.out.println(compressorName+" testing render data"); -// this.testCompressor(compressorName, RENDER_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); -// System.out.println(compressorName+" testing full data"); -// this.testCompressor(compressorName, FULL_DATA_PATH, createInputStreamFunc, createOutputStreamFunc); -// } @@ -172,149 +232,139 @@ public class CompressionTest // testing methods // //=================// - @FunctionalInterface - public interface CreateInputStreamFunc + private void testCompressor(String compressorName, EDhApiDataCompressionMode compressionMode) { - InputStream apply(InputStream inputStream) throws Exception; + System.out.println("\n"); + System.out.println("Testing " + compressorName); - } - - @FunctionalInterface - public interface CreateOutputStreamFunc - { - OutputStream apply(OutputStream outputStream) throws Exception; - } - - private void testCompressor( - String compressorName, String inputFolderPath, - CreateInputStreamFunc createInputStreamFunc, - CreateOutputStreamFunc createOutputStreamFunc) - { - long totalUncompressedFileSizeInBytes = 0; - long totalCompressedFileSizeInBytes = 0; + long minUncompressedDtoSizeInBytes = Long.MAX_VALUE; + long maxUncompressedDtoSizeInBytes = 0; + long avgUncompressedDtoSizeInBytes = 0; + + long minCompressedDtoSizeInBytes = Long.MAX_VALUE; + long maxCompressedDtoSizeInBytes = 0; + long avgCompressedDtoSizeInBytes = 0; + + long totalReadTimeInNano = 0; + long totalWriteTimeInNano = 0; + + + long totalUncompressedFileSizeInBytes; + long totalCompressedFileSizeInBytes; - long totalReadTimeInMs = 0; - long totalWriteTimeInMs = 0; try { - File inputFolder = new File(inputFolderPath); - File[] inputFileArray = inputFolder.listFiles(); - Assert.assertNotNull(inputFileArray); + String uncompressedDatabaseFilePath = TEST_DIR + "/" + UNCOMPRESSED_DB_FILE_NAME; + File uncompressedDatabaseFile = new File(uncompressedDatabaseFilePath); + Assert.assertTrue(uncompressedDatabaseFile.exists()); - File compressedFolder = new File(inputFolderPath + "\\" + compressorName); - compressedFolder.delete(); - compressedFolder.mkdirs(); + NewFullDataSourceRepo uncompressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", uncompressedDatabaseFilePath); - int processedFileCount = 0; - for (File inputFile : inputFileArray) + String compressedDatabaseFilePath = TEST_DIR + "/output/" + DB_FILE_NAME_PREFIX + "_" + compressorName + ".sqlite"; + File compressedDatabaseFile = new File(compressedDatabaseFilePath); + compressedDatabaseFile.mkdirs(); + compressedDatabaseFile.delete(); + Assert.assertTrue(!compressedDatabaseFile.exists()); + NewFullDataSourceRepo compressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", compressedDatabaseFilePath); + + + + ArrayList positionList = uncompressedRepo.getAllPositions(); + totalUncompressedFileSizeInBytes = uncompressedRepo.getTotalDataSizeInBytes(); + System.out.println("Found [" + positionList.size() + "] DTOs."); + + long processedDtoCount = 0; + int maxTestPosition = (MAX_DTO_TEST_COUNT == -1) ? positionList.size() : MAX_DTO_TEST_COUNT; + for (int i = 0; i < maxTestPosition; i++) { - if (inputFile.isDirectory()) + try { - continue; - } - - // can be used to speed up the tests - if (processedFileCount >= MAX_NUMBER_OF_FILES_TO_TEST) - { - break; - } - - - - // uncompressed file input // - ArrayList originalFileByteArray = new ArrayList<>(); - totalUncompressedFileSizeInBytes += Files.size(inputFile.toPath()); - - try (FileInputStream fileStream = new FileInputStream(inputFile); - BufferedInputStream bufferedStream = new BufferedInputStream(fileStream); - DataInputStream dataStream = new DataInputStream(bufferedStream)) - { - try + DhSectionPos pos = positionList.get(i); + if (i % 20 == 0) { - while (true) - { - byte nextByte = dataStream.readByte(); - originalFileByteArray.add(nextByte); - } + System.out.println(i + "/" + maxTestPosition); } - catch (EOFException e) - { /* end of file reached */ } + + + + // uncompressed input // + + NewFullDataSourceDTO uncompressedDto = uncompressedRepo.getByKey(pos); + Assert.assertEquals(uncompressedDto.compressionModeEnum, EDhApiDataCompressionMode.UNCOMPRESSED); + NewFullDataSource uncompressedDataSource = uncompressedDto.createUnitTestDataSource(); + + long uncompressedDtoSize = uncompressedRepo.getDataSizeInBytes(pos); + minUncompressedDtoSizeInBytes = Math.min(uncompressedDtoSize, minUncompressedDtoSizeInBytes); + maxUncompressedDtoSizeInBytes = Math.max(uncompressedDtoSize, maxUncompressedDtoSizeInBytes); + avgUncompressedDtoSizeInBytes += uncompressedDtoSize; + + + + // compress file // + + long startWriteNanoTime = System.nanoTime(); + + NewFullDataSourceDTO compressedDto = NewFullDataSourceDTO.CreateFromDataSource(uncompressedDataSource, compressionMode); + compressedRepo.save(compressedDto); + + long endWriteNanoTime = System.nanoTime(); + totalWriteTimeInNano += (endWriteNanoTime - startWriteNanoTime); + + + long compressedDtoSize = compressedRepo.getDataSizeInBytes(pos); + minCompressedDtoSizeInBytes = Math.min(compressedDtoSize, minCompressedDtoSizeInBytes); + maxCompressedDtoSizeInBytes = Math.max(compressedDtoSize, maxCompressedDtoSizeInBytes); + avgCompressedDtoSizeInBytes += compressedDtoSize; + + + + // read compressed file // + + long startReadNanoTime = System.nanoTime(); + + compressedDto = compressedRepo.getByKey(pos); + NewFullDataSource compressedDataSource = compressedDto.createUnitTestDataSource(); + + long endReadMsTime = System.nanoTime(); + totalReadTimeInNano += (endReadMsTime - startReadNanoTime); + + + processedDtoCount++; } - - - - // compress file // - long startWriteMsTime = System.currentTimeMillis(); - - File compressedFile = new File(inputFolderPath + "\\" + compressorName + "\\" + inputFile.getName()); - compressedFile.delete(); - compressedFile.createNewFile(); - - try (FileOutputStream fileStream = new FileOutputStream(compressedFile); - BufferedOutputStream bufferedStream = new BufferedOutputStream(fileStream); - OutputStream compressorStream = createOutputStreamFunc.apply(bufferedStream); - DataOutputStream dataStream = new DataOutputStream(compressorStream)) + catch (Exception | Error e) { - for (byte nextByte : originalFileByteArray) - { - dataStream.writeByte(nextByte); - } + e.printStackTrace(); + Assert.fail(e.getMessage()); } - - long endWriteMsTime = System.currentTimeMillis(); - totalWriteTimeInMs += (endWriteMsTime - startWriteMsTime); - - totalCompressedFileSizeInBytes += Files.size(compressedFile.toPath()); - - - - // read compressed file // - long startReadMsTime = System.currentTimeMillis(); - ArrayList compressedFileByteArray = new ArrayList<>(); - - try (FileInputStream fileStream = new FileInputStream(compressedFile); - BufferedInputStream bufferedStream = new BufferedInputStream(fileStream); - InputStream compressorStream = createInputStreamFunc.apply(bufferedStream); - DataInputStream dataStream = new DataInputStream(compressorStream)) - { - try - { - while (true) - { - byte nextByte = dataStream.readByte(); - compressedFileByteArray.add(nextByte); - } - } - catch (EOFException e) - { /* end of file reached */ } - } - - long endReadMsTime = System.currentTimeMillis(); - totalReadTimeInMs += (endReadMsTime - startReadMsTime); - - - // confirm the file contents are the same - Assert.assertEquals("byte array size mismatch", compressedFileByteArray.size(), originalFileByteArray.size()); - for (int i = 0; i < compressedFileByteArray.size(); i++) - { - Assert.assertEquals("array content mismatch at index [" + i + "]", compressedFileByteArray.get(i), originalFileByteArray.get(i)); - } - - - processedFileCount++; } + + totalCompressedFileSizeInBytes = compressedRepo.getTotalDataSizeInBytes(); + + avgCompressedDtoSizeInBytes /= processedDtoCount; + avgUncompressedDtoSizeInBytes /= processedDtoCount; + + double compressionRatio = (totalCompressedFileSizeInBytes / (double) totalUncompressedFileSizeInBytes); String compressionRatioString = compressionRatio + ""; compressionRatioString = compressionRatioString.substring(0, Math.min(6, compressionRatioString.length())); - System.out.println("Uncompressed file size: [" + humanReadableByteCountSI(totalUncompressedFileSizeInBytes) + "] Compressed file size: [" + humanReadableByteCountSI(totalCompressedFileSizeInBytes) + "]. Compression ratio: [" + compressionRatioString + "]."); - System.out.println("Total read time in MS: [" + totalReadTimeInMs + "] Average read time per file: [" + (totalReadTimeInMs / processedFileCount) + "]"); - System.out.println("Total write time in MS: [" + totalWriteTimeInMs + "] Average write time per file: [" + (totalWriteTimeInMs / processedFileCount) + "]"); + + System.out.println("\n"); + System.out.println("Results: " + compressorName); + System.out.println(); + System.out.println("Total uncompressed data: [" + humanReadableByteCountSI(totalUncompressedFileSizeInBytes) + "] Total compressed data: [" + humanReadableByteCountSI(totalCompressedFileSizeInBytes) + "]. Compression ratio: [" + compressionRatioString + "]."); + System.out.println("Min uncompressed data: [" + humanReadableByteCountSI(minUncompressedDtoSizeInBytes) + "] Min compressed data: [" + humanReadableByteCountSI(minCompressedDtoSizeInBytes) + "]."); + System.out.println("Max uncompressed data: [" + humanReadableByteCountSI(maxUncompressedDtoSizeInBytes) + "] Max compressed data: [" + humanReadableByteCountSI(maxCompressedDtoSizeInBytes) + "]."); + System.out.println("Avg uncompressed data: [" + humanReadableByteCountSI(avgUncompressedDtoSizeInBytes) + "] Avg compressed data: [" + humanReadableByteCountSI(avgCompressedDtoSizeInBytes) + "]."); + System.out.println(); + System.out.println("Total read time in MS: [" + totalReadTimeInNano / 1_000_000.0 + "] Average read time per dto: [" + (totalReadTimeInNano / processedDtoCount) / 1_000_000.0 + "]"); + System.out.println("Total write time in MS: [" + totalWriteTimeInNano / 1_000_000.0 + "] Average write time per dto: [" + (totalWriteTimeInNano / processedDtoCount) / 1_000_000.0 + "]"); + System.out.println(); } catch (Exception e) { From c081b6c57c6c6320553a46d80f2a4cfc00532a1f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 17:47:06 -0500 Subject: [PATCH 048/183] Remove unusused AbstractDhRepo.get(TDTO) --- .../seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java | 1 - .../0020-sqlite-createGeneratedFullDataSourceTables.sql | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index 1a5dfd945..aa45199f9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -126,7 +126,6 @@ public abstract class AbstractDhRepo> // high level DB // //===============// - public TDTO get(TDTO dto) { return this.getByKey(dto.getKey()); } public TDTO getByKey(TKey primaryKey) { Map objectMap = this.queryDictionaryFirst(this.createSelectByKeySql(primaryKey)); diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql index 374783d3e..82fc3360b 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql @@ -13,7 +13,7 @@ where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0; --batch-- --- shrink the database file for the removed legacy detail levels +-- shrink the database file after having removed low detail legacy LODs VACUUM; --batch-- From 031bf754e87be544737ec7578f010ff09f0f9ebf Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 19:16:50 -0500 Subject: [PATCH 049/183] Remove SingleColumnFullDataAccessor and IFullDataAccessor --- .../methods/data/DhApiTerrainDataRepo.java | 7 +- .../accessor/FullDataArrayAccessor.java | 92 +-------- .../fullData/accessor/IFullDataAccessor.java | 77 -------- .../SingleColumnFullDataAccessor.java | 182 ------------------ .../sources/CompleteFullDataSource.java | 12 +- .../fullData/sources/NewFullDataSource.java | 31 ++- .../render/ColumnRenderSource.java | 10 +- .../FullDataToRenderDataTransformer.java | 34 ++-- .../SubDimensionLevelMatcher.java | 13 +- .../core/level/DhServerLevel.java | 1 - .../core/pos/DhSectionPos.java | 10 +- 11 files changed, 59 insertions(+), 410 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/SingleColumnFullDataAccessor.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 056f263de..913588b4d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -27,7 +27,6 @@ import com.seibel.distanthorizons.api.interfaces.data.IDhApiTerrainDataRepo; import com.seibel.distanthorizons.api.objects.math.DhApiVec3i; import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -221,10 +220,10 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.getMapping(); - SingleColumnFullDataAccessor dataColumn = dataSource.get(relativePos.x, relativePos.z); + long[] dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { - int dataColumnIndexCount = dataColumn.getSingleLength(); + int dataColumnIndexCount = dataColumn.length; DhApiTerrainDataPoint[] returnArray = new DhApiTerrainDataPoint[dataColumnIndexCount]; long dataPoint; @@ -235,7 +234,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // search for a datapoint that contains the block y position for (int i = 0; i < dataColumnIndexCount; i++) { - dataPoint = dataColumn.getSingle(i); + dataPoint = dataColumn[i]; if (!getSpecificYCoordinate) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java index 123a29653..f3fff6741 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; @@ -30,8 +29,8 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFull * * @see CompleteFullDataSource */ -@Deprecated -public class FullDataArrayAccessor implements IFullDataAccessor +@Deprecated // TODO merge into FullDataSourceV1 +public class FullDataArrayAccessor { protected final FullDataPointIdMap mapping; @@ -82,92 +81,14 @@ public class FullDataArrayAccessor implements IFullDataAccessor - //=========// - // methods // - //=========// - - @Override - public FullDataArrayAccessor subView(int width, int xOffset, int zOffset) { return new FullDataArrayAccessor(this, width, xOffset, zOffset); } - - /** WARNING: This will potentially share the underlying array object! */ - public void shadowCopyTo(FullDataArrayAccessor target) - { - if (target.width != this.width) - { - throw new IllegalArgumentException("Target view must have same size as this view"); - } - - - if (target.mapping.equals(this.mapping)) - { - for (int x = 0; x < this.width; x++) - { - System.arraycopy(this.dataArrays, this.offset + x * this.dataWidth, - target.dataArrays, target.offset + x * target.dataWidth, this.width); - } - } - else - { - int[] remappedIds = target.mapping.mergeAndReturnRemappedEntityIds(this.mapping); - for (int x = 0; x < this.width; x++) - { - for (int z = 0; z < this.width; z++) - { - long[] currentData = this.dataArrays[this.offset + x * this.dataWidth + z]; - // may be null if no data exists for this column yet - if (currentData != null) - { - long[] newData = new long[currentData.length]; // TODO what to do if null? - for (int dataPointIndex = 0; dataPointIndex < newData.length; dataPointIndex++) - { - newData[dataPointIndex] = FullDataPointUtil.remap(remappedIds, currentData[dataPointIndex]); - } - - target.dataArrays[target.offset + x * target.dataWidth + z] = newData; - } - } - } - } - } - - /** - * Takes a higher detail {@link FullDataArrayAccessor}'s and converts the data to a lower detail level. - * - * @param incomingFullDataAccessor must be larger than this {@link FullDataArrayAccessor} and its width must a power of two larger (example: this.width = 4, other.width = 8) - */ - public void downsampleFrom(FullDataArrayAccessor incomingFullDataAccessor) - { - // validate that the incoming data isn't smaller than this accessor - LodUtil.assertTrue(incomingFullDataAccessor.width >= this.width && incomingFullDataAccessor.width % this.width == 0); - - int dataPointsPerWidthUnit = incomingFullDataAccessor.width / this.width; - for (int xOffset = 0; xOffset < this.width; xOffset++) - { - for (int zOffset = 0; zOffset < this.width; zOffset++) - { - FullDataArrayAccessor subView = incomingFullDataAccessor.subView(dataPointsPerWidthUnit, - xOffset * dataPointsPerWidthUnit, - zOffset * dataPointsPerWidthUnit); - - SingleColumnFullDataAccessor column = this.get(xOffset, zOffset); - column.downsampleFrom(subView); - } - } - } - - - //=========// // getters // //=========// - @Override public FullDataPointIdMap getMapping() { return this.mapping; } - @Override - public SingleColumnFullDataAccessor get(int index) { return this.get(index / this.width, index % this.width); } - @Override - public SingleColumnFullDataAccessor get(int relativeX, int relativeZ) + public long[] get(int index) { return this.get(index / this.width, index % this.width); } + public long[] get(int relativeX, int relativeZ) { int dataArrayIndex = (relativeX * this.width) + relativeZ + this.offset; if (dataArrayIndex >= this.dataArrays.length) @@ -179,10 +100,7 @@ public class FullDataArrayAccessor implements IFullDataAccessor "dataArrays.length: [" + this.dataArrays.length + "] dataArrayIndex: [" + dataArrayIndex + "]."); } - return new SingleColumnFullDataAccessor(this.mapping, this.dataArrays, dataArrayIndex); + return this.dataArrays[dataArrayIndex]; } - @Override - public int width() { return this.width; } - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java deleted file mode 100644 index 62cd58cc4..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/IFullDataAccessor.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; - -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; - -import java.util.Iterator; - -/** - * Contains raw full data points, which must be interpreted by the {@link FullDataPointUtil}.
- * Often used by {@link CompleteFullDataSource}'s. - * - * @see FullDataArrayAccessor - * @see FullDataPointUtil - */ -@Deprecated -public interface IFullDataAccessor -{ - FullDataPointIdMap getMapping(); - - /** generally used for iterating through the whole data set */ - SingleColumnFullDataAccessor get(int index); - SingleColumnFullDataAccessor get(int relativeX, int relativeZ); - - /** measured in full data points */ - int width(); - - /** - * Creates a new {@link IFullDataAccessor} with the given width and starting at the given X and Z offsets.
- * The returned object will use the same underlining data structure (IE memory addresses) as the source {@link IFullDataAccessor}. - */ - IFullDataAccessor subView(int width, int xOffset, int zOffset); - - - - - /** Returns an iterator that goes over each data column */ - default Iterator iterator() - { - return new Iterator() - { - private int index = 0; - private final int size = width() * width(); - - @Override - public boolean hasNext() { return this.index < this.size; } - - @Override - public SingleColumnFullDataAccessor next() - { - LodUtil.assertTrue(this.hasNext(), "No more data to iterate!"); - return get(this.index++); - } - }; - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/SingleColumnFullDataAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/SingleColumnFullDataAccessor.java deleted file mode 100644 index b9f66e242..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/SingleColumnFullDataAccessor.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; - -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; - -/** - * Represents a single column of Full LOD data. - * - * @see FullDataPointUtil - */ -public class SingleColumnFullDataAccessor implements IFullDataAccessor -{ - /** - * A flattened 2D array (for the X and Z directions) containing an array for the Y direction. - * TODO the flattened array is probably to reduce garbage collection overhead, but is doing it this way worth while? Having a 3D array would be much easier to understand - * - * @see FullDataArrayAccessor#dataArrays - */ - private final long[][] dataArrays; - /** indicates what index of the {@link SingleColumnFullDataAccessor#dataArrays} is used by this accessor */ - private final int dataArrayIndex; - private final FullDataPointIdMap mapping; - - - - //=============// - // constructor // - //=============// - - public SingleColumnFullDataAccessor(FullDataPointIdMap mapping, long[][] dataArrays, int dataArrayIndex) - { - this.dataArrays = dataArrays; - this.dataArrayIndex = dataArrayIndex; - this.mapping = mapping; - - LodUtil.assertTrue(this.dataArrayIndex < this.dataArrays.length, "dataArrays.length [" + this.dataArrays.length + "] is less than the dataArrayIndex [" + this.dataArrayIndex + "]."); - } - - - - //=========// - // methods // - //=========// - - /** @return true if any data exists in this column. */ - public boolean doesColumnExist() - { - long[] dataColumn = this.dataArrays[this.dataArrayIndex]; - return dataColumn != null && dataColumn.length != 0; - } - - @Override - public FullDataPointIdMap getMapping() { return this.mapping; } - - @Override - public SingleColumnFullDataAccessor get(int index) - { - if (index != 0) - { - throw new IllegalArgumentException("Only contains 1 column of full data!"); - } - - return this; - } - - @Override - public SingleColumnFullDataAccessor get(int relativeX, int relativeZ) - { - if (relativeX != 0 || relativeZ != 0) - { - throw new IllegalArgumentException("Only contains 1 column of full data!"); - } - - return this; - } - - /** @return the entire array of raw full data points. */ - public long[] getRaw() { return this.dataArrays[this.dataArrayIndex]; } - - public long getSingle(int yIndex) { return this.dataArrays[this.dataArrayIndex][yIndex]; } - public void setSingle(int yIndex, long fullDataPoint) { this.dataArrays[this.dataArrayIndex][yIndex] = fullDataPoint; } - - public void setNew(long[] newArray) { this.dataArrays[this.dataArrayIndex] = newArray; } - - /** @return how many data points are in this column */ - public int getSingleLength() - { - long[] singleDataArray = this.dataArrays[this.dataArrayIndex]; - return singleDataArray != null ? singleDataArray.length : 0; - } - - @Override - public int width() { return 1; } - - @Override - public IFullDataAccessor subView(int width, int xOffset, int zOffset) - { - if (width != 1 || xOffset != 1 || zOffset != 1) - { - throw new IllegalArgumentException("Getting invalid range of subView from SingleColumnFullDataAccessor!"); - } - - return this; - } - - /** WARNING: This may potentially share the underlying array objects! */ - public void shadowCopyTo(SingleColumnFullDataAccessor target) - { - if (target.mapping.equals(this.mapping)) - { - target.dataArrays[target.dataArrayIndex] = this.dataArrays[this.dataArrayIndex]; - } - else - { - int[] remappedEntryIds = target.mapping.mergeAndReturnRemappedEntityIds(this.mapping); - long[] sourceData = this.dataArrays[this.dataArrayIndex]; - long[] newData = new long[sourceData.length]; - - for (int i = 0; i < newData.length; i++) - { - newData[i] = FullDataPointUtil.remap(remappedEntryIds, sourceData[i]); - } - target.dataArrays[target.dataArrayIndex] = newData; - } - } - - /** Copies both ID data and mapping data. */ - public void deepCopyTo(SingleColumnFullDataAccessor target) - { - if (target.mapping.equals(this.mapping)) - { - System.arraycopy(this.dataArrays[this.dataArrayIndex], 0, target.dataArrays[target.dataArrayIndex], 0, this.dataArrays[this.dataArrayIndex].length); - } - else - { - int[] remappedEntryIds = target.mapping.mergeAndReturnRemappedEntityIds(this.mapping); - long[] sourceData = this.dataArrays[this.dataArrayIndex]; - // FIXME sourceData.length != 0 may not be a good solution and may end up breaking issues down the line, but fixes exceptions being fired here - if (sourceData != null && sourceData.length != 0) - { - long[] newData = new long[sourceData.length]; - for (int i = 0; i < newData.length; i++) - { - newData[i] = FullDataPointUtil.remap(remappedEntryIds, sourceData[i]); - } - target.dataArrays[target.dataArrayIndex] = newData; - } - } - } - - /** - * Replaces this column's data with data from the input {@link IFullDataAccessor}.
- * This is used to convert higher detail LOD data to lower detail LOD data. - */ - public void downsampleFrom(IFullDataAccessor source) - { - //TODO: average the data instead of just picking the first column - SingleColumnFullDataAccessor firstColumn = source.get(0); - firstColumn.deepCopyTo(this); - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index 7b52a298f..f5841435b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -42,7 +41,7 @@ import java.util.Arrays; /** * This data source contains every datapoint over its given {@link DhSectionPos}. * - * @see FullDataPointUtil + * @see FullDataPointUtil // FullDataSourceV1 */ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDataSource { @@ -246,7 +245,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa { for (int z = 0; z < this.width; z++) { - outputStream.writeInt(this.get(x, z).getSingleLength()); + outputStream.writeInt(this.get(x, z).length); } } @@ -258,11 +257,10 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa { for (int z = 0; z < this.width; z++) { - SingleColumnFullDataAccessor columnAccessor = this.get(x, z); - if (columnAccessor.doesColumnExist()) + long[] dataColumn = this.get(x, z); + if (dataColumn != null) { - long[] dataPointArray = columnAccessor.getRaw(); - for (long dataPoint : dataPointArray) + for (long dataPoint : dataColumn) { outputStream.writeLong(dataPoint); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 588c0776b..252a0b6eb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; @@ -147,16 +146,12 @@ public class NewFullDataSource implements IDataSource { for (int z = 0; z < WIDTH; z++) { - SingleColumnFullDataAccessor accessor = legacyData.get(x, z); - if (accessor.doesColumnExist()) + long[] dataColumn = legacyData.get(x, z); + if (dataColumn != null && dataColumn.length != 0) { - int index = relativePosToIndex(x, z); - dataPoints[index] = accessor.getRaw(); - // reverse the array so index 0 is the lowest, // this is necessary for later logic // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - long[] dataColumn = dataPoints[index]; for(int i = 0; i < dataColumn.length / 2; i++) { long temp = dataColumn[i]; @@ -164,6 +159,9 @@ public class NewFullDataSource implements IDataSource dataColumn[dataColumn.length - i - 1] = temp; } + int index = relativePosToIndex(x, z); + dataPoints[index] = dataColumn; + // convert the data point format boolean columnHasNonAirBlock = false; @@ -212,27 +210,24 @@ public class NewFullDataSource implements IDataSource { for (int z = 0; z < WIDTH; z++) { - SingleColumnFullDataAccessor legacyAccessor = legacyData.get(x, z); - if (legacyAccessor.doesColumnExist()) + long[] legacyDataColumn = legacyData.get(x, z); + if (legacyDataColumn != null && legacyDataColumn.length != 0) { - SingleColumnFullDataAccessor newAccessor = newFullDataSource.get(x, z); + long[] newDataColumn = newFullDataSource.get(x, z); - if (newAccessor == null) + if (newDataColumn == null) { LodUtil.assertNotReach("Accessor column mismatch"); } - else if (legacyAccessor.getRaw().length != newAccessor.getRaw().length) + else if (legacyDataColumn.length != newDataColumn.length) { LodUtil.assertNotReach("Accessor column length mismatch"); } else { - long[] legacyRaw = legacyAccessor.getRaw(); - long[] newRaw = newAccessor.getRaw(); - - for (int i = 0; i < legacyRaw.length; i++) + for (int i = 0; i < legacyDataColumn.length; i++) { - if (legacyRaw[i] != newRaw[i]) + if (legacyDataColumn[i] != newDataColumn[i]) { LodUtil.assertNotReach("Data mismatch"); } @@ -253,7 +248,7 @@ public class NewFullDataSource implements IDataSource // data // //======// - public SingleColumnFullDataAccessor get(int relX, int relZ) { return new SingleColumnFullDataAccessor(this.mapping, this.dataPoints, relativePosToIndex(relX, relZ)); } + public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override public boolean update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 39024ff91..8e1c776ad 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.IDataSource; @@ -318,14 +317,15 @@ public class ColumnRenderSource implements IDataSource ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); - SingleColumnFullDataAccessor fullArrayView = inputFullDataSource.get(x, z); + long[] dataColumn = inputFullDataSource.get(x, z); EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z); - if (fullArrayView != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) + if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { - FullDataToRenderDataTransformer.convertColumnData(level, + FullDataToRenderDataTransformer.convertColumnData( + level, inputFullDataSource.getMapping(), minBlockPos.x + x, minBlockPos.z + z, - columnArrayView, fullArrayView); + columnArrayView, dataColumn); dataChanged |= columnHash != columnArrayView.getDataHash(); this.fillDebugFlag(x, z, 1, 1, ColumnRenderSource.DebugSourceFlag.DIRECT); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index b9293258c..817c6802d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -22,7 +22,6 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import com.seibel.distanthorizons.api.enums.config.EBlocksToAvoid; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; @@ -118,8 +117,8 @@ public class FullDataToRenderDataTransformer throwIfThreadInterrupted(); ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - SingleColumnFullDataAccessor fullArrayView = fullDataSource.get(x, z); - convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView); + long[] dataColumn = fullDataSource.get(x, z); + convertColumnData(level, fullDataSource.getMapping(), baseX + x, baseZ + z, columnArrayView, dataColumn); } } @@ -157,14 +156,14 @@ public class FullDataToRenderDataTransformer // TODO what does this mean? - private static void iterateAndConvert(IDhClientLevel level, + private static void iterateAndConvert( + IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, - ColumnArrayView column, SingleColumnFullDataAccessor data) + ColumnArrayView renderColumnData, long[] fullColumnData) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); - FullDataPointIdMap fullDataMapping = data.getMapping(); HashSet blockStatesToIgnore = WRAPPER_FACTORY.getRendererIgnoredBlocks(level.getLevelWrapper()); boolean isVoid = true; @@ -172,9 +171,9 @@ public class FullDataToRenderDataTransformer int columnOffset = 0; // goes from the top down - for (int i = 0; i < data.getSingleLength(); i++) + for (int i = 0; i < fullColumnData.length; i++) { - long fullData = data.getSingle(i); + long fullData = fullColumnData[i]; int bottomY = FullDataPointUtil.getBottomY(fullData); int blockHeight = FullDataPointUtil.getHeight(fullData); int id = FullDataPointUtil.getId(fullData); @@ -259,40 +258,35 @@ public class FullDataToRenderDataTransformer // add the block isVoid = false; long columnData = RenderDataPointUtil.createDataPoint(bottomY + blockHeight, bottomY, color, skyLight, blockLight, block.getIrisBlockMaterialId()); - column.set(columnOffset, columnData); + renderColumnData.set(columnOffset, columnData); columnOffset++; } if (isVoid) { - column.set(0, RenderDataPointUtil.createVoidDataPoint()); + renderColumnData.set(0, RenderDataPointUtil.createVoidDataPoint()); } } // TODO what does this mean? - public static void convertColumnData(IDhClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleColumnFullDataAccessor fullArrayView) + public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, long[] fullDataColumn) { - if (fullArrayView == null || !fullArrayView.doesColumnExist()) - { - return; - } - - int dataTotalLength = fullArrayView.getSingleLength(); - if (dataTotalLength == 0) + if (fullDataColumn == null || fullDataColumn.length == 0) { return; } + int dataTotalLength = fullDataColumn.length; if (dataTotalLength > columnArrayView.verticalSize()) { ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); - iterateAndConvert(level, blockX, blockZ, totalColumnData, fullArrayView); + iterateAndConvert(level, fullDataMapping, blockX, blockZ, totalColumnData, fullDataColumn); columnArrayView.changeVerticalSizeFrom(totalColumnData); } else { - iterateAndConvert(level, blockX, blockZ, columnArrayView, fullArrayView); //Directly use the arrayView since it fits. + iterateAndConvert(level, fullDataMapping, blockX, blockZ, columnArrayView, fullDataColumn); //Directly use the arrayView since it fits. } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index c48569e79..3ed6b977e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.file.subDimMatching; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.SingleColumnFullDataAccessor; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -234,8 +233,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable { for (int z = 0; z < CompleteFullDataSource.WIDTH; z++) { - SingleColumnFullDataAccessor newColumn = newDataSource.get(x, z); - SingleColumnFullDataAccessor testColumn = testFullDataSource.get(x, z); + long[] newColumn = newDataSource.get(x, z); + long[] testColumn = testFullDataSource.get(x, z); if (newColumn != null && testColumn != null) { @@ -245,11 +244,11 @@ public class SubDimensionLevelMatcher implements AutoCloseable FullDataPointIdMap testDataMap = testFullDataSource.getMapping(); // use min to prevent going out of bounds - int minColumnIndex = Math.min(newColumn.getSingleLength(), testColumn.getSingleLength()); + int minColumnIndex = Math.min(newColumn.length, testColumn.length); for (int i = 0; i < minColumnIndex; i++) { - long newDataPoint = newColumn.getSingle(i); - long testDataPoint = testColumn.getSingle(i); + long newDataPoint = newColumn[i]; + long testDataPoint = testColumn[i]; int newId = FullDataPointUtil.getId(newDataPoint); int testId = FullDataPointUtil.getId(testDataPoint); @@ -298,7 +297,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable else if (newColumn != null) { // missing test column - totalDataPointCount += newColumn.getSingleLength(); + totalDataPointCount += newColumn.length; } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index ec62cf12a..f3e4b3164 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java index d64bfe83c..8be858116 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhSectionPos.java @@ -154,10 +154,16 @@ public class DhSectionPos // getters // //=========// - /** @return the corner with the smallest X and Z coordinate */ + /** + * @deprecated use DhSectionPos instead + * @return the corner with the smallest X and Z coordinate + */ @Deprecated public DhLodPos getMinCornerLodPos() { return this.getMinCornerLodPos((byte) (this.detailLevel - 1)); } - /** @return the corner with the smallest X and Z coordinate */ + /** + * @deprecated use DhSectionPos instead + * @return the corner with the smallest X and Z coordinate + */ @Deprecated public DhLodPos getMinCornerLodPos(byte returnDetailLevel) { From 4ec1dea1ba11035c5c4725a531766c78c4802598 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 19:34:36 -0500 Subject: [PATCH 050/183] merge CompleteFullDataSource and FullDataArrayAccessor --- .../accessor/FullDataArrayAccessor.java | 106 ------------------ .../sources/CompleteFullDataSource.java | 57 ++++++---- .../fullData/sources/NewFullDataSource.java | 4 +- 3 files changed, 38 insertions(+), 129 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java deleted file mode 100644 index f3fff6741..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/accessor/FullDataArrayAccessor.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.fullData.accessor; - -import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; - -/** - * Contains Full Data points and basic methods for getting and setting them.
- * Can be used standalone or as the base for Full data sources. - * - * @see CompleteFullDataSource - */ -@Deprecated // TODO merge into FullDataSourceV1 -public class FullDataArrayAccessor -{ - protected final FullDataPointIdMap mapping; - - /** A flattened 2D array (for the X and Z directions) containing an array for the Y direction. */ - protected final long[][] dataArrays; - - /** measured in data points */ - protected final int width; - /** measured in data points */ - protected final int dataWidth; - - /** index offset used when getting/setting data in {@link FullDataArrayAccessor#dataArrays}. */ - protected final int offset; - - - - //==============// - // constructors // - //==============// - - public FullDataArrayAccessor(FullDataPointIdMap mapping, long[][] dataArrays, int width) - { - if (dataArrays.length != width * width) - { - throw new IllegalArgumentException("tried constructing dataArrayView with invalid input!"); - } - - this.dataArrays = dataArrays; - this.width = width; - this.dataWidth = width; - this.mapping = mapping; - this.offset = 0; - } - - public FullDataArrayAccessor(FullDataArrayAccessor source, int width, int offsetX, int offsetZ) - { - if (source.width < width || source.width < width + offsetX || source.width < width + offsetZ) - { - throw new IllegalArgumentException("tried constructing dataArrayView subview with invalid input!"); - } - - this.dataArrays = source.dataArrays; - this.width = width; - this.dataWidth = source.dataWidth; - this.mapping = source.mapping; - this.offset = source.offset + offsetX * this.dataWidth + offsetZ; - } - - - - //=========// - // getters // - //=========// - - public FullDataPointIdMap getMapping() { return this.mapping; } - - public long[] get(int index) { return this.get(index / this.width, index % this.width); } - public long[] get(int relativeX, int relativeZ) - { - int dataArrayIndex = (relativeX * this.width) + relativeZ + this.offset; - if (dataArrayIndex >= this.dataArrays.length) - { - LodUtil.assertNotReach( - "FullDataArrayAccessor.get() called with a relative position that is outside the data source. \n" + - "source width: [" + this.width + "] source offset: [" + this.offset + "]\n" + - "given relative pos X: [" + relativeX + "] Z: [" + relativeZ + "]\n" + - "dataArrays.length: [" + this.dataArrays.length + "] dataArrayIndex: [" + dataArrayIndex + "]."); - } - - return this.dataArrays[dataArrayIndex]; - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java index f5841435b..8b691279b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java @@ -20,13 +20,12 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.accessor.FullDataArrayAccessor; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; @@ -41,9 +40,9 @@ import java.util.Arrays; /** * This data source contains every datapoint over its given {@link DhSectionPos}. * - * @see FullDataPointUtil // FullDataSourceV1 + * @see FullDataPointUtilV1 */ -public class CompleteFullDataSource extends FullDataArrayAccessor implements IDataSource +public class CompleteFullDataSource implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -52,6 +51,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa public static final int WIDTH = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); public static final byte DATA_FORMAT_VERSION = 3; + /** never used but should stay here. */ public static final String DATA_TYPE_NAME = "CompleteFullDataSource"; /** @@ -63,10 +63,16 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa private static final int NO_DATA_FLAG_BYTE = 0x00000001; + public final FullDataPointIdMap mapping; + public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; + + + /** A flattened 2D array (for the X and Z directions) containing an array for the Y direction. */ + private final long[][] dataArrays; + private DhSectionPos sectionPos; private boolean isEmpty = true; - public EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.EMPTY; @@ -77,18 +83,11 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa public static CompleteFullDataSource createEmpty(DhSectionPos pos) { return new CompleteFullDataSource(pos); } private CompleteFullDataSource(DhSectionPos sectionPos) { - super(new FullDataPointIdMap(sectionPos), new long[WIDTH * WIDTH][0], WIDTH); + this.dataArrays = new long[WIDTH * WIDTH][0]; + this.mapping = new FullDataPointIdMap(sectionPos); this.sectionPos = sectionPos; } - public CompleteFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data) - { - super(mapping, data, WIDTH); - LodUtil.assertTrue(data.length == WIDTH * WIDTH); - - this.sectionPos = pos; - this.isEmpty = false; - } @@ -132,6 +131,22 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa public boolean isEmpty() { return this.isEmpty; } + public long[] get(int index) { return this.get(index / WIDTH, index % WIDTH); } + public long[] get(int relativeX, int relativeZ) + { + int dataArrayIndex = (relativeX * WIDTH) + relativeZ; + if (dataArrayIndex >= this.dataArrays.length) + { + LodUtil.assertNotReach( + "FullDataArrayAccessor.get() called with a relative position that is outside the data source. \n" + + "given relative pos X: [" + relativeX + "] Z: [" + relativeZ + "]\n" + + "dataArrays.length: [" + this.dataArrays.length + "] dataArrayIndex: [" + dataArrayIndex + "]."); + } + + return this.dataArrays[dataArrayIndex]; + } + + //=================// // stream handling // @@ -145,7 +160,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa { // clear/overwrite the old data this.resizeDataStructuresForRepopulation(dto.pos); - this.getMapping().clear(dto.pos); + this.mapping.clear(dto.pos); // set the new data this.populateFromStream(dto, inputStream, level); @@ -189,7 +204,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException { outputStream.writeInt(this.getDataDetailLevel()); - outputStream.writeInt(this.width); + outputStream.writeInt(WIDTH); outputStream.writeInt(level.getMinY()); outputStream.writeByte(this.worldGenStep.value); @@ -241,9 +256,9 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa // Data array length - for (int x = 0; x < this.width; x++) + for (int x = 0; x < WIDTH; x++) { - for (int z = 0; z < this.width; z++) + for (int z = 0; z < WIDTH; z++) { outputStream.writeInt(this.get(x, z).length); } @@ -253,9 +268,9 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa // Data array content (only on non-empty columns) outputStream.writeInt(DATA_GUARD_BYTE); - for (int x = 0; x < this.width; x++) + for (int x = 0; x < WIDTH; x++) { - for (int z = 0; z < this.width; z++) + for (int z = 0; z < WIDTH; z++) { long[] dataColumn = this.get(x, z); if (dataColumn != null) @@ -288,7 +303,7 @@ public class CompleteFullDataSource extends FullDataArrayAccessor implements IDa long[][] dataPointArrays; - if (this.width == width) // attempt to use the existing dataArrays if possible + if (WIDTH == width) // attempt to use the existing dataArrays if possible { dataPointArrays = this.dataArrays; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 252a0b6eb..4c529157d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -175,7 +175,7 @@ public class NewFullDataSource implements IDataSource byte blockLight = (byte) FullDataPointUtilV1.getBlockLight(dataPoint); byte skyLight = (byte) FullDataPointUtilV1.getSkyLight(dataPoint); - IBlockStateWrapper blockState = legacyData.getMapping().getBlockStateWrapper(id); + IBlockStateWrapper blockState = legacyData.mapping.getBlockStateWrapper(id); if (blockState.isAir()) { // air shouldn't have any light, otherwise down sampling will look weird @@ -200,7 +200,7 @@ public class NewFullDataSource implements IDataSource } } - NewFullDataSource newFullDataSource = NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.getMapping(), dataPoints, columnGenerationSteps); + NewFullDataSource newFullDataSource = NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps); // should only be used if debugging, this is a very expensive operation From 98183a4e7584f2533c5f35baae3b64c4a076fb8d Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 19:40:36 -0500 Subject: [PATCH 051/183] Rename CompleteFullDataSource -> FullDataSourceV1 --- ...lDataSource.java => FullDataSourceV1.java} | 13 +++++----- .../fullData/sources/NewFullDataSource.java | 14 +++++------ ...andler.java => FullDataFileHandlerV1.java} | 24 +++++++++---------- .../fullDatafile/NewFullDataFileHandler.java | 10 ++++---- .../SubDimensionLevelMatcher.java | 6 ++--- .../core/sql/dto/LegacyDataSourceDTO.java | 10 +++++++- .../core/util/FullDataPointUtilV1.java | 6 ++--- 7 files changed, 45 insertions(+), 38 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/{CompleteFullDataSource.java => FullDataSourceV1.java} (95%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{LegacyFullDataFileHandler.java => FullDataFileHandlerV1.java} (77%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java similarity index 95% rename from core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java rename to core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 8b691279b..f15ae49a6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/CompleteFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -38,11 +38,12 @@ import java.io.*; import java.util.Arrays; /** - * This data source contains every datapoint over its given {@link DhSectionPos}. + * Formerly "CompleteFullDataSource".
+ * Should be fully populated, containing 1 data point for each column. * * @see FullDataPointUtilV1 */ -public class CompleteFullDataSource implements IDataSource +public class FullDataSourceV1 implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -80,8 +81,8 @@ public class CompleteFullDataSource implements IDataSource // constructors // //==============// - public static CompleteFullDataSource createEmpty(DhSectionPos pos) { return new CompleteFullDataSource(pos); } - private CompleteFullDataSource(DhSectionPos sectionPos) + public static FullDataSourceV1 createEmpty(DhSectionPos pos) { return new FullDataSourceV1(pos); } + private FullDataSourceV1(DhSectionPos sectionPos) { this.dataArrays = new long[WIDTH * WIDTH][0]; this.mapping = new FullDataPointIdMap(sectionPos); @@ -154,7 +155,7 @@ public class CompleteFullDataSource implements IDataSource /** * Clears and then overwrites any data in this object with the data from the given file and stream. - * This is expected to be used with an existing {@link CompleteFullDataSource} and can be used in place of a constructor to reuse an existing {@link CompleteFullDataSource} object. + * This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object. */ public void repopulateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { @@ -168,7 +169,7 @@ public class CompleteFullDataSource implements IDataSource /** * Overwrites any data in this object with the data from the given file and stream. - * This is expected to be used with an empty {@link CompleteFullDataSource} and functions similar to a constructor. + * This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor. */ public void populateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java index 4c529157d..4227ed3d1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java @@ -48,7 +48,7 @@ import java.util.concurrent.locks.ReentrantLock; * that can be pooled to reduce GC overhead * * @see FullDataPointUtil - * @see CompleteFullDataSource + * @see FullDataSourceV1 */ public class NewFullDataSource implements IDataSource { @@ -128,14 +128,14 @@ public class NewFullDataSource implements IDataSource public static NewFullDataSource createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } - public static NewFullDataSource createFromCompleteDataSource(CompleteFullDataSource legacyData) + public static NewFullDataSource createFromCompleteDataSource(FullDataSourceV1 legacyData) { - if (CompleteFullDataSource.WIDTH != WIDTH) + if (FullDataSourceV1.WIDTH != WIDTH) { throw new UnsupportedOperationException( "Unable to convert CompleteFullDataSource into NewFullDataSource. " + "Data sources have different data point widths and no converter is present. " + - "CompleteFullDataSource width ["+CompleteFullDataSource.WIDTH+"], NewFullDataSource width ["+WIDTH+"]."); + "CompleteFullDataSource width ["+ FullDataSourceV1.WIDTH+"], NewFullDataSource width ["+WIDTH+"]."); } @@ -780,12 +780,12 @@ public class NewFullDataSource implements IDataSource private static class Pooling { /** used when pooling data sources */ - private final ArrayList cachedSources = new ArrayList<>(); + private final ArrayList cachedSources = new ArrayList<>(); private final ReentrantLock cacheLock = new ReentrantLock(); /** @return null if no pooled source exists */ - public CompleteFullDataSource tryGetPooledSource() + public FullDataSourceV1 tryGetPooledSource() { try { @@ -811,7 +811,7 @@ public class NewFullDataSource implements IDataSource * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. */ - public void returnPooledDataSource(CompleteFullDataSource dataSource) + public void returnPooledDataSource(FullDataSourceV1 dataSource) { if (dataSource == null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV1.java similarity index 77% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV1.java index c95eeade1..441364b9d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/LegacyFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV1.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -36,8 +36,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -public class LegacyFullDataFileHandler - extends AbstractLegacyDataSourceHandler +public class FullDataFileHandlerV1 extends AbstractLegacyDataSourceHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -47,7 +46,7 @@ public class LegacyFullDataFileHandler // constructor // //=============// - public LegacyFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + public FullDataFileHandlerV1(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } @@ -74,21 +73,20 @@ public class LegacyFullDataFileHandler } @Override - protected CompleteFullDataSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException + protected FullDataSourceV1 createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException { - CompleteFullDataSource dataSource = CompleteFullDataSource.createEmpty(dto.pos); + FullDataSourceV1 dataSource = FullDataSourceV1.createEmpty(dto.pos); dataSource.populateFromStream(dto, dto.getInputStream(), this.level); return dataSource; } /** Creates a new data source using any DTOs already present in the database. */ @Deprecated @Override - protected CompleteFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) { return null; } - + protected FullDataSourceV1 createNewDataSourceFromExistingDtos(DhSectionPos pos) { return null; } @Deprecated @Override - protected CompleteFullDataSource makeEmptyDataSource(DhSectionPos pos) { return null; } + protected FullDataSourceV1 makeEmptyDataSource(DhSectionPos pos) { return null; } @@ -98,7 +96,7 @@ public class LegacyFullDataFileHandler @Deprecated @Override - public void writeDataSourceToFile(CompleteFullDataSource fullDataSource) throws IOException + public void writeDataSourceToFile(FullDataSourceV1 fullDataSource) throws IOException { throw new UnsupportedOperationException("Deprecated"); } @@ -109,15 +107,15 @@ public class LegacyFullDataFileHandler public int getDataSourceMigrationCount() { return ((LegacyFullDataRepo) this.repo).getMigrationCount(); } - public ArrayList getDataSourcesToMigrate(int limit) + public ArrayList getDataSourcesToMigrate(int limit) { - ArrayList dataSourceList = new ArrayList<>(); + ArrayList dataSourceList = new ArrayList<>(); ArrayList migrationPosList = ((LegacyFullDataRepo) this.repo).getPositionsToMigrate(limit); for (int i = 0; i < migrationPosList.size(); i++) { DhSectionPos pos = migrationPosList.get(i); - CompleteFullDataSource dataSource = this.get(pos); + FullDataSourceV1 dataSource = this.get(pos); if (dataSource != null) { dataSourceList.add(dataSource); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java index 4c1dc582e..43e5bf85e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java @@ -21,7 +21,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; @@ -72,7 +72,7 @@ public class NewFullDataFileHandler * vs gracefully shutting down the thread ourselves. */ protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); - protected final LegacyFullDataFileHandler legacyFileHandler; + protected final FullDataFileHandlerV1 legacyFileHandler; /** * Tracks which positions are currently being updated @@ -97,7 +97,7 @@ public class NewFullDataFileHandler public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); - this.legacyFileHandler = new LegacyFullDataFileHandler(level, saveStructure, saveDirOverride); + this.legacyFileHandler = new FullDataFileHandlerV1(level, saveStructure, saveDirOverride); DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); @@ -315,7 +315,7 @@ public class NewFullDataFileHandler LOGGER.info("Found ["+totalCount+"] data sources that need migration."); - ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { // keep going until every data source has been migrated @@ -327,7 +327,7 @@ public class NewFullDataFileHandler ArrayList> updateFutureList = new ArrayList<>(); for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) { - CompleteFullDataSource legacyDataSource = legacyDataSourceList.get(i); + FullDataSourceV1 legacyDataSource = legacyDataSourceList.get(i); try { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 3ed6b977e..61ceaed91 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -21,7 +21,7 @@ package com.seibel.distanthorizons.core.file.subDimMatching; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.structure.ClientOnlySaveStructure; @@ -229,9 +229,9 @@ public class SubDimensionLevelMatcher implements AutoCloseable // compare the data sources int equalDataPoints = 0; int totalDataPointCount = 0; - for (int x = 0; x < CompleteFullDataSource.WIDTH; x++) + for (int x = 0; x < FullDataSourceV1.WIDTH; x++) { - for (int z = 0; z < CompleteFullDataSource.WIDTH; z++) + for (int z = 0; z < FullDataSourceV1.WIDTH; z++) { long[] newColumn = newDataSource.get(x, z); long[] testColumn = testFullDataSource.get(x, z); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java index f68869e49..e89a3c7c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java @@ -21,6 +21,7 @@ package com.seibel.distanthorizons.core.sql.dto; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -30,7 +31,14 @@ import java.io.IOException; import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; -/** handles storing {@link ColumnRenderSource}'s in the database. */ +/** + * Handles storing {@link ColumnRenderSource}'s and {@link FullDataSourceV1}'s in the database. + * + * @deprecated {@link ColumnRenderSource} should store the actual GPU buffer data, + * at that point this DTO should be just used for {@link FullDataSourceV1} + * and could be renamed to FullDataSourceV1DTO + */ +@Deprecated public class LegacyDataSourceDTO implements IBaseDTO { public DhSectionPos pos; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java index aa40ee8a3..d6b4274bd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java @@ -1,13 +1,13 @@ package com.seibel.distanthorizons.core.util; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; /** * Only for Legacy support
* Used by DH versions 2.0.0 and 2.0.1.

* * Specifically used by the data sources:
- * - {@link CompleteFullDataSource} aka CompleteFullDataSource
+ * - {@link FullDataSourceV1} aka CompleteFullDataSource
* - (Deleted) HighDetailIncompleteFullDataSource
* - (Deleted) LowDetailIncompleteFullDataSource

* @@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.CompleteFull * ID ID ID ID ID ID ID ID <-- Bottom bits
* * - * @see CompleteFullDataSource + * @see FullDataSourceV1 * @see FullDataPointUtil */ public class FullDataPointUtilV1 From dcd16cb84b0eea6ed7d3052bd3f75bfdf0ab2630 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 19:53:01 -0500 Subject: [PATCH 052/183] rename NewFullDataSource -> FullDataSourceV2 and supporting objects --- .../methods/data/DhApiTerrainDataRepo.java | 20 +++--- .../fullData/sources/FullDataSourceV1.java | 9 ++- ...lDataSource.java => FullDataSourceV2.java} | 66 +++++++++---------- .../render/ColumnRenderSource.java | 8 +-- .../transformers/ChunkToLodBuilder.java | 12 ++-- .../FullDataToRenderDataTransformer.java | 20 +++--- .../transformers/LodDataBuilder.java | 22 +++---- .../file/AbstractLegacyDataSourceHandler.java | 6 +- .../file/AbstractNewDataSourceHandler.java | 6 +- .../core/file/IDataSource.java | 4 +- .../core/file/ISourceProvider.java | 4 +- .../DelayedFullDataSourceSaveCache.java | 12 ++-- ...andler.java => FullDataFileHandlerV2.java} | 34 +++++----- ...java => GeneratedFullDataFileHandler.java} | 20 +++--- .../fullDatafile/IFullDataSourceProvider.java | 14 ++-- ...er.java => RemoteFullDataFileHandler.java} | 6 +- .../renderfile/IRenderSourceProvider.java | 4 +- .../renderfile/RenderSourceFileHandler.java | 13 ++-- .../SubDimensionLevelMatcher.java | 22 +++---- .../core/generation/WorldGenerationQueue.java | 8 +-- .../tasks/IWorldGenTaskTracker.java | 4 +- .../generation/tasks/WorldGenTaskGroup.java | 6 +- .../core/level/AbstractDhLevel.java | 6 +- .../core/level/ClientLevelModule.java | 13 ++-- .../core/level/DhClientLevel.java | 15 ++--- .../core/level/DhClientServerLevel.java | 9 ++- .../core/level/DhServerLevel.java | 8 +-- .../distanthorizons/core/level/IDhLevel.java | 8 +-- .../core/level/IDhWorldGenLevel.java | 4 +- .../core/level/ServerLevelModule.java | 6 +- .../core/level/WorldGenModule.java | 12 ++-- .../distanthorizons/core/pos/DhLodPos.java | 4 +- ...ourceDTO.java => FullDataSourceV2DTO.java} | 33 +++++----- ...rceRepo.java => FullDataSourceV2Repo.java} | 16 ++--- .../core/util/FullDataPointUtilV1.java | 2 +- ...ointUtil.java => FullDataPointUtilV2.java} | 2 +- .../core/util/RenderDataPointUtil.java | 4 +- ...0-sqlite-createFullDataSourceV2Tables.sql} | 0 .../main/resources/sqlScripts/scriptList.txt | 2 +- core/src/test/java/tests/CompressionTest.java | 22 +++---- 40 files changed, 242 insertions(+), 244 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/{NewFullDataSource.java => FullDataSourceV2.java} (89%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{NewFullDataFileHandler.java => FullDataFileHandlerV2.java} (90%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{NewGeneratedFullDataFileHandler.java => GeneratedFullDataFileHandler.java} (93%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{NewRemoteFullDataFileHandler.java => RemoteFullDataFileHandler.java} (74%) rename core/src/main/java/com/seibel/distanthorizons/core/sql/dto/{NewFullDataSourceDTO.java => FullDataSourceV2DTO.java} (85%) rename core/src/main/java/com/seibel/distanthorizons/core/sql/repo/{NewFullDataSourceRepo.java => FullDataSourceV2Repo.java} (93%) rename core/src/main/java/com/seibel/distanthorizons/core/util/{FullDataPointUtil.java => FullDataPointUtilV2.java} (99%) rename core/src/main/resources/sqlScripts/{0020-sqlite-createGeneratedFullDataSourceTables.sql => 0020-sqlite-createFullDataSourceV2Tables.sql} (100%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 913588b4d..ce7ef834a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -27,12 +27,12 @@ import com.seibel.distanthorizons.api.interfaces.data.IDhApiTerrainDataRepo; import com.seibel.distanthorizons.api.objects.math.DhApiVec3i; import com.seibel.distanthorizons.core.api.internal.SharedApi; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RayCastUtil; import com.seibel.distanthorizons.core.world.AbstractDhWorld; @@ -211,7 +211,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo try { // attempt to get/generate the data source for this section - NewFullDataSource dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); + FullDataSourceV2 dataSource = level.getFullDataProvider().getAsync(sectionPos).get(); if (dataSource == null) { return DhApiResult.createFail("Unable to find/generate any data at the " + DhSectionPos.class.getSimpleName() + " [" + sectionPos + "]."); @@ -248,8 +248,8 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo if (dataPoint != 0) { int requestedY = nullableBlockYPos; - int bottomY = FullDataPointUtil.getBottomY(dataPoint) + levelMinimumHeight; - int height = FullDataPointUtil.getHeight(dataPoint); + int bottomY = FullDataPointUtilV2.getBottomY(dataPoint) + levelMinimumHeight; + int height = FullDataPointUtilV2.getHeight(dataPoint); int topY = bottomY + height; // does this datapoint contain the requested Y position? @@ -281,15 +281,15 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo private static DhApiTerrainDataPoint generateApiDatapoint(IDhApiLevelWrapper levelWrapper, FullDataPointIdMap mapping, byte detailLevel, long dataPoint) { - IBlockStateWrapper blockState = mapping.getBlockStateWrapper(FullDataPointUtil.getId(dataPoint)); - IBiomeWrapper biomeWrapper = mapping.getBiomeWrapper(FullDataPointUtil.getId(dataPoint)); + IBlockStateWrapper blockState = mapping.getBlockStateWrapper(FullDataPointUtilV2.getId(dataPoint)); + IBiomeWrapper biomeWrapper = mapping.getBiomeWrapper(FullDataPointUtilV2.getId(dataPoint)); - int bottomY = FullDataPointUtil.getBottomY(dataPoint) + levelWrapper.getMinHeight(); - int height = FullDataPointUtil.getHeight(dataPoint); + int bottomY = FullDataPointUtilV2.getBottomY(dataPoint) + levelWrapper.getMinHeight(); + int height = FullDataPointUtilV2.getHeight(dataPoint); int topY = bottomY + height; return new DhApiTerrainDataPoint(detailLevel, - FullDataPointUtil.getBlockLight(dataPoint), FullDataPointUtil.getSkyLight(dataPoint), + FullDataPointUtilV2.getBlockLight(dataPoint), FullDataPointUtilV2.getSkyLight(dataPoint), topY, bottomY, blockState, biomeWrapper); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index f15ae49a6..9a6f4c0e5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -39,9 +39,12 @@ import java.util.Arrays; /** * Formerly "CompleteFullDataSource".
- * Should be fully populated, containing 1 data point for each column. - * + * Should be fully populated, containing 1 data point for each column.

+ * + * Replaced by {@link FullDataSourceV2}. + * * @see FullDataPointUtilV1 + * @see FullDataSourceV2 */ public class FullDataSourceV1 implements IDataSource { @@ -98,7 +101,7 @@ public class FullDataSourceV1 implements IDataSource @Deprecated @Override - public boolean update(NewFullDataSource dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } + public boolean update(FullDataSourceV2 dataSource, IDhLevel level) { throw new UnsupportedOperationException("Deprecated"); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java similarity index 89% rename from core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java rename to core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 4227ed3d1..c0d311889 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/NewFullDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -23,11 +23,11 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; @@ -47,10 +47,10 @@ import java.util.concurrent.locks.ReentrantLock; * TODO create a child object that extends AutoClosable * that can be pooled to reduce GC overhead * - * @see FullDataPointUtil + * @see FullDataPointUtilV2 * @see FullDataSourceV1 */ -public class NewFullDataSource implements IDataSource +public class FullDataSourceV2 implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -100,8 +100,8 @@ public class NewFullDataSource implements IDataSource // constructors // //==============// - public static NewFullDataSource createEmpty(DhSectionPos pos) { return new NewFullDataSource(pos); } - private NewFullDataSource(DhSectionPos pos) + public static FullDataSourceV2 createEmpty(DhSectionPos pos) { return new FullDataSourceV2(pos); } + private FullDataSourceV2(DhSectionPos pos) { this.pos = pos; this.dataPoints = new long[WIDTH * WIDTH][]; @@ -113,8 +113,8 @@ public class NewFullDataSource implements IDataSource this.columnGenerationSteps = new byte[WIDTH * WIDTH]; } - public static NewFullDataSource createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new NewFullDataSource(pos, mapping, data, columnGenerationStep); } - private NewFullDataSource(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -126,16 +126,16 @@ public class NewFullDataSource implements IDataSource this.columnGenerationSteps = columnGenerationSteps; } - public static NewFullDataSource createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } + public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } - public static NewFullDataSource createFromCompleteDataSource(FullDataSourceV1 legacyData) + public static FullDataSourceV2 createFromCompleteDataSource(FullDataSourceV1 legacyData) { if (FullDataSourceV1.WIDTH != WIDTH) { throw new UnsupportedOperationException( - "Unable to convert CompleteFullDataSource into NewFullDataSource. " + + "Unable to convert ["+FullDataSourceV1.class.getSimpleName()+"] into ["+FullDataSourceV2.class.getSimpleName()+"]. " + "Data sources have different data point widths and no converter is present. " + - "CompleteFullDataSource width ["+ FullDataSourceV1.WIDTH+"], NewFullDataSource width ["+WIDTH+"]."); + "input width ["+ FullDataSourceV1.WIDTH+"], recipient width ["+WIDTH+"]."); } @@ -182,7 +182,7 @@ public class NewFullDataSource implements IDataSource blockLight = 0; } - long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, blockLight, skyLight); + long newDataPoint = FullDataPointUtilV2.encode(id, height, bottomY, blockLight, skyLight); dataColumn[i] = newDataPoint; @@ -200,7 +200,7 @@ public class NewFullDataSource implements IDataSource } } - NewFullDataSource newFullDataSource = NewFullDataSource.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps); + FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps); // should only be used if debugging, this is a very expensive operation @@ -213,7 +213,7 @@ public class NewFullDataSource implements IDataSource long[] legacyDataColumn = legacyData.get(x, z); if (legacyDataColumn != null && legacyDataColumn.length != 0) { - long[] newDataColumn = newFullDataSource.get(x, z); + long[] newDataColumn = fullDataSource.get(x, z); if (newDataColumn == null) { @@ -239,7 +239,7 @@ public class NewFullDataSource implements IDataSource } } - return newFullDataSource; + return fullDataSource; } @@ -251,8 +251,8 @@ public class NewFullDataSource implements IDataSource public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override - public boolean update(NewFullDataSource inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } - public boolean update(NewFullDataSource inputDataSource) + public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } + public boolean update(FullDataSourceV2 inputDataSource) { // shouldn't happen, but James saw it happen once if (inputDataSource.mapping.getMaxValidId() == 0) @@ -288,7 +288,7 @@ public class NewFullDataSource implements IDataSource throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); } - if (dataChanged && this.pos.getDetailLevel() < NewFullDataFileHandler.TOP_SECTION_DETAIL_LEVEL) + if (dataChanged && this.pos.getDetailLevel() < FullDataFileHandlerV2.TOP_SECTION_DETAIL_LEVEL) { // mark that this data source should be applied to its parent this.applyToParent = true; @@ -302,7 +302,7 @@ public class NewFullDataSource implements IDataSource return dataChanged; } - public boolean updateFromSameDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) + public boolean updateFromSameDetailLevel(FullDataSourceV2 inputDataSource, int[] remappedIds) { // both data sources should have the same detail level if (inputDataSource.pos.getDetailLevel() != this.pos.getDetailLevel()) @@ -351,7 +351,7 @@ public class NewFullDataSource implements IDataSource return dataChanged; } - public boolean updateFromOneBelowDetailLevel(NewFullDataSource inputDataSource, int[] remappedIds) + public boolean updateFromOneBelowDetailLevel(FullDataSourceV2 inputDataSource, int[] remappedIds) { if (inputDataSource.pos.getDetailLevel() + 1 != this.pos.getDetailLevel()) { @@ -406,7 +406,7 @@ public class NewFullDataSource implements IDataSource return dataChanged; } - private static long[] mergeInputTwoByTwoDataColumn(NewFullDataSource inputDataSource, int x, int z) + private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { ArrayList newColumnList = new ArrayList<>(); @@ -460,8 +460,8 @@ public class NewFullDataSource implements IDataSource } long datapoint = inputDataArray[dataPointIndex]; - int datapointMinY = FullDataPointUtil.getBottomY(datapoint); - int numbOfBlocksTall = FullDataPointUtil.getHeight(datapoint); + int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint); + int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint); int datapointMaxY = (datapointMinY + numbOfBlocksTall); @@ -504,9 +504,9 @@ public class NewFullDataSource implements IDataSource int[] mergeSkyLights = new int[4]; for (int i = 0; i < 4; i++) { - mergeIds[i] = FullDataPointUtil.getId(datapointsForYSlice[i]); - mergeBlockLights[i] = FullDataPointUtil.getBlockLight(datapointsForYSlice[i]); - mergeSkyLights[i] = FullDataPointUtil.getSkyLight(datapointsForYSlice[i]); + mergeIds[i] = FullDataPointUtilV2.getId(datapointsForYSlice[i]); + mergeBlockLights[i] = FullDataPointUtilV2.getBlockLight(datapointsForYSlice[i]); + mergeSkyLights[i] = FullDataPointUtilV2.getSkyLight(datapointsForYSlice[i]); } @@ -523,7 +523,7 @@ public class NewFullDataSource implements IDataSource { if (height != 0) { - newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + newColumnList.add(FullDataPointUtilV2.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); } lastId = id; @@ -537,7 +537,7 @@ public class NewFullDataSource implements IDataSource // add the last slice if present if (height != 0) { - newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + newColumnList.add(FullDataPointUtilV2.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); } @@ -560,7 +560,7 @@ public class NewFullDataSource implements IDataSource long[] dataColumn = this.dataPoints[dataPointIndex]; for (int i = 0; i < dataColumn.length; i++) { - dataColumn[i] = FullDataPointUtil.remap(remappedIds, dataColumn[i]); + dataColumn[i] = FullDataPointUtilV2.remap(remappedIds, dataColumn[i]); } } private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) @@ -705,7 +705,7 @@ public class NewFullDataSource implements IDataSource for (int i = 0; i < longArray.length; i++) { long dataPoint = longArray[i]; - int id = FullDataPointUtil.getId(dataPoint); + int id = FullDataPointUtilV2.getId(dataPoint); if (id > maxValidId) { LodUtil.assertNotReach("Column set with higher than possible ID. ID [" + id + "], max valid ID [" + maxValidId + "]."); @@ -738,11 +738,11 @@ public class NewFullDataSource implements IDataSource @Override public boolean equals(Object obj) { - if (!(obj instanceof NewFullDataSource)) + if (!(obj instanceof FullDataSourceV2)) { return false; } - NewFullDataSource other = (NewFullDataSource) obj; + FullDataSourceV2 other = (FullDataSourceV2) obj; if (!other.pos.equals(this.pos)) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 8e1c776ad..9eb58d3bb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -289,7 +289,7 @@ public class ColumnRenderSource implements IDataSource } @Override - public boolean update(NewFullDataSource inputFullDataSource, IDhClientLevel level) + public boolean update(FullDataSourceV2 inputFullDataSource, IDhClientLevel level) { final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputFullDataSource.getSectionPos() + "]. Error:"; @@ -310,9 +310,9 @@ public class ColumnRenderSource implements IDataSource int halfBlockWidth = inputFullDataSource.getSectionPos().getBlockWidth() / 2; DhBlockPos2D minBlockPos = new DhBlockPos2D(centerBlockPos.x - halfBlockWidth, centerBlockPos.z - halfBlockWidth); - for (int x = 0; x < NewFullDataSource.WIDTH; x++) + for (int x = 0; x < FullDataSourceV2.WIDTH; x++) { - for (int z = 0; z < NewFullDataSource.WIDTH; z++) + for (int z = 0; z < FullDataSourceV2.WIDTH; z++) { ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index a3a727f65..663a6f033 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -23,7 +23,7 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.pos.DhChunkPos; @@ -57,7 +57,7 @@ public class ChunkToLodBuilder implements AutoCloseable // data generation // //=================// - public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) + public CompletableFuture tryGenerateData(IChunkWrapper chunkWrapper) { if (chunkWrapper == null) { @@ -74,7 +74,7 @@ public class ChunkToLodBuilder implements AutoCloseable } // Otherwise, it means we're the first to do so. Let's submit our task to this entry. - CompletableFuture future = new CompletableFuture<>(); + CompletableFuture future = new CompletableFuture<>(); this.concurrentTaskToBuildList.addLast(new Task(chunkWrapper.getChunkPos(), future)); return future; } @@ -158,7 +158,7 @@ public class ChunkToLodBuilder implements AutoCloseable { if (LodDataBuilder.canGenerateLodFromChunk(latestChunk)) { - NewFullDataSource dataSource = LodDataBuilder.createGeneratedDataSource(latestChunk); + FullDataSourceV2 dataSource = LodDataBuilder.createGeneratedDataSource(latestChunk); if (dataSource != null) { task.future.complete(dataSource); @@ -233,11 +233,11 @@ public class ChunkToLodBuilder implements AutoCloseable private static class Task { public final DhChunkPos chunkPos; - public final CompletableFuture future; + public final CompletableFuture future; /** This is tracked so impossible tasks can be removed from the queue */ public long generationAttemptExpirationTimeMs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); - Task(DhChunkPos chunkPos, CompletableFuture future) + Task(DhChunkPos chunkPos, CompletableFuture future) { this.chunkPos = chunkPos; this.future = future; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 817c6802d..37cbfb931 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import com.seibel.distanthorizons.api.enums.config.EBlocksToAvoid; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -30,7 +30,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -41,7 +41,7 @@ import org.apache.logging.log4j.Logger; import java.util.HashSet; /** - * Handles converting {@link NewFullDataSource}'s to {@link ColumnRenderSource}. + * Handles converting {@link FullDataSourceV2}'s to {@link ColumnRenderSource}. */ public class FullDataToRenderDataTransformer { @@ -56,7 +56,7 @@ public class FullDataToRenderDataTransformer // public transformer interface // //==============================// - public static ColumnRenderSource transformFullDataToRenderSource(NewFullDataSource fullDataSource, IDhClientLevel level) + public static ColumnRenderSource transformFullDataToRenderSource(FullDataSourceV2 fullDataSource, IDhClientLevel level) { if (fullDataSource == null) { @@ -92,7 +92,7 @@ public class FullDataToRenderDataTransformer * @throws InterruptedException Can be caused by interrupting the thread upstream. * Generally thrown if the method is running after the client leaves the current world. */ - private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, NewFullDataSource fullDataSource) throws InterruptedException + private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, FullDataSourceV2 fullDataSource) throws InterruptedException { final DhSectionPos pos = fullDataSource.getSectionPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); @@ -174,11 +174,11 @@ public class FullDataToRenderDataTransformer for (int i = 0; i < fullColumnData.length; i++) { long fullData = fullColumnData[i]; - int bottomY = FullDataPointUtil.getBottomY(fullData); - int blockHeight = FullDataPointUtil.getHeight(fullData); - int id = FullDataPointUtil.getId(fullData); - int blockLight = FullDataPointUtil.getBlockLight(fullData); - int skyLight = FullDataPointUtil.getSkyLight(fullData); + int bottomY = FullDataPointUtilV2.getBottomY(fullData); + int blockHeight = FullDataPointUtilV2.getHeight(fullData); + int id = FullDataPointUtilV2.getId(fullData); + int blockLight = FullDataPointUtilV2.getBlockLight(fullData); + int skyLight = FullDataPointUtilV2.getSkyLight(fullData); // TODO how should corrupted data be handled? // TODO why is the full data corrupted in the first place? FullDataPointUtil hasn't been changed in a long time, could one of the full data point objects be corrupted? diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 26d34d8bb..b2af153c1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -24,12 +24,12 @@ import java.util.List; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtil; +import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; @@ -42,8 +42,8 @@ public class LodDataBuilder { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IBlockStateWrapper AIR = SingletonInjector.INSTANCE.get(IWrapperFactory.class).getAirBlockStateWrapper(); - /** how many chunks wide the {@link NewFullDataSource} is. */ - private static final int NUMB_OF_CHUNKS_WIDE = NewFullDataSource.WIDTH / LodUtil.CHUNK_WIDTH; + /** how many chunks wide the {@link FullDataSourceV2} is. */ + private static final int NUMB_OF_CHUNKS_WIDE = FullDataSourceV2.WIDTH / LodUtil.CHUNK_WIDTH; private static boolean getTopErrorLogged = false; @@ -53,7 +53,7 @@ public class LodDataBuilder // converters // //============// - public static NewFullDataSource createGeneratedDataSource(IChunkWrapper chunkWrapper) + public static FullDataSourceV2 createGeneratedDataSource(IChunkWrapper chunkWrapper) { if (!canGenerateLodFromChunk(chunkWrapper)) { @@ -70,7 +70,7 @@ public class LodDataBuilder sectionPosZ = (sectionPosZ < 0) ? ((sectionPosZ + 1) / NUMB_OF_CHUNKS_WIDE) - 1 : (sectionPosZ / NUMB_OF_CHUNKS_WIDE); DhSectionPos pos = new DhSectionPos(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ); - NewFullDataSource dataSource = NewFullDataSource.createEmpty(pos); + FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos); dataSource.markNotEmpty(); @@ -177,7 +177,7 @@ public class LodDataBuilder if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) { - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); biome = newBiome; blockState = newBlockState; mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); @@ -186,7 +186,7 @@ public class LodDataBuilder lastY = y; } } - longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); // reverse the array so index 0 is the lowest, // this is necessary for later logic @@ -211,9 +211,9 @@ public class LodDataBuilder /** @throws ClassCastException if an API user returns the wrong object type(s) */ - public static NewFullDataSource createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException + public static FullDataSourceV2 createFromApiChunkData(DhApiChunk dataPoints) throws ClassCastException { - NewFullDataSource accessor = NewFullDataSource.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ))); + FullDataSourceV2 accessor = FullDataSourceV2.createEmpty(new DhSectionPos(new DhChunkPos(dataPoints.chunkPosX, dataPoints.chunkPosZ))); for (int relZ = 0; relZ < LodUtil.CHUNK_WIDTH; relZ++) { for (int relX = 0; relX < LodUtil.CHUNK_WIDTH; relX++) @@ -237,7 +237,7 @@ public class LodDataBuilder (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); - packedDataPoints[index] = FullDataPointUtil.encode( + packedDataPoints[index] = FullDataPointUtilV2.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 7a3c65dc1..71ca44501 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -1,7 +1,7 @@ package com.seibel.distanthorizons.core.file; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -190,7 +190,7 @@ public abstract class AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) + public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) { ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) @@ -218,7 +218,7 @@ public abstract class AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) + public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) { ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); if (executor == null || executor.isTerminated()) @@ -202,7 +202,7 @@ public abstract class AbstractNewDataSourceHandler * After this method returns the inputData will be written to file. * @param updatePos the position to update */ - protected void updateDataSourceAtPos(DhSectionPos updatePos, NewFullDataSource inputData, boolean lockOnUpdatePos) + protected void updateDataSourceAtPos(DhSectionPos updatePos, FullDataSourceV2 inputData, boolean lockOnUpdatePos) { boolean methodLocked = false; // a lock is necessary to prevent two threads from writing to the same position at once, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index c4ae71ee1..e7a1fa5e8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -2,7 +2,7 @@ package com.seibel.distanthorizons.core.file; import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; @@ -26,7 +26,7 @@ public interface IDataSource extends IBaseDTO, TDhL { CompletableFuture getAsync(DhSectionPos pos); - CompletableFuture updateDataSourceAsync(NewFullDataSource inputData); + CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java index 733af7001..f92cfa7dc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java @@ -1,6 +1,6 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.TimerUtil; @@ -22,7 +22,7 @@ public class DelayedFullDataSourceSaveCache private static final Timer DELAY_UPDATE_TIMER = TimerUtil.CreateTimer("Delayed Full Datasource Save Timer"); - public final ConcurrentHashMap dataSourceByPosition = new ConcurrentHashMap<>(); + public final ConcurrentHashMap dataSourceByPosition = new ConcurrentHashMap<>(); private final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); private final ISaveDataSourceFunc onSaveTimeoutFunc; @@ -46,14 +46,14 @@ public class DelayedFullDataSourceSaveCache // update queue // //==============// - public void queueDataSourceForUpdateAndSave(NewFullDataSource inputDataSource) + public void queueDataSourceForUpdateAndSave(FullDataSourceV2 inputDataSource) { DhSectionPos dataSourcePos = inputDataSource.getSectionPos(); this.dataSourceByPosition.compute(dataSourcePos, (inputPos, temporaryDataSource) -> { if (temporaryDataSource == null) { - temporaryDataSource = NewFullDataSource.createEmpty(inputPos); + temporaryDataSource = FullDataSourceV2.createEmpty(inputPos); } temporaryDataSource.update(inputDataSource); @@ -67,7 +67,7 @@ public class DelayedFullDataSourceSaveCache try { - NewFullDataSource dataSourceToSave = DelayedFullDataSourceSaveCache.this.dataSourceByPosition.remove(dataSourcePos); + FullDataSourceV2 dataSourceToSave = DelayedFullDataSourceSaveCache.this.dataSourceByPosition.remove(dataSourcePos); if (dataSourceToSave != null) { DelayedFullDataSourceSaveCache.this.onSaveTimeoutFunc.save(dataSourceToSave); @@ -115,7 +115,7 @@ public class DelayedFullDataSourceSaveCache public interface ISaveDataSourceFunc { /** called after the timeout expires */ - void save(NewFullDataSource inputDataSource); + void save(FullDataSourceV2 inputDataSource); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java similarity index 90% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java index 43e5bf85e..ead649c20 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -30,8 +30,8 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; @@ -46,8 +46,8 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; -public class NewFullDataFileHandler - extends AbstractNewDataSourceHandler +public class FullDataFileHandlerV2 + extends AbstractNewDataSourceHandler implements IFullDataSourceProvider, IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -93,8 +93,8 @@ public class NewFullDataFileHandler // constructor // //=============// - public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public NewFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + public FullDataFileHandlerV2(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public FullDataFileHandlerV2(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); this.legacyFileHandler = new FullDataFileHandlerV1(level, saveStructure, saveDirOverride); @@ -120,11 +120,11 @@ public class NewFullDataFileHandler //====================// @Override - protected NewFullDataSourceRepo createRepo() + protected FullDataSourceV2Repo createRepo() { try { - return new NewFullDataSourceRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + return new FullDataSourceV2Repo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); } catch (SQLException e) { @@ -135,13 +135,13 @@ public class NewFullDataFileHandler } @Override - protected NewFullDataSourceDTO createDtoFromDataSource(NewFullDataSource dataSource) + protected FullDataSourceV2DTO createDtoFromDataSource(FullDataSourceV2 dataSource) { try { // when creating new data use the compressor currently selected in the config EDhApiDataCompressionMode compressionModeEnum = Config.Client.Advanced.LodBuilding.dataCompression.get(); - return NewFullDataSourceDTO.CreateFromDataSource(dataSource, compressionModeEnum); + return FullDataSourceV2DTO.CreateFromDataSource(dataSource, compressionModeEnum); } catch (IOException e) { @@ -151,17 +151,17 @@ public class NewFullDataFileHandler } @Override - protected NewFullDataSource createDataSourceFromDto(NewFullDataSourceDTO dto) throws InterruptedException, IOException + protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException { return dto.createDataSource(this.level.getLevelWrapper()); } @Override - protected NewFullDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos) + protected FullDataSourceV2 createNewDataSourceFromExistingDtos(DhSectionPos pos) { // TODO maybe just set children update flags to true? - return NewFullDataSource.createEmpty(pos); + return FullDataSourceV2.createEmpty(pos); } @Override - protected NewFullDataSource makeEmptyDataSource(DhSectionPos pos) { return NewFullDataSource.createEmpty(pos); } + protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); } @Override public boolean canQueueRetrieval() @@ -253,7 +253,7 @@ public class NewFullDataFileHandler childReadLock.lock(); this.lockedPosSet.add(childPos); - NewFullDataSource dataSource = this.get(childPos); + FullDataSourceV2 dataSource = this.get(childPos); this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); this.repo.setApplyToParent(childPos, false); } @@ -333,7 +333,7 @@ public class NewFullDataFileHandler { // convert the legacy data source to the new format, // this is a relatively cheap operation - NewFullDataSource newDataSource = NewFullDataSource.createFromCompleteDataSource(legacyDataSource); + FullDataSourceV2 newDataSource = FullDataSourceV2.createFromCompleteDataSource(legacyDataSource); newDataSource.applyToParent = true; // the actual update process can be moderately expensive due to having to update diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java similarity index 93% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index 2f08a75bd..d3c338711 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/NewGeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -21,7 +21,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; @@ -43,7 +43,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; -public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler implements IDebugRenderable +public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -61,7 +61,7 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl // constructor // //=============// - public NewGeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public GeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } @@ -248,11 +248,11 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl EDhApiWorldGenerationStep currentMinWorldGenStep = EDhApiWorldGenerationStep.LIGHT; checkWorldGenLoop: - for (int x = 0; x < NewFullDataSource.WIDTH; x++) + for (int x = 0; x < FullDataSourceV2.WIDTH; x++) { - for (int z = 0; z < NewFullDataSource.WIDTH; z++) + for (int z = 0; z < FullDataSourceV2.WIDTH; z++) { - int index = NewFullDataSource.relativePosToIndex(x, z); + int index = FullDataSourceV2.relativePosToIndex(x, z); byte genStepValue = columnGenerationSteps[index]; if (genStepValue < currentMinWorldGenStep.value) @@ -338,16 +338,16 @@ public class NewGeneratedFullDataFileHandler extends NewFullDataFileHandler impl public boolean isMemoryAddressValid() { return true; } @Override - public Consumer getChunkDataConsumer() + public Consumer getChunkDataConsumer() { return (chunkSizedFullDataSource) -> { - NewGeneratedFullDataFileHandler.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(chunkSizedFullDataSource); + GeneratedFullDataFileHandler.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(chunkSizedFullDataSource); }; } } - private void onDataSourceSave(NewFullDataSource fullDataSource) - { NewGeneratedFullDataFileHandler.this.updateDataSourceAsync(fullDataSource); } + private void onDataSourceSave(FullDataSourceV2 fullDataSource) + { GeneratedFullDataFileHandler.this.updateDataSourceAsync(fullDataSource); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java index cf1a7bdbb..7b5a81291 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.file.fullDatafile; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -30,15 +30,15 @@ import java.util.ArrayList; import java.util.concurrent.CompletableFuture; /** - * Handles reading, writing, and updating {@link NewFullDataSource}'s.
+ * Handles reading, writing, and updating {@link FullDataSourceV2}'s.
* Should be backed by a database handled by a {@link LegacyFullDataRepo}. */ -public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable +public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable { - CompletableFuture getAsync(DhSectionPos pos); - NewFullDataSource get(DhSectionPos pos); + CompletableFuture getAsync(DhSectionPos pos); + FullDataSourceV2 get(DhSectionPos pos); - CompletableFuture updateDataSourceAsync(NewFullDataSource chunkData); + CompletableFuture updateDataSourceAsync(FullDataSourceV2 chunkData); /** @return -1 if this provider never has unsaved data sources */ default int getUnsavedDataSourceCount() { return -1; } @@ -49,7 +49,7 @@ public interface IFullDataSourceProvider extends ISourceProvider getAsync(DhSectionPos pos); - CompletableFuture updateDataSourceAsync(NewFullDataSource dataSource); + CompletableFuture updateDataSourceAsync(FullDataSourceV2 dataSource); /** Deletes any data stored in the render cache so it can be re-created */ void deleteRenderCache(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index d7ae22c10..b0510559b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -19,11 +19,11 @@ package com.seibel.distanthorizons.core.file.renderfile; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -49,7 +49,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler updateDataSourceAsync(NewFullDataSource inputDataSource) + public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) { // TODO once the legacy data provider has been replaced this can be removed this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); @@ -140,8 +140,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler chunkDataConsumer + Consumer chunkDataConsumer ) { EDhApiDistantGeneratorMode generatorMode = Config.Client.Advanced.WorldGenerator.distantGeneratorMode.get(); @@ -450,7 +450,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender try { IChunkWrapper chunk = WRAPPER_FACTORY.createChunkWrapper(generatedObjectArray); - NewFullDataSource dataSource = LodDataBuilder.createGeneratedDataSource(chunk); + FullDataSourceV2 dataSource = LodDataBuilder.createGeneratedDataSource(chunk); LodUtil.assertTrue(dataSource != null); chunkDataConsumer.accept(dataSource); } @@ -475,7 +475,7 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender { try { - NewFullDataSource dataSource = LodDataBuilder.createFromApiChunkData(dataPoints); + FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints); chunkDataConsumer.accept(dataSource); } catch (ClassCastException e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java index 4568f9821..d0eecf2ea 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/IWorldGenTaskTracker.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.generation.tasks; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import java.util.function.Consumer; @@ -32,6 +32,6 @@ public interface IWorldGenTaskTracker /** Returns true if the task hasn't been garbage collected. */ boolean isMemoryAddressValid(); - Consumer getChunkDataConsumer(); + Consumer getChunkDataConsumer(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java index 39c145c01..ea7b5b91e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/tasks/WorldGenTaskGroup.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.generation.tasks; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.pos.DhSectionPos; import java.util.Iterator; @@ -46,13 +46,13 @@ public final class WorldGenTaskGroup this.dataDetail = dataDetail; } - public void consumeChunkData(NewFullDataSource chunkSizedFullDataView) + public void consumeChunkData(FullDataSourceV2 chunkSizedFullDataView) { Iterator tasks = this.worldGenTasks.iterator(); while (tasks.hasNext()) { WorldGenTask task = tasks.next(); - Consumer chunkDataConsumer = task.taskTracker.getChunkDataConsumer(); + Consumer chunkDataConsumer = task.taskTracker.getChunkDataConsumer(); if (chunkDataConsumer == null) { tasks.remove(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java index d4a9d8a27..46c7ad97d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.abstractEvents.DhApiChunkModifiedEvent; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.transformers.ChunkToLodBuilder; import com.seibel.distanthorizons.core.file.fullDatafile.DelayedFullDataSourceSaveCache; import com.seibel.distanthorizons.core.pos.DhChunkPos; @@ -59,7 +59,7 @@ public abstract class AbstractDhLevel implements IDhLevel @Override public void updateChunkAsync(IChunkWrapper chunkWrapper) { - NewFullDataSource dataSource = NewFullDataSource.createFromChunk(chunkWrapper); + FullDataSourceV2 dataSource = FullDataSourceV2.createFromChunk(chunkWrapper); if (dataSource == null) { // This can happen if, among other reasons, a chunk save is superseded by a later event @@ -81,7 +81,7 @@ public abstract class AbstractDhLevel implements IDhLevel this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(dataSource); } - private void onDataSourceSave(NewFullDataSource fullDataSource) + private void onDataSourceSave(FullDataSourceV2 fullDataSource) { this.updateDataSourcesAsync(fullDataSource).thenRun(() -> { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index c1648dab7..d5cddb202 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -22,12 +22,11 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; -import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -47,14 +46,14 @@ import java.io.Closeable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc +public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); private final IDhClientLevel parentClientLevel; - public final NewFullDataFileHandler fullDataSourceProvider; + public final FullDataFileHandlerV2 fullDataSourceProvider; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); public final F3Screen.NestedMessage f3Message; @@ -210,9 +209,9 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // data handling // //===============// - public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } @Override - public void OnDataSourceUpdated(NewFullDataSource updatedFullDataSource) + public void OnDataSourceUpdated(FullDataSourceV2 updatedFullDataSource) { // if rendering also update the render sources ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 439b0228d..285008ab2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -20,10 +20,9 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.NewRemoteFullDataFileHandler; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; @@ -46,7 +45,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public final ClientLevelModule clientside; public final IClientLevelWrapper levelWrapper; public final AbstractSaveStructure saveStructure; - public final NewRemoteFullDataFileHandler dataFileHandler; + public final RemoteFullDataFileHandler dataFileHandler; @@ -59,7 +58,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel { this.levelWrapper = clientLevelWrapper; this.saveStructure = saveStructure; - this.dataFileHandler = new NewRemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); + this.dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); this.clientside = new ClientLevelModule(this); if (enableRendering) @@ -119,7 +118,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public ILevelWrapper getLevelWrapper() { return levelWrapper; } @Override - public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.clientside.updateDataSourcesAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientside.updateDataSourcesAsync(data); } @Override public int getMinY() { return levelWrapper.getMinHeight(); } @@ -138,7 +137,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel //=======================// @Override - public NewFullDataFileHandler getFullDataProvider() { return this.dataFileHandler; } + public FullDataFileHandlerV2 getFullDataProvider() { return this.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index f2cf7a8e8..5b5a24a62 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -20,10 +20,9 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; import com.seibel.distanthorizons.core.render.LodRenderSection; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; @@ -175,7 +174,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public NewFullDataFileHandler getFullDataProvider() { return this.serverside.dataFileHandler; } + public FullDataFileHandlerV2 getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() @@ -187,7 +186,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public boolean hasSkyLight() { return this.serverLevelWrapper.hasSkyLight(); } @Override - public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.clientside.updateDataSourcesAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientside.updateDataSourcesAsync(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index f3e4b3164..5f7f06136 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -19,8 +19,8 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -51,7 +51,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public void serverTick() { this.chunkToLodBuilder.tick(); } @Override - public CompletableFuture updateDataSourcesAsync(NewFullDataSource data) { return this.getFullDataProvider().updateDataSourceAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.getFullDataProvider().updateDataSourceAsync(data); } @Override public int getMinY() { return getLevelWrapper().getMinHeight(); } @@ -93,7 +93,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public NewFullDataFileHandler getFullDataProvider() { return this.serverside.dataFileHandler; } + public FullDataFileHandlerV2 getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index f8baafd4a..7dd2450e6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -19,8 +19,8 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.NewFullDataFileHandler; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; @@ -39,13 +39,13 @@ public interface IDhLevel extends AutoCloseable void updateChunkAsync(IChunkWrapper chunk); - NewFullDataFileHandler getFullDataProvider(); + FullDataFileHandlerV2 getFullDataProvider(); AbstractSaveStructure getSaveStructure(); boolean hasSkyLight(); - CompletableFuture updateDataSourcesAsync(NewFullDataSource data); + CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data); /** * this number is generally related to how many data sources have been updated diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java index 118d4a272..94108b380 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; -public interface IDhWorldGenLevel extends IDhLevel, NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener +public interface IDhWorldGenLevel extends IDhLevel, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener { void doWorldGen(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java index cdc32f285..9e8b132be 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.distanthorizons.core.config.AppliedConfigState; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.BatchGenerator; import com.seibel.distanthorizons.core.generation.WorldGenerationQueue; @@ -36,7 +36,7 @@ public class ServerLevelModule public final IDhServerLevel parentServerLevel; public final AbstractSaveStructure saveStructure; - public final NewGeneratedFullDataFileHandler dataFileHandler; + public final GeneratedFullDataFileHandler dataFileHandler; public final AppliedConfigState worldGeneratorEnabledConfig; public final WorldGenModule worldGenModule; @@ -47,7 +47,7 @@ public class ServerLevelModule { this.parentServerLevel = parentServerLevel; this.saveStructure = saveStructure; - this.dataFileHandler = new NewGeneratedFullDataFileHandler(parentServerLevel, saveStructure); + this.dataFileHandler = new GeneratedFullDataFileHandler(parentServerLevel, saveStructure); this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration); this.worldGenModule = new WorldGenModule(this.dataFileHandler, this.parentServerLevel); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index 0d7f5bf98..302e31b1a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.NewGeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -34,15 +34,15 @@ public class WorldGenModule implements Closeable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final NewGeneratedFullDataFileHandler dataFileHandler; - private final NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; + private final GeneratedFullDataFileHandler dataFileHandler; + private final GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; private final AtomicReference worldGenStateRef = new AtomicReference<>(); private final F3Screen.DynamicMessage worldGenF3Message; - public WorldGenModule(NewGeneratedFullDataFileHandler dataFileHandler, NewGeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) + public WorldGenModule(GeneratedFullDataFileHandler dataFileHandler, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) { this.dataFileHandler = dataFileHandler; this.onWorldGenCompleteListener = onWorldGenCompleteListener; @@ -71,7 +71,7 @@ public class WorldGenModule implements Closeable // world gen control // //===================// - public void startWorldGen(NewGeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) + public void startWorldGen(GeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) { // create the new world generator if (!this.worldGenStateRef.compareAndSet(null, newWgs)) @@ -83,7 +83,7 @@ public class WorldGenModule implements Closeable dataFileHandler.setWorldGenerationQueue(newWgs.worldGenerationQueue); } - public void stopWorldGen(NewGeneratedFullDataFileHandler dataFileHandler) + public void stopWorldGen(GeneratedFullDataFileHandler dataFileHandler) { AbstractWorldGenState worldGenState = this.worldGenStateRef.get(); if (worldGenState == null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java index 1706e9365..cbc08fb64 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.pos; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.LodUtil; import org.jetbrains.annotations.NotNull; @@ -122,7 +122,7 @@ public class DhLodPos implements Comparable public DhLodPos getDhSectionRelativePositionForDetailLevel() throws IllegalArgumentException { return this.getDhSectionRelativePositionForDetailLevel(this.detailLevel); } /** * Returns a DhLodPos with the given detail level and an X/Z position somewhere between (0,0) and (63,63). - * This is done to access specific sections from a {@link NewFullDataSource} where LOD columns are stored + * This is done to access specific sections from a {@link FullDataSourceV2} where LOD columns are stored * in 64 x 64 blocks. * * @throws IllegalArgumentException if this position's detail level is lower than the output detail level diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java similarity index 85% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index d61f0e2a2..dc631e72a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/NewFullDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -21,9 +21,8 @@ package com.seibel.distanthorizons.core.sql.dto; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; @@ -36,8 +35,8 @@ import java.io.IOException; import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; -/** handles storing {@link NewFullDataSource}'s in the database. */ -public class NewFullDataSourceDTO implements IBaseDTO +/** handles storing {@link FullDataSourceV2}'s in the database. */ +public class FullDataSourceV2DTO implements IBaseDTO { public DhSectionPos pos; @@ -67,20 +66,20 @@ public class NewFullDataSourceDTO implements IBaseDTO // constructor // //=============// - public static NewFullDataSourceDTO CreateFromDataSource(NewFullDataSource dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException + public static FullDataSourceV2DTO CreateFromDataSource(FullDataSourceV2 dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException { CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping(), compressionModeEnum); - return new NewFullDataSourceDTO( + return new FullDataSourceV2DTO( dataSource.getSectionPos(), - checkedDataPointArray.checksum, dataSource.columnGenerationSteps, NewFullDataSource.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, dataSource.columnGenerationSteps, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY ); } - public NewFullDataSourceDTO( + public FullDataSourceV2DTO( DhSectionPos pos, int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] dataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, @@ -111,22 +110,22 @@ public class NewFullDataSourceDTO implements IBaseDTO // data source population // //========================// - public NewFullDataSource createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException - { return this.populateDataSource(NewFullDataSource.createEmpty(this.pos), levelWrapper); } + public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), levelWrapper); } - public NewFullDataSource populateDataSource(NewFullDataSource dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } /** * May be missing one or more data fields.
* Designed to be used without access to Minecraft or any supporting objects. */ - public NewFullDataSource createUnitTestDataSource() throws IOException, InterruptedException - { return this.internalPopulateDataSource(NewFullDataSource.createEmpty(this.pos), null, true); } + public FullDataSourceV2 createUnitTestDataSource() throws IOException, InterruptedException + { return this.internalPopulateDataSource(FullDataSourceV2.createEmpty(this.pos), null, true); } - private NewFullDataSource internalPopulateDataSource(NewFullDataSource dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException + private FullDataSourceV2 internalPopulateDataSource(FullDataSourceV2 dataSource, ILevelWrapper levelWrapper, boolean unitTest) throws IOException, InterruptedException { - if (NewFullDataSource.DATA_FORMAT_VERSION != this.dataFormatVersion) + if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { throw new IllegalStateException("There should only be one data format right now anyway."); } @@ -175,7 +174,7 @@ public class NewFullDataSourceDTO implements IBaseDTO // write the data - int dataArrayLength = NewFullDataSource.WIDTH * NewFullDataSource.WIDTH; + int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; for (int xz = 0; xz < dataArrayLength; xz++) { long[] dataColumn = dataArray[xz]; @@ -206,7 +205,7 @@ public class NewFullDataSourceDTO implements IBaseDTO // read the data - int dataArrayLength = NewFullDataSource.WIDTH * NewFullDataSource.WIDTH; + int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; long[][] dataArray = new long[dataArrayLength][]; for (int xz = 0; xz < dataArray.length; xz++) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java similarity index 93% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 410c5a3e4..ebfa1240d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/NewFullDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -21,7 +21,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -29,11 +29,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -public class NewFullDataSourceRepo extends AbstractDhRepo +public class FullDataSourceV2Repo extends AbstractDhRepo { - public NewFullDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + public FullDataSourceV2Repo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, NewFullDataSourceDTO.class); + super(databaseType, databaseLocation, FullDataSourceV2DTO.class); } @@ -55,7 +55,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo objectMap) throws ClassCastException + public FullDataSourceV2DTO convertDictionaryToDto(Map objectMap) throws ClassCastException { byte detailLevel = (Byte) objectMap.get("DetailLevel"); byte sectionDetailLevel = (byte) (detailLevel + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); @@ -80,7 +80,7 @@ public class NewFullDataSourceRepo extends AbstractDhRepo * * @see FullDataSourceV1 - * @see FullDataPointUtil + * @see FullDataPointUtilV2 */ public class FullDataPointUtilV1 { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java similarity index 99% rename from core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java rename to core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java index 1e1ebfc49..0d9540567 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java @@ -49,7 +49,7 @@ import org.jetbrains.annotations.Contract; * @see RenderDataPointUtil * @see FullDataPointUtilV1 */ -public class FullDataPointUtil +public class FullDataPointUtilV2 { /** Represents the data held by an empty data point */ public static final int EMPTY_DATA_POINT = 0; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 240a0b4aa..316255595 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -29,7 +29,7 @@ import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDat * A helper class that is used to access the data from a long * formatted as a render data point.

* - * To access data from a long formatted as a full data point see: {@link FullDataPointUtil} + * To access data from a long formatted as a full data point see: {@link FullDataPointUtilV2} * * DataPoint Format:
* @@ -55,7 +55,7 @@ import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDat * BL BL BL BL SL SL SL SL |
* * - * @see FullDataPointUtil + * @see FullDataPointUtilV2 */ public class RenderDataPointUtil { diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql similarity index 100% rename from core/src/main/resources/sqlScripts/0020-sqlite-createGeneratedFullDataSourceTables.sql rename to core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index 2a2f2e905..b41618ae7 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -1,3 +1,3 @@ 0010-sqlite-createInitialDataTables.sql -0020-sqlite-createGeneratedFullDataSourceTables.sql +0020-sqlite-createFullDataSourceV2Tables.sql diff --git a/core/src/test/java/tests/CompressionTest.java b/core/src/test/java/tests/CompressionTest.java index 228c6bc92..68319522c 100644 --- a/core/src/test/java/tests/CompressionTest.java +++ b/core/src/test/java/tests/CompressionTest.java @@ -20,10 +20,10 @@ package tests; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.NewFullDataSource; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.NewFullDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.NewFullDataSourceRepo; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import org.junit.Assert; import java.io.*; @@ -155,7 +155,7 @@ public class CompressionTest // byte[] dictionary; // { // String uncompressedDatabaseFilePath = TEST_DIR + "/" + UNCOMPRESSED_DB_FILE_NAME; - // NewFullDataSourceRepo uncompressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", uncompressedDatabaseFilePath); + // FullDataSourceV2Repo uncompressedRepo = new FullDataSourceV2Repo("jdbc:sqlite", uncompressedDatabaseFilePath); // ArrayList positionList = uncompressedRepo.getAllPositions(); // // // sample size of 10 MB or less @@ -165,7 +165,7 @@ public class CompressionTest // for (int i = 0; i < positionList.size(); i++) // { // DhSectionPos pos = positionList.get(i); - // NewFullDataSourceDTO uncompressedDto = uncompressedRepo.getByKey(pos); + // FullDataSourceV2DTO uncompressedDto = uncompressedRepo.getByKey(pos); // // dictTrainer.addSample(uncompressedDto.dataByteArray); // } @@ -260,7 +260,7 @@ public class CompressionTest File uncompressedDatabaseFile = new File(uncompressedDatabaseFilePath); Assert.assertTrue(uncompressedDatabaseFile.exists()); - NewFullDataSourceRepo uncompressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", uncompressedDatabaseFilePath); + FullDataSourceV2Repo uncompressedRepo = new FullDataSourceV2Repo("jdbc:sqlite", uncompressedDatabaseFilePath); String compressedDatabaseFilePath = TEST_DIR + "/output/" + DB_FILE_NAME_PREFIX + "_" + compressorName + ".sqlite"; @@ -268,7 +268,7 @@ public class CompressionTest compressedDatabaseFile.mkdirs(); compressedDatabaseFile.delete(); Assert.assertTrue(!compressedDatabaseFile.exists()); - NewFullDataSourceRepo compressedRepo = new NewFullDataSourceRepo("jdbc:sqlite", compressedDatabaseFilePath); + FullDataSourceV2Repo compressedRepo = new FullDataSourceV2Repo("jdbc:sqlite", compressedDatabaseFilePath); @@ -292,9 +292,9 @@ public class CompressionTest // uncompressed input // - NewFullDataSourceDTO uncompressedDto = uncompressedRepo.getByKey(pos); + FullDataSourceV2DTO uncompressedDto = uncompressedRepo.getByKey(pos); Assert.assertEquals(uncompressedDto.compressionModeEnum, EDhApiDataCompressionMode.UNCOMPRESSED); - NewFullDataSource uncompressedDataSource = uncompressedDto.createUnitTestDataSource(); + FullDataSourceV2 uncompressedDataSource = uncompressedDto.createUnitTestDataSource(); long uncompressedDtoSize = uncompressedRepo.getDataSizeInBytes(pos); minUncompressedDtoSizeInBytes = Math.min(uncompressedDtoSize, minUncompressedDtoSizeInBytes); @@ -307,7 +307,7 @@ public class CompressionTest long startWriteNanoTime = System.nanoTime(); - NewFullDataSourceDTO compressedDto = NewFullDataSourceDTO.CreateFromDataSource(uncompressedDataSource, compressionMode); + FullDataSourceV2DTO compressedDto = FullDataSourceV2DTO.CreateFromDataSource(uncompressedDataSource, compressionMode); compressedRepo.save(compressedDto); long endWriteNanoTime = System.nanoTime(); @@ -326,7 +326,7 @@ public class CompressionTest long startReadNanoTime = System.nanoTime(); compressedDto = compressedRepo.getByKey(pos); - NewFullDataSource compressedDataSource = compressedDto.createUnitTestDataSource(); + FullDataSourceV2 compressedDataSource = compressedDto.createUnitTestDataSource(); long endReadMsTime = System.nanoTime(); totalReadTimeInNano += (endReadMsTime - startReadNanoTime); From e14e122b6c24445ec9ee96b9b98c8f1fe460b5db Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 20:19:24 -0500 Subject: [PATCH 053/183] Refactor FullDataSourceV2 variables --- .../methods/data/DhApiTerrainDataRepo.java | 2 +- .../fullData/sources/FullDataSourceV1.java | 3 -- .../fullData/sources/FullDataSourceV2.java | 34 ++++++++++--------- .../render/ColumnRenderSource.java | 7 +--- .../FullDataToRenderDataTransformer.java | 4 +-- .../transformers/LodDataBuilder.java | 10 +++--- .../file/AbstractLegacyDataSourceHandler.java | 3 +- .../core/file/IDataSource.java | 4 --- .../SubDimensionLevelMatcher.java | 4 +-- .../core/sql/dto/FullDataSourceV2DTO.java | 8 ++--- 10 files changed, 35 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index ce7ef834a..933b29210 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -219,7 +219,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo else { // attempt to get the LOD data from the data source - FullDataPointIdMap mapping = dataSource.getMapping(); + FullDataPointIdMap mapping = dataSource.mapping; long[] dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 9a6f4c0e5..22b9db0c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -127,10 +127,7 @@ public class FullDataSourceV1 implements IDataSource @Override public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - @Override public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - @Override - public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { return this.worldGenStep; } public boolean isEmpty() { return this.isEmpty; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index c0d311889..9f012419c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -54,7 +54,7 @@ public class FullDataSourceV2 implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ - private static final boolean RUN_UPDATE_DEV_VALIDATION = false; //ModInfo.IS_DEV_BUILD; + private static final boolean RUN_UPDATE_DEV_VALIDATION = false; private static final boolean RUN_V1_MIGRATION_VALIDATION = false; /** measured in data columns */ @@ -64,18 +64,21 @@ public class FullDataSourceV2 implements IDataSource - // TODO make these fields private + private int cachedHashCode = 0; + private DhSectionPos pos; @Override public DhSectionPos getKey() { return this.pos; } + + public final FullDataPointIdMap mapping; + + public long lastModifiedUnixDateTime; public long createdUnixDateTime; public int levelMinY; - private int cachedHashCode = 0; - /** * stores how far each column has been generated should start with {@link EDhApiWorldGenerationStep#EMPTY} * @see EDhApiWorldGenerationStep @@ -87,11 +90,8 @@ public class FullDataSourceV2 implements IDataSource * The y data should be sorted from bottom to top */ public long[][] dataPoints; - private boolean isEmpty; - - private FullDataPointIdMap mapping; - public FullDataPointIdMap getMapping() { return this.mapping; } + public boolean isEmpty; public boolean applyToParent = false; @@ -651,7 +651,10 @@ public class FullDataSourceV2 implements IDataSource // helper methods // //================// - // TODO make private, any external logic should go through a method, not interact with the arrays directly + /** + * Usually this should just be used internally, but there may be instances + * where the raw data arrays are available without the data source object. + */ public static int relativePosToIndex(int relX, int relZ) throws IndexOutOfBoundsException { if (relX < 0 || relZ < 0 || @@ -678,19 +681,12 @@ public class FullDataSourceV2 implements IDataSource @Override public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - @Deprecated - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return this.getWorldGenStepAtRelativePos(0, 0); } - @Override public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { int index = relativePosToIndex(relX, relZ); return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public boolean isEmpty() { return this.isEmpty; } - public void markNotEmpty() { this.isEmpty = false; } - public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) { int index = relativePosToIndex(relX, relZ); @@ -714,6 +710,12 @@ public class FullDataSourceV2 implements IDataSource } } + + + //================// + // base overrides // + //================// + @Override public String toString() { return this.pos.toString(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 9eb58d3bb..208aa4df5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -322,7 +322,7 @@ public class ColumnRenderSource implements IDataSource if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { FullDataToRenderDataTransformer.convertColumnData( - level, inputFullDataSource.getMapping(), + level, inputFullDataSource.mapping, minBlockPos.x + x, minBlockPos.z + z, columnArrayView, dataColumn); @@ -373,11 +373,6 @@ public class ColumnRenderSource implements IDataSource public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - @Override - public EDhApiWorldGenerationStep getWorldGenStep() { return EDhApiWorldGenerationStep.EMPTY; } - @Override - public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { return EDhApiWorldGenerationStep.EMPTY; } - /** @return how many data points wide this {@link ColumnRenderSource} is. */ public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); } public byte getDetailOffset() { return SECTION_SIZE_OFFSET; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 37cbfb931..b4f12126c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -98,7 +98,7 @@ public class FullDataToRenderDataTransformer final byte dataDetail = fullDataSource.getDataDetailLevel(); final int vertSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(fullDataSource.getDataDetailLevel()); final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY()); - if (fullDataSource.isEmpty()) + if (fullDataSource.isEmpty) { return columnSource; } @@ -118,7 +118,7 @@ public class FullDataToRenderDataTransformer ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); long[] dataColumn = fullDataSource.get(x, z); - convertColumnData(level, fullDataSource.getMapping(), baseX + x, baseZ + z, columnArrayView, dataColumn); + convertColumnData(level, fullDataSource.mapping, baseX + x, baseZ + z, columnArrayView, dataColumn); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index b2af153c1..eb796421f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -71,7 +71,7 @@ public class LodDataBuilder DhSectionPos pos = new DhSectionPos(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ); FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos); - dataSource.markNotEmpty(); + dataSource.isEmpty = false; @@ -135,7 +135,7 @@ public class LodDataBuilder int lastY = chunkWrapper.getMaxBuildHeight(); IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; - int mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); + int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting byte blockLight = (byte) chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ); byte skyLight = (byte) chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ); @@ -180,7 +180,7 @@ public class LodDataBuilder longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); biome = newBiome; blockState = newBlockState; - mappedId = dataSource.getMapping().addIfNotPresentAndGetId(biome, blockState); + mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); blockLight = newBlockLight; skyLight = newSkyLight; lastY = y; @@ -205,7 +205,7 @@ public class LodDataBuilder } } - LodUtil.assertTrue(!dataSource.isEmpty()); + LodUtil.assertTrue(!dataSource.isEmpty); return dataSource; } @@ -232,7 +232,7 @@ public class LodDataBuilder { DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index); - int id = accessor.getMapping().addIfNotPresentAndGetId( + int id = accessor.mapping.addIfNotPresentAndGetId( (IBiomeWrapper) (dataPoint.biomeWrapper), (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 71ca44501..117177e65 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -1,6 +1,7 @@ package com.seibel.distanthorizons.core.file; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; @@ -353,7 +354,7 @@ public abstract class AbstractLegacyDataSourceHandler extends IBaseDTO public static FullDataSourceV2DTO CreateFromDataSource(FullDataSourceV2 dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException { CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); - byte[] mappingByteArray = writeDataMappingToBlob(dataSource.getMapping(), compressionModeEnum); + byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum); return new FullDataSourceV2DTO( dataSource.getSectionPos(), @@ -133,7 +133,7 @@ public class FullDataSourceV2DTO implements IBaseDTO dataSource.columnGenerationSteps = this.columnGenStepByteArray; dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray, this.compressionModeEnum); - dataSource.getMapping().clear(dataSource.getSectionPos()); + dataSource.mapping.clear(dataSource.getSectionPos()); // should only be null when used in a unit test if (!unitTest) { @@ -142,7 +142,7 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); } - dataSource.getMapping().mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum)); + dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum)); } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; @@ -150,7 +150,7 @@ public class FullDataSourceV2DTO implements IBaseDTO dataSource.levelMinY = this.levelMinY; - dataSource.markNotEmpty(); + dataSource.isEmpty = false; return dataSource; } From 6d2912e320d25ebc9699fcd91dd435339518a578 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 20:32:01 -0500 Subject: [PATCH 054/183] Add created and last modified unix time to FullDataSourceV2 --- .../distanthorizons/core/sql/repo/FullDataSourceV2Repo.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index ebfa1240d..2a01b0528 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -124,8 +124,8 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Sat, 16 Mar 2024 20:54:12 -0500 Subject: [PATCH 055/183] Fix incorrect FullDataSourceV2 downsampling for worldGenStep --- .../fullData/sources/FullDataSourceV2.java | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 9f012419c..4a0c4b0a6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -355,7 +355,7 @@ public class FullDataSourceV2 implements IDataSource { if (inputDataSource.pos.getDetailLevel() + 1 != this.pos.getDetailLevel()) { - throw new IllegalArgumentException("Input data source must be exactly 1 detail level below this data source. Expected ["+(this.pos.getDetailLevel() - 1)+"], received ["+inputDataSource.pos.getDetailLevel()+"]."); + throw new IllegalArgumentException("Input data source must be exactly 1 detail level below this data source. Expected [" + (this.pos.getDetailLevel() - 1) + "], received [" + inputDataSource.pos.getDetailLevel() + "]."); } // input is one detail level lower (higher detail) @@ -377,35 +377,56 @@ public class FullDataSourceV2 implements IDataSource for (int x = 0; x < WIDTH; x += 2) { for (int z = 0; z < WIDTH; z += 2) - { - long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); - // TODO - byte inputGenStep = inputDataSource.columnGenerationSteps[0]; - - + { int recipientX = (x / 2) + recipientOffsetX; int recipientZ = (z / 2) + recipientOffsetZ; int recipientIndex = relativePosToIndex(recipientX, recipientZ); - - long[] oldDataArray = this.dataPoints[recipientIndex]; - + + + // world gen + byte inputGenStep = determineMinWorldGenStepForTwoByTwoColumn(inputDataSource.columnGenerationSteps, x, z); this.columnGenerationSteps[recipientIndex] = inputGenStep; + + // data points + long[] oldDataArray = this.dataPoints[recipientIndex]; + long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); this.dataPoints[recipientIndex] = mergedInputDataArray; + + // mapping this.remapDataColumn(recipientIndex, remappedIds); - + + // we only need to see if the data was changed in one column if (!dataChanged) { // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[recipientIndex]); } - + this.isEmpty = false; } } return dataChanged; } + /** + * The minimum value is used because we don't want to accidentally record that + * something was generated when it wasn't. + */ + private static byte determineMinWorldGenStepForTwoByTwoColumn(byte[] columnGenerationSteps, int relX, int relZ) + { + byte minWorldGenStepValue = Byte.MAX_VALUE; + for (int x = 0; x < 2; x++) + { + for (int z = 0; z < 2; z++) + { + int index = relativePosToIndex(x + relX, z + relZ); + byte worldGenStepValue = columnGenerationSteps[index]; + minWorldGenStepValue = (byte) Math.min(minWorldGenStepValue, worldGenStepValue); + } + } + return minWorldGenStepValue; + } private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { ArrayList newColumnList = new ArrayList<>(); From b20cbab012b1edd0386b671625e5917a9d4678db Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 21:03:15 -0500 Subject: [PATCH 056/183] Remove Seamless Overdraw Test The result wasn't very good due to rendering issues with entities --- .../distanthorizons/core/config/Config.java | 15 --------------- .../assets/distanthorizons/lang/en_us.json | 4 ---- 2 files changed, 19 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 4cdaf7096..b79167dbb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -561,21 +561,6 @@ public class Config .setPerformance(EConfigEntryPerformance.NONE) .build(); - @Deprecated // TODO remove failed experiment - public static ConfigEntry seamlessOverdraw = new ConfigEntry.Builder() - .set(false) - .comment("" - + "Buggy experimental option that will attempt to match up \n" - + "Distant Horizons' and Minecraft's near/far clip planes, \n" - + "reducing overdraw. \n" - + "\n" - + "Only functional on Fabric.\n" - + "Works best with an overdraw prevention setting of " + EOverdrawPrevention.MEDIUM + " or higher \n" - + " and cave culling is disabled. \n" - + "") - .setPerformance(EConfigEntryPerformance.NONE) - .build(); - // move into "shader compatibility" public static ConfigEntry brightnessMultiplier = new ConfigEntry.Builder() // TODO: Make this a float (the ClassicConfigGUI doesnt support floats) .set(1.0) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index becb94c31..7d9df80fc 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -269,10 +269,6 @@ "Overdraw Prevention", "distanthorizons.config.client.advanced.graphics.advancedGraphics.overdrawPrevention.@tooltip": "Determines how far from the camera Distant Horizons will start rendering. \nMeasured as a percentage of the vanilla render distance.\n\nHigher values will prevent LODs from rendering behind vanilla blocks at a higher distance,\nbut may cause holes to appear in the LODs. \nHoles are most likely to appear when flying through unloaded terrain. \n\nIncreasing the vanilla render distance increases the effectiveness of this setting.", - "distanthorizons.config.client.advanced.graphics.advancedGraphics.seamlessOverdraw": - "(Experimental) Seamless Overdraw", - "distanthorizons.config.client.advanced.graphics.advancedGraphics.seamlessOverdraw.@tooltip": - "Buggy experimental option that will attempt to match up \nDistant Horizons' and Minecraft's near/far clip planes, \nreducing overdraw. \n\nNote: only works with Fabric.", "distanthorizons.config.client.advanced.graphics.advancedGraphics.brightnessMultiplier": "Brightness Multiplier", "distanthorizons.config.client.advanced.graphics.advancedGraphics.brightnessMultiplier.@tooltip": From 69e3e2bca3a3cc6d8ae46110195e3ae10096fd42 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 22:23:20 -0500 Subject: [PATCH 057/183] Add a config to render overlapping quad errors --- .../seibel/distanthorizons/core/config/Config.java | 8 ++++++++ .../render/bufferBuilding/BufferQuad.java | 14 +++++++++++--- .../render/bufferBuilding/LodQuadBuilder.java | 4 ++-- .../assets/distanthorizons/lang/en_us.json | 4 ++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index b79167dbb..b9c1a0e24 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -1209,6 +1209,14 @@ public class Config + "Useful for debugging shaders") .build(); + public static ConfigEntry showOverlappingQuadErrors = new ConfigEntry.Builder() + .set(false) + .comment("" + + "If true overlapping quads will be rendered as bright red for easy identification. \n" + + "If false the quads will be rendered normally. \n" + + "") + .build(); + // Note: This will reset on game restart, and should have a warning on the tooltip public static ConfigEntry allowUnsafeValues = new ConfigEntry.Builder() .set(false) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java index d9e83458f..8392c25c3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/BufferQuad.java @@ -151,6 +151,7 @@ public final class BufferQuad { if (quad.hasError || this.hasError) return false; + // only merge quads that are in the same direction if (this.direction != quad.direction) return false; @@ -284,10 +285,17 @@ public final class BufferQuad if (thisPerpendicularCompareStartPos < otherPerpendicularCompareStartPos + otherPerpendicularCompareWidth) { // these quads are overlapping, they can't be merged - //EVENT_LOGGER.warn("Overlapping quads detected!"); - quad.hasError = true; - this.hasError = true; + + // Overlapping quads appear to render correctly, why are we marking them as errored? + // Is it possible the wrong quad will be extended thus the wrong color is rendered? + // Or is that the height/depth might be wrong? + if (Config.Client.Advanced.Debugging.showOverlappingQuadErrors.get()) + { + quad.hasError = true; + this.hasError = true; + } } + return false; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index 03ff71bf5..1e4932de1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -254,8 +254,8 @@ public class LodQuadBuilder default: throw new IllegalArgumentException("Invalid Axis enum: " + axis); } - putVertex(bb, (short) (quad.x + dx), (short) (quad.y + dy), (short) (quad.z + dz), - quad.hasError ? ColorUtil.RED : quad.color, // TODO add debug config that allows toggling this + this.putVertex(bb, (short) (quad.x + dx), (short) (quad.y + dy), (short) (quad.z + dz), + quad.hasError ? ColorUtil.RED : quad.color, quad.hasError ? 0 : normalIndex, quad.hasError ? 0 : quad.irisBlockMaterialId, quad.hasError ? 15 : quad.skyLight, diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 7d9df80fc..19f7497fc 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -440,6 +440,10 @@ "Enable white world", "distanthorizons.config.client.advanced.debugging.enableWhiteWorld.@tooltip": "If enabled, vertex color will not be passed.\nUseful for debugging shaders.", + "distanthorizons.config.client.advanced.debugging.showOverlappingQuadErrors": + "Show overlapping quad errors", + "distanthorizons.config.client.advanced.debugging.showOverlappingQuadErrors.@tooltip": + "f true overlapping quads will be rendered as bright red for easy identification. \nIf false the quads will be rendered normally.", "distanthorizons.config.client.advanced.debugging.allowUnsafeValues": "Allow Unsafe UI Values", "distanthorizons.config.client.advanced.debugging.allowUnsafeValues.@tooltip": From c435b55576e965d32fee44da681b3f683698ea6e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 22:27:11 -0500 Subject: [PATCH 058/183] Fix updating empty LODs (IE in the end) --- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 4a0c4b0a6..7e21978bf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -254,10 +254,9 @@ public class FullDataSourceV2 implements IDataSource public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } public boolean update(FullDataSourceV2 inputDataSource) { - // shouldn't happen, but James saw it happen once if (inputDataSource.mapping.getMaxValidId() == 0) { - LOGGER.warn("Invalid mapping given from input update data source at pos: ["+inputDataSource.pos+"], skipping update."); + // can happen in the end or where empty chunks exist return false; } From cbadfab62a0f11e5150466c554fa974885333a85 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 22:43:07 -0500 Subject: [PATCH 059/183] disable cave culling in the end --- .../bufferBuilding/ColumnRenderBufferBuilder.java | 10 +++++++++- .../wrapperInterfaces/world/IDimensionTypeWrapper.java | 7 +++---- .../resources/assets/distanthorizons/lang/en_us.json | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index f133f424e..4c9bd2b43 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -86,7 +86,15 @@ public class ColumnRenderBufferBuilder boolean enableTransparency = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled; EVENT_LOGGER.trace("RenderRegion start QuadBuild @ " + renderSource.sectionPos); - boolean enableSkyLightCulling = !clientLevel.getLevelWrapper().hasCeiling() && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); + boolean enableSkyLightCulling = + ( + // dimensions with a ceiling will be all caves so we don't want cave culling + !clientLevel.getLevelWrapper().hasCeiling() + // the end has a lot of overhangs with 0 lighting above the void, which look broken with + // the current cave culling logic (this could probably be improved, but just skipping it works best for now) + && !clientLevel.getLevelWrapper().getDimensionType().isTheEnd() + ) + && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); int skyLightCullingBelow = Config.Client.Advanced.Graphics.AdvancedGraphics.caveCullingHeight.get(); // FIXME: Clamp also to the max world height. diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IDimensionTypeWrapper.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IDimensionTypeWrapper.java index 0422e0ba9..bf9097608 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IDimensionTypeWrapper.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IDimensionTypeWrapper.java @@ -22,10 +22,6 @@ package com.seibel.distanthorizons.core.wrapperInterfaces.world; import com.seibel.distanthorizons.api.interfaces.world.IDhApiDimensionTypeWrapper; import com.seibel.distanthorizons.coreapi.interfaces.dependencyInjection.IBindable; -/** - * @author James Seibel - * @version 2023-6-17 - */ public interface IDimensionTypeWrapper extends IDhApiDimensionTypeWrapper, IBindable { @Override @@ -37,4 +33,7 @@ public interface IDimensionTypeWrapper extends IDhApiDimensionTypeWrapper, IBind @Override boolean hasSkyLight(); + // there's definitely a better way of doing this, but it should work well enough for now + default boolean isTheEnd() { return this.getDimensionName().equalsIgnoreCase("the_end"); } + } diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 19f7497fc..0f5e989b4 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -440,7 +440,7 @@ "Enable white world", "distanthorizons.config.client.advanced.debugging.enableWhiteWorld.@tooltip": "If enabled, vertex color will not be passed.\nUseful for debugging shaders.", - "distanthorizons.config.client.advanced.debugging.showOverlappingQuadErrors": + "distanthorizons.config.client.advanced.debugging.showOverlappingQuadErrors": "Show overlapping quad errors", "distanthorizons.config.client.advanced.debugging.showOverlappingQuadErrors.@tooltip": "f true overlapping quads will be rendered as bright red for easy identification. \nIf false the quads will be rendered normally.", From c55880bcb9b3c60aa1a95cb86c108669ff8a66a2 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 16 Mar 2024 22:43:22 -0500 Subject: [PATCH 060/183] Fix empty data sources not updating and being regenerated --- .../dataObjects/fullData/sources/FullDataSourceV2.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 7e21978bf..0f330d659 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -254,14 +254,6 @@ public class FullDataSourceV2 implements IDataSource public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } public boolean update(FullDataSourceV2 inputDataSource) { - if (inputDataSource.mapping.getMaxValidId() == 0) - { - // can happen in the end or where empty chunks exist - return false; - } - - - byte thisDetailLevel = this.pos.getDetailLevel(); byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); From fada27257c363da6fda08e516b4f41a643b081e4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 17 Mar 2024 16:14:24 -0500 Subject: [PATCH 061/183] Write FullDataSourceV2 column length as a short instead of a int --- .../distanthorizons/core/sql/dto/FullDataSourceV2DTO.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index b23e0a865..a9f5f77bb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -180,8 +180,10 @@ public class FullDataSourceV2DTO implements IBaseDTO long[] dataColumn = dataArray[xz]; // write column length - int columnLength = (dataColumn != null) ? dataColumn.length : 0; - compressedOut.writeInt(columnLength); /// TODO + short columnLength = (dataColumn != null) ? (short) dataColumn.length : 0; + // a short is used instead of an int because at most we store 4096 vertical slices and a + // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) + compressedOut.writeShort(columnLength); // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) @@ -210,7 +212,7 @@ public class FullDataSourceV2DTO implements IBaseDTO for (int xz = 0; xz < dataArray.length; xz++) { // read the column length - int dataColumnLength = compressedIn.readInt(); // separate variables are used for debugging and in case validation wants to be added later + short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later long[] dataColumn = new long[dataColumnLength]; // read column data (will be skipped if no data was present) From 9d539c476622baeeb13d2c0a53052b9f1e171885 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 17 Mar 2024 16:15:06 -0500 Subject: [PATCH 062/183] Update several todo comments --- .../core/dataObjects/fullData/FullDataPointIdMap.java | 1 + .../core/file/AbstractNewDataSourceHandler.java | 2 +- .../core/file/fullDatafile/FullDataFileHandlerV2.java | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index 4df982758..468d57bee 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -44,6 +44,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; * since it stringifies every block and biome name, which is quite bulky. * It might be worth while to have a biome and block ID that then both get mapped * to the data point ID to reduce file size. + * And/or it would be good to dynamically remove IDs that aren't currently in use. * * @author Leetom */ diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index bfe9cabfc..7648f16de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -35,7 +35,7 @@ public abstract class AbstractNewDataSourceHandler * * @see AbstractNewDataSourceHandler#MIN_SECTION_DETAIL_LEVEL */ - public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL; // TODO add "section" to detail level + public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL; /** * The lowest numerical detail level possible. * diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java index ead649c20..2c28070de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV2.java @@ -156,7 +156,8 @@ public class FullDataFileHandlerV2 @Override protected FullDataSourceV2 createNewDataSourceFromExistingDtos(DhSectionPos pos) { - // TODO maybe just set children update flags to true? + // TODO maybe just set children update flags to true? + // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway return FullDataSourceV2.createEmpty(pos); } From 8ec22189fd51aa6d4f00a263da85d786bd5d88cf Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 17 Mar 2024 16:24:46 -0500 Subject: [PATCH 063/183] remove unneeded casting in RenderSourceFileHandler --- .../core/file/renderfile/RenderSourceFileHandler.java | 8 +++----- .../distanthorizons/core/level/ClientLevelModule.java | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index b0510559b..0096af7b3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -29,7 +29,6 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; @@ -47,9 +46,8 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler Date: Sun, 17 Mar 2024 16:29:49 -0500 Subject: [PATCH 064/183] Improve the nightly build warning text --- .../com/seibel/distanthorizons/core/api/internal/ClientApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index 4a81bd1c1..b4d0b3f04 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -478,7 +478,7 @@ public class ClientApi this.configOverrideReminderPrinted = true; // remind the user that this is a development build - MC.sendChatMessage(ModInfo.READABLE_NAME + " experimental build " + ModInfo.VERSION); + MC.sendChatMessage("Distant Horizons nightly experimental build version [" + ModInfo.VERSION+"]."); MC.sendChatMessage("You are running an unsupported version of Distant Horizons!"); MC.sendChatMessage("Here be dragons!"); } From bf00a23499235788b8c9221d9e358629d32ead70 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 17 Mar 2024 16:46:56 -0500 Subject: [PATCH 065/183] Rename IWorldGenerationQueue -> IFullDataSourceRetrievalQueue --- .../GeneratedFullDataFileHandler.java | 18 ++--- ...ava => IFullDataSourceRetrievalQueue.java} | 72 ++++++++++++++++--- .../core/generation/WorldGenerationQueue.java | 5 +- .../core/level/WorldGenModule.java | 9 +-- 4 files changed, 78 insertions(+), 26 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/generation/{IWorldGenerationQueue.java => IFullDataSourceRetrievalQueue.java} (58%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java index d3c338711..43fb41532 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; +import com.seibel.distanthorizons.core.generation.IFullDataSourceRetrievalQueue; import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -50,7 +50,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen public static final int MAX_WORLD_GEN_REQUESTS_PER_THREAD = 20; - private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); + private final AtomicReference worldGenQueueRef = new AtomicReference<>(null); private final ArrayList onWorldGenTaskCompleteListeners = new ArrayList<>(); protected final DelayedFullDataSourceSaveCache delayedFullDataSourceSaveCache = new DelayedFullDataSourceSaveCache(this::onDataSourceSave, 5_000); @@ -73,7 +73,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen * Assigns the queue for handling world gen and does first time setup as well.
* Assumes there isn't a pre-existing queue. */ - public void setWorldGenerationQueue(IWorldGenerationQueue newWorldGenQueue) + public void setWorldGenerationQueue(IFullDataSourceRetrievalQueue newWorldGenQueue) { boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); @@ -87,7 +87,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen { // TODO there has to be a better way to do this - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue != null) { worldGenQueue.removeGenRequestIf(removeIf); @@ -163,7 +163,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen @Override public void setTotalRetrievalPositionCount(int newCount) { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue != null) { worldGenQueue.setEstimatedTotalTaskCount(newCount); @@ -179,7 +179,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen } - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue == null) { // we can't queue anything if the world generator isn't set up yet @@ -203,7 +203,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen @Override public boolean queuePositionForRetrieval(DhSectionPos genPos) { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue == null) { return false; @@ -219,7 +219,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen @Override public ArrayList getPositionsToRetrieve(DhSectionPos pos) { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue == null) { return null; @@ -288,7 +288,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen @Override public int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) { - IWorldGenerationQueue worldGenQueue = this.worldGenQueueRef.get(); + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); if (worldGenQueue == null) { return -1; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java similarity index 58% rename from core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java rename to core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java index a7a834929..5bf47abd8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IWorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java @@ -23,24 +23,80 @@ import com.seibel.distanthorizons.core.generation.tasks.IWorldGenTaskTracker; import com.seibel.distanthorizons.core.generation.tasks.WorldGenResult; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.LodQuadTree; import java.io.Closeable; import java.util.concurrent.CompletableFuture; import java.util.function.Function; -// TODO This doesn't need an interface, remove the interface -@Deprecated -public interface IWorldGenerationQueue extends Closeable +/** + * Used to track what full data sources the system currently + * wants but doesn't have.
+ * IE, what sections should be generated via the world generator.

+ * + * Note:
+ * This won't contain every position that needs to be retrieved + * (due to causing issues at extreme render distances). + * TODO does that mean this object isn't necessary or + * should just be renamed since it isn't the full queue

+ * + * Use by both world gen and server networking. + * + * @see LodQuadTree + */ +public interface IFullDataSourceRetrievalQueue extends Closeable { + //=========// + // getters // + //=========// + /** the largest numerical detail level */ byte lowestDataDetail(); /** the smallest numerical detail level */ byte highestDataDetail(); + + + //=======// + // setup // + //=======// + + /** + * Starts the retrieval process if not already running, + * and if running updates the target position. + * + * @param targetPos the position that retrieval should be centered around, + * generally this will be the player's position. + * */ + void startAndSetTargetPos(DhBlockPos2D targetPos); + + + + //===============// + // task handling // + //===============// + + /** @deprecated replace with {@link IFullDataSourceRetrievalQueue#removeGenTask(DhSectionPos)} */ + @Deprecated + void removeGenRequestIf(Function removeIf); + void removeGenTask(DhSectionPos pos); + CompletableFuture submitGenTask(DhSectionPos pos, byte requiredDataDetail, IWorldGenTaskTracker tracker); - /** @param targetPos the position that world generation should be centered around, generally this will be the player's position. */ - void startGenerationQueueAndSetTargetPos(DhBlockPos2D targetPos); + + + //==========// + // shutdown // + //==========// + + CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning); + void close(); + + + + //===============// + // debug display // + //===============// int getWaitingTaskCount(); int getInProgressTaskCount(); @@ -49,11 +105,5 @@ public interface IWorldGenerationQueue extends Closeable int getEstimatedTotalTaskCount(); void setEstimatedTotalTaskCount(int newEstimate); - CompletableFuture startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning); - void close(); - - void removeGenRequestIf(Function removeIf); - void removeGenTask(DhSectionPos pos); - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index d63fe0c17..e946d2698 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -49,7 +49,7 @@ import java.util.concurrent.*; import java.util.function.Consumer; import java.util.function.Function; -public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRenderable +public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); @@ -187,7 +187,8 @@ public class WorldGenerationQueue implements IWorldGenerationQueue, IDebugRender // running tasks // //===============// - public void startGenerationQueueAndSetTargetPos(DhBlockPos2D targetPos) + @Override + public void startAndSetTargetPos(DhBlockPos2D targetPos) { // update the target pos this.generationTargetPos = targetPos; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index 302e31b1a..9e866d2d1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; -import com.seibel.distanthorizons.core.generation.IWorldGenerationQueue; +import com.seibel.distanthorizons.core.generation.IFullDataSourceRetrievalQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; @@ -156,10 +156,10 @@ public class WorldGenModule implements Closeable // helper classes // //================// - /** Handles the {@link IWorldGenerationQueue} and any other necessary world gen information. */ + /** Handles the {@link IFullDataSourceRetrievalQueue} and any other necessary world gen information. */ public static abstract class AbstractWorldGenState { - public IWorldGenerationQueue worldGenerationQueue; + public IFullDataSourceRetrievalQueue worldGenerationQueue; CompletableFuture closeAsync(boolean doInterrupt) { @@ -178,7 +178,8 @@ public class WorldGenModule implements Closeable } /** @param targetPosForGeneration the position that world generation should be centered around */ - public void startGenerationQueueAndSetTargetPos(DhBlockPos2D targetPosForGeneration) { this.worldGenerationQueue.startGenerationQueueAndSetTargetPos(targetPosForGeneration); } + public void startGenerationQueueAndSetTargetPos(DhBlockPos2D targetPosForGeneration) + { this.worldGenerationQueue.startAndSetTargetPos(targetPosForGeneration); } } } From 6b13e9141c9da6e8405f6827933edf30e7158306 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 17 Mar 2024 17:32:41 -0500 Subject: [PATCH 066/183] rename FullDataFileHandler -> FullDataSourceProvider and Remove IFullDataSourceProvider IFullDataSourceProvider was removed so legacy providers don't have to implement methods they don't need --- .../fullData/sources/FullDataSourceV2.java | 4 +- .../core/file/ISourceProvider.java | 2 - ...rV1.java => FullDataSourceProviderV1.java} | 4 +- ...rV2.java => FullDataSourceProviderV2.java} | 91 +++++++++++++++---- ...a => GeneratedFullDataSourceProvider.java} | 20 +++- .../fullDatafile/IFullDataSourceProvider.java | 77 ---------------- ...java => RemoteFullDataSourceProvider.java} | 6 +- .../renderfile/RenderSourceFileHandler.java | 17 +++- .../core/level/ClientLevelModule.java | 6 +- .../core/level/DhClientLevel.java | 10 +- .../core/level/DhClientServerLevel.java | 4 +- .../core/level/DhServerLevel.java | 4 +- .../distanthorizons/core/level/IDhLevel.java | 4 +- .../core/level/IDhWorldGenLevel.java | 4 +- .../core/level/ServerLevelModule.java | 6 +- .../core/level/WorldGenModule.java | 12 +-- .../core/render/LodQuadTree.java | 6 +- .../core/render/LodRenderSection.java | 4 +- 18 files changed, 140 insertions(+), 141 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{FullDataFileHandlerV1.java => FullDataSourceProviderV1.java} (94%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{FullDataFileHandlerV2.java => FullDataSourceProviderV2.java} (85%) rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{GeneratedFullDataFileHandler.java => GeneratedFullDataSourceProvider.java} (93%) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java rename core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/{RemoteFullDataFileHandler.java => RemoteFullDataSourceProvider.java} (73%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 0f330d659..ea7629db4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -279,7 +279,7 @@ public class FullDataSourceV2 implements IDataSource throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); } - if (dataChanged && this.pos.getDetailLevel() < FullDataFileHandlerV2.TOP_SECTION_DETAIL_LEVEL) + if (dataChanged && this.pos.getDetailLevel() < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL) { // mark that this data source should be applied to its parent this.applyToParent = true; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java index c22adefd9..76f04dd70 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java @@ -1,7 +1,6 @@ package com.seibel.distanthorizons.core.file; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -11,7 +10,6 @@ import java.util.concurrent.CompletableFuture; /** * Base for all data source providers * - * @see IFullDataSourceProvider * @see IRenderSourceProvider */ public interface ISourceProvider, TDhLevel extends IDhLevel> extends AutoCloseable diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java similarity index 94% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV1.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index 441364b9d..c6b3a4ec4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataFileHandlerV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -public class FullDataFileHandlerV1 extends AbstractLegacyDataSourceHandler +public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -46,7 +46,7 @@ public class FullDataFileHandlerV1 extends AbstractLegacyDataSourceHandler - implements IFullDataSourceProvider, IDebugRenderable + implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -72,7 +76,7 @@ public class FullDataFileHandlerV2 * vs gracefully shutting down the thread ourselves. */ protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); - protected final FullDataFileHandlerV1 legacyFileHandler; + protected final FullDataSourceProviderV1 legacyFileHandler; /** * Tracks which positions are currently being updated @@ -93,11 +97,11 @@ public class FullDataFileHandlerV2 // constructor // //=============// - public FullDataFileHandlerV2(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public FullDataFileHandlerV2(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + public FullDataSourceProviderV2(IDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public FullDataSourceProviderV2(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); - this.legacyFileHandler = new FullDataFileHandlerV1(level, saveStructure, saveDirOverride); + this.legacyFileHandler = new FullDataSourceProviderV1(level, saveStructure, saveDirOverride); DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); @@ -164,16 +168,6 @@ public class FullDataFileHandlerV2 @Override protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); } - @Override - public boolean canQueueRetrieval() - { - // Retrieval shouldn't happen while an unknown number of - // legacy data sources are present. - // If retrieval was allowed we might run into concurrency issues. - return !this.migrationThreadRunning.get(); - } - - //================// @@ -391,6 +385,71 @@ public class FullDataFileHandlerV2 + //=======================// + // retrieval (world gen) // + //=======================// + + /** + * Returns true if this provider can generate or retrieve + * {@link FullDataSourceV2}'s that aren't currently in the database. + */ + public boolean canRetrieveMissingDataSources() + { + // the base handler just handles basic reading/writing + // to the database and as such can't retrieve anything else. + return false; + } + + /** + * Returns false if this provider isn't accepting new requests, + * this can be due to having a full queue or some other + * limiting factor.

+ * + * Note: when overriding make sure to add:
+ * + * if (!super.canQueueRetrieval())
+ * {
+ * return false;
+ * }
+ * + * to the beginning of your override. + * Otherwise, parent retrieval limits will be ignored. + */ + public boolean canQueueRetrieval() + { + // Retrieval shouldn't happen while an unknown number of + // legacy data sources are present. + // If retrieval was allowed we might run into concurrency issues. + return !this.migrationThreadRunning.get(); + } + + /** + * @return null if this provider can't generate any positions and + * an empty array if all positions were generated + */ + @Nullable + public ArrayList getPositionsToRetrieve(DhSectionPos pos) { return null; } + /** + * Returns how many positions could potentially be generated for this position assuming the position is empty. + * Used when estimating the total number of retrieval requests. + */ + public int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) { return -1; } + + /** @return true if the position was queued, false if not */ + public boolean queuePositionForRetrieval(DhSectionPos genPos) { return false; } + + /** Can be used to display how many total retrieval requests might be available. */ + public void setTotalRetrievalPositionCount(int newCount) { } + + /** + * Returns how many data sources are currently in memory and haven't + * been saved to the database. + * Returns -1 if this provider never stores data sources to memory. + */ + public int getUnsavedDataSourceCount() { return -1; } + + + //===========// // overrides // //===========// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java similarity index 93% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 43fb41532..3e3117458 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -43,7 +43,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; -public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implements IDebugRenderable +public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -61,7 +61,7 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen // constructor // //=============// - public GeneratedFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public GeneratedFullDataSourceProvider(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } @@ -195,8 +195,18 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen } - // don't queue additional world gen requests beyond the max allotted int maxQueueCount = MAX_WORLD_GEN_REQUESTS_PER_THREAD * Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads.get(); + + if (this.delayedFullDataSourceSaveCache.getUnsavedCount() >= maxQueueCount) + { + // don't queue additional world gen requests if there are + // a lot of data sources in memory + // (this is done to prevent infinite memory growth) + return false; + } + + + // don't queue additional world gen requests beyond the max allotted count return worldGenQueue.getWaitingTaskCount() < maxQueueCount; } @@ -342,12 +352,12 @@ public class GeneratedFullDataFileHandler extends FullDataFileHandlerV2 implemen { return (chunkSizedFullDataSource) -> { - GeneratedFullDataFileHandler.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(chunkSizedFullDataSource); + GeneratedFullDataSourceProvider.this.delayedFullDataSourceSaveCache.queueDataSourceForUpdateAndSave(chunkSizedFullDataSource); }; } } private void onDataSourceSave(FullDataSourceV2 fullDataSource) - { GeneratedFullDataFileHandler.this.updateDataSourceAsync(fullDataSource); } + { GeneratedFullDataSourceProvider.this.updateDataSourceAsync(fullDataSource); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java deleted file mode 100644 index 7b5a81291..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/IFullDataSourceProvider.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.fullDatafile; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.ISourceProvider; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayList; -import java.util.concurrent.CompletableFuture; - -/** - * Handles reading, writing, and updating {@link FullDataSourceV2}'s.
- * Should be backed by a database handled by a {@link LegacyFullDataRepo}. - */ -public interface IFullDataSourceProvider extends ISourceProvider, AutoCloseable -{ - CompletableFuture getAsync(DhSectionPos pos); - FullDataSourceV2 get(DhSectionPos pos); - - CompletableFuture updateDataSourceAsync(FullDataSourceV2 chunkData); - - /** @return -1 if this provider never has unsaved data sources */ - default int getUnsavedDataSourceCount() { return -1; } - - - - // retrieval (world gen) // - - /** - * If true this {@link IFullDataSourceProvider} can generate or retrieve - * {@link FullDataSourceV2}'s that aren't currently in the database. - */ - default boolean canRetrieveMissingDataSources() { return false; } - - /** @return null if it was unable to generate any positions, an empty array if all positions were generated */ - @Nullable - default ArrayList getPositionsToRetrieve(DhSectionPos pos) { return null; } - /** - * Returns how many positions could potentially be generated for this position assuming the position is empty. - * Used when estimating the total number of retrieval requests. - */ - default int getMaxPossibleRetrievalPositionCountForPos(DhSectionPos pos) { return -1; } - - /** @return true if the position was queued, false if not */ - default boolean queuePositionForRetrieval(DhSectionPos genPos) { return false; } - /** - * @return false if the provider isn't accepting new requests, - * this can be due to having a full queue or some other - * limiting factor. - */ - default boolean canQueueRetrieval() { return false; } - - /** Can be used to display how many total retrieval requests might be available. */ - default void setTotalRetrievalPositionCount(int newCount) { } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java similarity index 73% rename from core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java index 5d43f5ef3..efdf28122 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/RemoteFullDataSourceProvider.java @@ -25,9 +25,9 @@ import org.jetbrains.annotations.Nullable; import java.io.File; -public class RemoteFullDataFileHandler extends FullDataFileHandlerV2 +public class RemoteFullDataSourceProvider extends FullDataSourceProviderV2 { - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } - public RemoteFullDataFileHandler(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } + public RemoteFullDataSourceProvider(IDhLevel level, AbstractSaveStructure saveStructure) { super(level, saveStructure); } + public RemoteFullDataSourceProvider(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 0096af7b3..46f971f16 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -47,7 +47,7 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler lines = new ArrayList<>(); lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]"); lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); lines.add(" Level Unsaved #: " + this.level.getUnsavedDataSourceCount()); - lines.add(" Full Data Unsaved #: " + this.fullDataSourceProvider.getUnsavedDataSourceCount()); + if (unsavedDataSourceCount != -1) + { + lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount); + } lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size()); lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size()); + + return lines.toArray(new String[0]); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index c6763cc1d..47ea5d305 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -52,7 +52,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle private final IDhClientLevel parentClientLevel; - public final FullDataFileHandlerV2 fullDataSourceProvider; + public final FullDataSourceProviderV2 fullDataSourceProvider; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); public final F3Screen.NestedMessage f3Message; @@ -305,7 +305,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle public final LodRenderer renderer; public ClientRenderState( - IDhClientLevel dhClientLevel, IClientLevelWrapper clientLevelWrapper, FullDataFileHandlerV2 fullDataSourceProvider, + IDhClientLevel dhClientLevel, IClientLevelWrapper clientLevelWrapper, FullDataSourceProviderV2 fullDataSourceProvider, AbstractSaveStructure saveStructure) { this.clientLevelWrapper = clientLevelWrapper; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index 285008ab2..f3af49231 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -21,8 +21,8 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; -import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; +import com.seibel.distanthorizons.core.file.fullDatafile.RemoteFullDataSourceProvider; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; @@ -45,7 +45,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel public final ClientLevelModule clientside; public final IClientLevelWrapper levelWrapper; public final AbstractSaveStructure saveStructure; - public final RemoteFullDataFileHandler dataFileHandler; + public final RemoteFullDataSourceProvider dataFileHandler; @@ -58,7 +58,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel { this.levelWrapper = clientLevelWrapper; this.saveStructure = saveStructure; - this.dataFileHandler = new RemoteFullDataFileHandler(this, saveStructure, fullDataSaveDirOverride); + this.dataFileHandler = new RemoteFullDataSourceProvider(this, saveStructure, fullDataSaveDirOverride); this.clientside = new ClientLevelModule(this); if (enableRendering) @@ -137,7 +137,7 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel //=======================// @Override - public FullDataFileHandlerV2 getFullDataProvider() { return this.dataFileHandler; } + public FullDataSourceProviderV2 getFullDataProvider() { return this.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 5b5a24a62..6510f3a89 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.render.LodRenderSection; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; @@ -174,7 +174,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public FullDataFileHandlerV2 getFullDataProvider() { return this.serverside.dataFileHandler; } + public FullDataSourceProviderV2 getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index 5f7f06136..9af6fdc32 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -93,7 +93,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public FullDataFileHandlerV2 getFullDataProvider() { return this.serverside.dataFileHandler; } + public FullDataSourceProviderV2 getFullDataProvider() { return this.serverside.dataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java index 7dd2450e6..0d717bf08 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhLevel.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataFileHandlerV2; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; @@ -39,7 +39,7 @@ public interface IDhLevel extends AutoCloseable void updateChunkAsync(IChunkWrapper chunk); - FullDataFileHandlerV2 getFullDataProvider(); + FullDataSourceProviderV2 getFullDataProvider(); AbstractSaveStructure getSaveStructure(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java index 94108b380..287d6ad62 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/IDhWorldGenLevel.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider; -public interface IDhWorldGenLevel extends IDhLevel, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener +public interface IDhWorldGenLevel extends IDhLevel, GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener { void doWorldGen(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java index 9e8b132be..1deb4a4e4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.level; import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator; import com.seibel.distanthorizons.core.config.AppliedConfigState; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.generation.BatchGenerator; import com.seibel.distanthorizons.core.generation.WorldGenerationQueue; @@ -36,7 +36,7 @@ public class ServerLevelModule public final IDhServerLevel parentServerLevel; public final AbstractSaveStructure saveStructure; - public final GeneratedFullDataFileHandler dataFileHandler; + public final GeneratedFullDataSourceProvider dataFileHandler; public final AppliedConfigState worldGeneratorEnabledConfig; public final WorldGenModule worldGenModule; @@ -47,7 +47,7 @@ public class ServerLevelModule { this.parentServerLevel = parentServerLevel; this.saveStructure = saveStructure; - this.dataFileHandler = new GeneratedFullDataFileHandler(parentServerLevel, saveStructure); + this.dataFileHandler = new GeneratedFullDataSourceProvider(parentServerLevel, saveStructure); this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration); this.worldGenModule = new WorldGenModule(this.dataFileHandler, this.parentServerLevel); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index 9e866d2d1..ce0040165 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataFileHandler; +import com.seibel.distanthorizons.core.file.fullDatafile.GeneratedFullDataSourceProvider; import com.seibel.distanthorizons.core.generation.IFullDataSourceRetrievalQueue; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -34,15 +34,15 @@ public class WorldGenModule implements Closeable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final GeneratedFullDataFileHandler dataFileHandler; - private final GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener; + private final GeneratedFullDataSourceProvider dataFileHandler; + private final GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener; private final AtomicReference worldGenStateRef = new AtomicReference<>(); private final F3Screen.DynamicMessage worldGenF3Message; - public WorldGenModule(GeneratedFullDataFileHandler dataFileHandler, GeneratedFullDataFileHandler.IOnWorldGenCompleteListener onWorldGenCompleteListener) + public WorldGenModule(GeneratedFullDataSourceProvider dataFileHandler, GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener) { this.dataFileHandler = dataFileHandler; this.onWorldGenCompleteListener = onWorldGenCompleteListener; @@ -71,7 +71,7 @@ public class WorldGenModule implements Closeable // world gen control // //===================// - public void startWorldGen(GeneratedFullDataFileHandler dataFileHandler, AbstractWorldGenState newWgs) + public void startWorldGen(GeneratedFullDataSourceProvider dataFileHandler, AbstractWorldGenState newWgs) { // create the new world generator if (!this.worldGenStateRef.compareAndSet(null, newWgs)) @@ -83,7 +83,7 @@ public class WorldGenModule implements Closeable dataFileHandler.setWorldGenerationQueue(newWgs.worldGenerationQueue); } - public void stopWorldGen(GeneratedFullDataFileHandler dataFileHandler) + public void stopWorldGen(GeneratedFullDataSourceProvider dataFileHandler) { AbstractWorldGenState worldGenState = this.worldGenStateRef.get(); if (worldGenState == null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index ed6ed0232..e2f512aa5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -59,7 +59,7 @@ public class LodQuadTree extends QuadTree implements AutoClose public final int blockRenderDistanceDiameter; - private final IFullDataSourceProvider fullDataSourceProvider; + private final FullDataSourceProviderV2 fullDataSourceProvider; private final IRenderSourceProvider renderSourceProvider; /** @@ -91,7 +91,7 @@ public class LodQuadTree extends QuadTree implements AutoClose public LodQuadTree( IDhClientLevel level, int viewDiameterInBlocks, int initialPlayerBlockX, int initialPlayerBlockZ, - IFullDataSourceProvider fullDataSourceProvider, + FullDataSourceProviderV2 fullDataSourceProvider, IRenderSourceProvider renderSourceProvider) { super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index dc557d1db..5eab9d782 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; import com.seibel.distanthorizons.core.enums.EDhDirection; -import com.seibel.distanthorizons.core.file.fullDatafile.IFullDataSourceProvider; +import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -418,7 +418,7 @@ public class LodRenderSection implements IDebugRenderable public boolean missingPositionsCalculated() { return this.missingPositionsCalculated; } public int ungeneratedPositionCount() { return (this.missingGenerationPos != null) ? this.missingGenerationPos.size() : 0; } - public void tryQueuingMissingLodRetrieval(IFullDataSourceProvider fullDataSourceProvider) + public void tryQueuingMissingLodRetrieval(FullDataSourceProviderV2 fullDataSourceProvider) { if (fullDataSourceProvider.canRetrieveMissingDataSources() && fullDataSourceProvider.canQueueRetrieval()) { From 94e34267792892752e6e2152297eedbcf820a8d5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 17 Mar 2024 17:36:28 -0500 Subject: [PATCH 067/183] Remove ISourceProvider --- .../file/AbstractLegacyDataSourceHandler.java | 7 +++---- .../file/AbstractNewDataSourceHandler.java | 6 ++---- .../core/file/ISourceProvider.java | 21 ------------------- .../renderfile/IRenderSourceProvider.java | 6 ++++-- 4 files changed, 9 insertions(+), 31 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java index 117177e65..18b82fe15 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java @@ -34,8 +34,9 @@ import java.util.zip.Adler32; import java.util.zip.CheckedOutputStream; @Deprecated -public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> - implements ISourceProvider +public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> + implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); @@ -128,7 +129,6 @@ public abstract class AbstractLegacyDataSourceHandler getAsync(DhSectionPos pos) { ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); @@ -190,7 +190,6 @@ public abstract class AbstractLegacyDataSourceHandler updateDataSourceAsync(FullDataSourceV2 inputDataSource) { ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 7648f16de..3d804d013 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -24,8 +24,8 @@ public abstract class AbstractNewDataSourceHandler , TDTO extends IBaseDTO, TRepo extends AbstractDhRepo, - TDhLevel extends IDhLevel> - implements ISourceProvider + TDhLevel extends IDhLevel> + implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -112,7 +112,6 @@ public abstract class AbstractNewDataSourceHandler * * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ - @Override public CompletableFuture getAsync(DhSectionPos pos) { ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); @@ -160,7 +159,6 @@ public abstract class AbstractNewDataSourceHandler // data updating // //===============// - @Override public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) { ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java deleted file mode 100644 index 76f04dd70..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/ISourceProvider.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.util.concurrent.CompletableFuture; - -/** - * Base for all data source providers - * - * @see IRenderSourceProvider - */ -public interface ISourceProvider, TDhLevel extends IDhLevel> extends AutoCloseable -{ - CompletableFuture getAsync(DhSectionPos pos); - - CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputData); - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java index d1b1f8dff..8db7634b8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core.file.renderfile; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.file.ISourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; @@ -31,8 +30,11 @@ import java.util.concurrent.CompletableFuture; /** * Handles reading, writing, and updating {@link ColumnRenderSource}'s.
* Should be backed by a database handled by a {@link RenderDataRepo}. + * + * @deprecated an interface isn't necessary for the single render source provider we have */ -public interface IRenderSourceProvider extends ISourceProvider +@Deprecated +public interface IRenderSourceProvider extends AutoCloseable { CompletableFuture getAsync(DhSectionPos pos); From 6557a1a7bd28a72ca852ad18637cb79c1bb946a6 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 18 Mar 2024 20:01:39 -0500 Subject: [PATCH 068/183] rename FullDataSourceV2.createFromCompleteDataSource --- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 2 +- .../core/file/fullDatafile/FullDataSourceProviderV2.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index ea7629db4..7ed55e6b2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -128,7 +128,7 @@ public class FullDataSourceV2 implements IDataSource public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } - public static FullDataSourceV2 createFromCompleteDataSource(FullDataSourceV1 legacyData) + public static FullDataSourceV2 createFromLegacyDataSourceV1(FullDataSourceV1 legacyData) { if (FullDataSourceV1.WIDTH != WIDTH) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 4f85c5bda..70dd8bf1e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -328,7 +328,7 @@ public class FullDataSourceProviderV2 { // convert the legacy data source to the new format, // this is a relatively cheap operation - FullDataSourceV2 newDataSource = FullDataSourceV2.createFromCompleteDataSource(legacyDataSource); + FullDataSourceV2 newDataSource = FullDataSourceV2.createFromLegacyDataSourceV1(legacyDataSource); newDataSource.applyToParent = true; // the actual update process can be moderately expensive due to having to update From 5b67f60e6ff62734190c7bbab3c62418c6079fe1 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 18 Mar 2024 20:05:12 -0500 Subject: [PATCH 069/183] Fix black block lighting for super flat worlds --- .../fullData/sources/FullDataSourceV2.java | 39 +++++++++---------- .../transformers/LodDataBuilder.java | 10 ----- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 7ed55e6b2..709da6dde 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -87,7 +87,9 @@ public class FullDataSourceV2 implements IDataSource /** * stored x/z, y - * The y data should be sorted from bottom to top + * The y data should be sorted from top to bottom + * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately + * there's something in the render data logic that expects this order so we can't change it right now */ public long[][] dataPoints; @@ -149,16 +151,6 @@ public class FullDataSourceV2 implements IDataSource long[] dataColumn = legacyData.get(x, z); if (dataColumn != null && dataColumn.length != 0) { - // reverse the array so index 0 is the lowest, - // this is necessary for later logic - // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < dataColumn.length / 2; i++) - { - long temp = dataColumn[i]; - dataColumn[i] = dataColumn[dataColumn.length - i - 1]; - dataColumn[dataColumn.length - i - 1] = temp; - } - int index = relativePosToIndex(x, z); dataPoints[index] = dataColumn; @@ -422,7 +414,10 @@ public class FullDataSourceV2 implements IDataSource { ArrayList newColumnList = new ArrayList<>(); - int[] currentDatapointIndex = new int[] { 0, 0, 0, 0 }; + // special numbers: + // -2 = the column's height hasn't been determined yet + // -1 = we've reached the end of the column + int[] currentDatapointIndex = new int[] { -2, -2, -2, -2 }; int lastId = 0; byte lastBlockLight = 0; @@ -444,18 +439,16 @@ public class FullDataSourceV2 implements IDataSource } - long[] datapointsForYSlice = new long[4]; - - // scary double loop but, // this will only ever loop 4 times, // once for each of the 4 input columns + long[] datapointsForYSlice = new long[4]; int colIndex = 0; - for (int inputX = x; inputX < x +2; inputX++) + for (int inputX = x; inputX < x + 2; inputX++) { for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) { - // TODO throw an assertion if the column isn't in order or just fix it... + // TODO throw an assertion if the column isn't in top-down order or just fix it... long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; if (inputDataArray == null || inputDataArray.length == 0) { @@ -463,6 +456,12 @@ public class FullDataSourceV2 implements IDataSource continue; } + // determine the last index (the lowest data point) for each column + if (currentDatapointIndex[colIndex] == -2) + { + currentDatapointIndex[colIndex] = inputDataArray.length - 1; + } + int dataPointIndex = currentDatapointIndex[colIndex]; if (dataPointIndex == -1) @@ -485,11 +484,11 @@ public class FullDataSourceV2 implements IDataSource } else if (blockY >= datapointMaxY) { - // this y-slice is above this datapoint, + // this y-slice is above the current datapoint, // try the next data point - int newDatapointIndex = currentDatapointIndex[colIndex] + 1; - if (newDatapointIndex >= inputDataArray.length) + int newDatapointIndex = currentDatapointIndex[colIndex] - 1; + if (newDatapointIndex < 0) { // went to far, no additional data present newDatapointIndex = -1; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index eb796421f..a4c326cba 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -188,16 +188,6 @@ public class LodDataBuilder } longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - // reverse the array so index 0 is the lowest, - // this is necessary for later logic - // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < longs.size() / 2; i++) - { - long temp = longs.getLong(i); - longs.set(i, longs.getLong(longs.size() - i - 1)); - longs.set(longs.size() - i - 1, temp); - } - dataSource.setSingleColumn(longs.toLongArray(), chunkX + chunkOffsetX, chunkZ + chunkOffsetZ, From 805429722fc1d4c4152ea7a181855820c4980e11 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 18 Mar 2024 21:14:35 -0500 Subject: [PATCH 070/183] Fix an issue where downsampling would sometimes corrupt the data souce --- .../fullData/sources/FullDataSourceV2.java | 81 +++++++++++++++++-- 1 file changed, 75 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 709da6dde..9994c39be 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -34,6 +34,7 @@ import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; +import com.seibel.distanthorizons.coreapi.ModInfo; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -55,7 +56,13 @@ public class FullDataSourceV2 implements IDataSource private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ private static final boolean RUN_UPDATE_DEV_VALIDATION = false; - private static final boolean RUN_V1_MIGRATION_VALIDATION = false; + private static final boolean RUN_V1_MIGRATION_CONSTRUCTOR_VALIDATION = ModInfo.IS_DEV_BUILD; + /** + * If the data column order isn't correct + * block lighting may appear broken + * and/or certain detail level LODs may not appear at all. + */ + private static final boolean RUN_DATA_ORDER_VALIDATION = ModInfo.IS_DEV_BUILD; /** measured in data columns */ public static final int WIDTH = 64; @@ -86,8 +93,8 @@ public class FullDataSourceV2 implements IDataSource public byte[] columnGenerationSteps; /** - * stored x/z, y - * The y data should be sorted from top to bottom + * stored x/z, y
+ * The y data should be sorted from top to bottom
* TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately * there's something in the render data logic that expects this order so we can't change it right now */ @@ -196,7 +203,7 @@ public class FullDataSourceV2 implements IDataSource // should only be used if debugging, this is a very expensive operation - if (RUN_V1_MIGRATION_VALIDATION) + if (RUN_V1_MIGRATION_CONSTRUCTOR_VALIDATION) { for (int x = 0; x < WIDTH; x++) { @@ -318,6 +325,11 @@ public class FullDataSourceV2 implements IDataSource System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); this.remapDataColumn(index, remappedIds); + if (RUN_DATA_ORDER_VALIDATION) + { + throwIfDataColumnInWrongOrder(inputDataSource.pos, this.dataPoints[index]); + } + // we only need to see if the data was changed in one column if (!dataChanged) { @@ -375,6 +387,11 @@ public class FullDataSourceV2 implements IDataSource long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); this.dataPoints[recipientIndex] = mergedInputDataArray; + if (RUN_DATA_ORDER_VALIDATION) + { + throwIfDataColumnInWrongOrder(inputDataSource.pos, this.dataPoints[recipientIndex]); + } + // mapping this.remapDataColumn(recipientIndex, remappedIds); @@ -460,6 +477,11 @@ public class FullDataSourceV2 implements IDataSource if (currentDatapointIndex[colIndex] == -2) { currentDatapointIndex[colIndex] = inputDataArray.length - 1; + + if (RUN_DATA_ORDER_VALIDATION) + { + throwIfDataColumnInWrongOrder(inputDataSource.pos, inputDataArray); + } } @@ -552,13 +574,38 @@ public class FullDataSourceV2 implements IDataSource } - - + // convert the arraylist to an array long[]mergedInputDataArray = new long[newColumnList.size()]; for (int i = 0; i < mergedInputDataArray.length; i++) { mergedInputDataArray[i] = newColumnList.get(i); } + + + // flip the array if necessary + // TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray + // to sometimes be in a different order? Is it potentially related to what detail level is coming in? + { + long firstDataPoint = mergedInputDataArray[0]; + int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); + + long lastDataPoint = mergedInputDataArray[mergedInputDataArray.length - 1]; + int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); + + if (firstBottomY < lastBottomY) + { + // reverse the array so index 0 is the highest, + // this is necessary for later logic + // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java + for(int i = 0; i < mergedInputDataArray.length / 2; i++) + { + long temp = mergedInputDataArray[i]; + mergedInputDataArray[i] = mergedInputDataArray[mergedInputDataArray.length - i - 1]; + mergedInputDataArray[mergedInputDataArray.length - i - 1] = temp; + } + } + } + return mergedInputDataArray; } /** @@ -677,6 +724,28 @@ public class FullDataSourceV2 implements IDataSource return (relX * WIDTH) + relZ; } + /** + * Throws an exception if the given + * full data column array is in the wrong order + * IE if the first data point is the lowest and the last data point is the highest. + * Data columns should be in reverse order, IE the first data point should be the highest data point. + * + * @see FullDataSourceV2#dataPoints + */ + public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, long[] dataArray) throws IllegalStateException + { + long firstDataPoint = dataArray[0]; + int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); + + long lastDataPoint = dataArray[dataArray.length - 1]; + int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); + + if (firstBottomY < lastBottomY) + { + throw new IllegalStateException("Incorrect data point order at pos: "+pos+", first datapoint bottom Y ["+firstBottomY+"], last datapoint bottom Y ["+lastBottomY+"]."); + } + } + //=====================// From 9f195231db7056c3ede667e3385bd7e6ef169da2 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 19 Mar 2024 06:48:26 -0500 Subject: [PATCH 071/183] up the version number 2.0.2 -> 2.0.3 --- .../main/java/com/seibel/distanthorizons/coreapi/ModInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java b/api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java index 6665b77be..723187c4e 100644 --- a/api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java +++ b/api/src/main/java/com/seibel/distanthorizons/coreapi/ModInfo.java @@ -34,7 +34,7 @@ public final class ModInfo public static final String NAME = "DistantHorizons"; /** Human-readable version of NAME */ public static final String READABLE_NAME = "Distant Horizons"; - public static final String VERSION = "2.0.2-a-dev"; + public static final String VERSION = "2.0.3-a-dev"; /** Returns true if the current build is an unstable developer build, false otherwise. */ public static boolean IS_DEV_BUILD = VERSION.toLowerCase().contains("dev"); From c537084d072463a139e8592110b3109467e01d5f Mon Sep 17 00:00:00 2001 From: cola98765 Date: Tue, 19 Mar 2024 12:27:05 +0000 Subject: [PATCH 072/183] potential fix to bright blocks when they reach build height limit --- .../transformers/LodDataBuilder.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index a4c326cba..2a3346f03 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -136,9 +136,22 @@ public class LodDataBuilder IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting - byte blockLight = (byte) chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ); - byte skyLight = (byte) chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ); + + + byte blockLight; + byte skyLight; + if (lastY < chunkWrapper.getMaxBuildHeight()) + { + // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting + blockLight = (byte) chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ); + skyLight = (byte) chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ); + } + else + { + //we are at the height limit. There are no torches here, and sky is not obscured. + blockLight = 0; + skyLight = 15; + } // determine the starting Y Pos From 8f9caa5d9a5eef68b6de034c5826957dacc46ca6 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 19 Mar 2024 07:27:57 -0500 Subject: [PATCH 073/183] Remove vacuum call in createFullDataSourceV2Tables SQL script --- .../sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql index 82fc3360b..2eb771e0e 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql @@ -13,11 +13,6 @@ where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0; --batch-- --- shrink the database file after having removed low detail legacy LODs -VACUUM; - ---batch-- - CREATE TABLE FullData ( -- compound primary key DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8 From a1950ebccc68dee9aa9a505abb5e6b81529e01fc Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 19 Mar 2024 07:28:14 -0500 Subject: [PATCH 074/183] prevent world gen queue until half of the update tasks have finished --- .../core/file/fullDatafile/GeneratedFullDataSourceProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 3e3117458..e2bc45e77 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -188,7 +188,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); - if (updateExecutor == null || updateExecutor.getQueue().size() >= MAX_UPDATE_TASK_COUNT) + if (updateExecutor == null || updateExecutor.getQueue().size() >= MAX_UPDATE_TASK_COUNT / 2) { // don't queue additional world gen requests if the updater is behind return false; From 9fb5182b785780a8f6cc55fd721003293eed6291 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 19 Mar 2024 20:07:21 -0500 Subject: [PATCH 075/183] Change the SQLite journal mode DELETE -> TRUNCATE --- .gitignore | 1 + .../core/sql/DatabaseUpdater.java | 55 +++++++++---------- .../0030-sqlite-changeTableJournaling.sql | 9 +++ core/src/main/resources/sqlScripts/readme.md | 9 +++ .../main/resources/sqlScripts/scriptList.txt | 1 + 5 files changed, 47 insertions(+), 28 deletions(-) create mode 100644 core/src/main/resources/sqlScripts/0030-sqlite-changeTableJournaling.sql diff --git a/.gitignore b/.gitignore index 42889b9cc..fe6126d6e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ forge*changelog.txt # Sqlite databases *.sqlite +*.sqlite-journal diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index c6ce1ee99..ae5738e1d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -41,6 +41,11 @@ public class DatabaseUpdater public static final String SCHEMA_TABLE_NAME = "Schema"; /** Since java can only run one sql query at a time this string is used to split up our scripts into individual queries. */ public static final String UPDATE_SCRIPT_BATCH_SEPARATOR = "--batch--"; + /** + * If this comment is present anywhere in the auto update script, then transactions won't be used.
+ * This is necessary for some commands that will auto-commit after running, IE: "PRAGMA journal_mode = TRUNCATE" + */ + public static final String UPDATE_SCRIPT_NO_TRANSACTION_FLAG = "--No Transactions--"; private static final String SQL_SCRIPT_RESOURCE_FOLDER = "sqlScripts/"; /** @@ -101,49 +106,43 @@ public class DatabaseUpdater // split up each individual statement so Java can handle the script as a whole String[] fileUpdateSqlArray = resource.queryString.split(UPDATE_SCRIPT_BATCH_SEPARATOR); + boolean transactScript = resource.queryString.contains(UPDATE_SCRIPT_NO_TRANSACTION_FLAG); + + + Connection connection = repo.getConnection(); + connection.setAutoCommit(!transactScript); + try (Statement statement = connection.createStatement()) { + statement.setQueryTimeout(AbstractDhRepo.TIMEOUT_SECONDS); + // adding the scripts to a batched statement allows them to execute together and rollback together if there are any issues for (String updateSql : fileUpdateSqlArray) { - statement.addBatch(updateSql); + statement.execute(updateSql); } - - statement.setQueryTimeout(AbstractDhRepo.TIMEOUT_SECONDS); - // TODO this should be a complete transaction, but doesn't appear to be - int[] numberOfRowsModifiedArray = statement.executeBatch(); - - - // confirm the scripts ran successfully - for (;sqlIndex < numberOfRowsModifiedArray.length; sqlIndex++) + if (transactScript) { - int numberOfRowsModified = numberOfRowsModifiedArray[sqlIndex]; - if (numberOfRowsModified >= 0) - { - // the statement completed successfully - continue; - } - else if (numberOfRowsModified == Statement.EXECUTE_FAILED) - { - LOGGER.error("Execute failed for auto update script: [" + resource.name + "], query: [" + fileUpdateSqlArray[sqlIndex] + "]. Changes have been rolled back.", new SQLException()); - } - else if (numberOfRowsModified == Statement.SUCCESS_NO_INFO) - { - LOGGER.error("Execute failed for auto update script: [" + resource.name + "], query: [" + fileUpdateSqlArray[sqlIndex] + "]. Changes may not have been rolled back.", new SQLException()); - } - else - { - LOGGER.error("Unexpected error state [" + numberOfRowsModified + "] returned for auto update script: [" + resource.name + "], query: [" + fileUpdateSqlArray[sqlIndex] + "].", new SQLException()); - } + connection.commit(); } } catch (SQLException e) { - LOGGER.error("Unexpected SQL Error: ["+e.getMessage()+"] returned for auto update script: [" + resource.name + "], query: [" + fileUpdateSqlArray[sqlIndex] + "].", new SQLException()); + connection.rollback(); + LOGGER.error( + "Unexpected SQL Error: ["+e.getMessage()+"] " + + "returned for auto update script: [" + resource.name + "], " + + "query: [" + fileUpdateSqlArray[sqlIndex] + "]. " + + "Changes should have been rolled back.", new SQLException()); throw e; } + + if (transactScript) + { + connection.setAutoCommit(true); + } } catch (RuntimeException e) { diff --git a/core/src/main/resources/sqlScripts/0030-sqlite-changeTableJournaling.sql b/core/src/main/resources/sqlScripts/0030-sqlite-changeTableJournaling.sql new file mode 100644 index 000000000..6dd0b6898 --- /dev/null +++ b/core/src/main/resources/sqlScripts/0030-sqlite-changeTableJournaling.sql @@ -0,0 +1,9 @@ + +-- this PRAGMA will automatically commit, so we have to disable +-- DH's automatic transactions, otherwise the connection will throw an error + +--No Transactions-- + +-- James ran into some issues where Windows had trouble deleting the Journal file, +-- using TRUNCATE should fix that issue +PRAGMA journal_mode = TRUNCATE; diff --git a/core/src/main/resources/sqlScripts/readme.md b/core/src/main/resources/sqlScripts/readme.md index 18fdb2456..683e39c0f 100644 --- a/core/src/main/resources/sqlScripts/readme.md +++ b/core/src/main/resources/sqlScripts/readme.md @@ -38,5 +38,14 @@ CREATE TABLE TableTwo( ); ``` +### PRAGMA Auto Commits +Certain queries will auto commit after running, specifically certain `PRAGMA` commands. In that case we have to disable DH's automatic transactions by putting `--No Transactions--` somewhere in the file. Otherwise, when the system attempts to commit, it will fail due to the PRAGMA having already committed itself. +Due to how these commands work it's best to only have a single command in the file to prevent confusion and potential database corruption. + +```roomsql +--No Transactions-- + +PRAGMA journal_mode = TRUNCATE; +``` diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index b41618ae7..ee42f56b0 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -1,3 +1,4 @@ 0010-sqlite-createInitialDataTables.sql 0020-sqlite-createFullDataSourceV2Tables.sql +0030-sqlite-changeTableJournaling.sql From d926d11d3cd7952f163d7921095cff76cc2cdad0 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 19 Mar 2024 20:53:07 -0500 Subject: [PATCH 076/183] Reduce string concatenations in assertions --- .../core/render/LodRenderSection.java | 6 +- .../core/render/glObject/buffer/GLBuffer.java | 12 +++- .../core/util/FullDataPointUtilV1.java | 41 +++++++++--- .../core/util/FullDataPointUtilV2.java | 41 +++++++++--- .../core/util/RenderDataPointUtil.java | 65 +++++++++++++++---- .../core/util/objects/quadTree/QuadTree.java | 5 +- .../core/world/DhClientServerWorld.java | 5 +- 7 files changed, 139 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 5eab9d782..9063ec741 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -377,7 +377,11 @@ public class LodRenderSection implements IDebugRenderable return false; } - LodUtil.assertTrue(newBuffer.buffersUploaded, "The buffer future for " + this.pos + " returned an un-built buffer."); + if (!newBuffer.buffersUploaded) + { + LodUtil.assertNotReach("The buffer future for " + this.pos + " returned an un-built buffer."); + } + ColumnRenderBuffer oldBuffer = this.activeRenderBufferRef.getAndSet(newBuffer); if (oldBuffer != null) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java index e4d8b8d52..da1d5f942 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java @@ -94,8 +94,10 @@ public class GLBuffer implements AutoCloseable protected void create(boolean asBufferStorage) { - LodUtil.assertTrue(GLProxy.getInstance().getGlContext() != EGLProxyContext.NONE, - "Thread ["+Thread.currentThread()+"] tried to create a GLBuffer outside a OpenGL context."); + if (GLProxy.getInstance().getGlContext() == EGLProxyContext.NONE) + { + LodUtil.assertNotReach("Thread ["+Thread.currentThread()+"] tried to create a GLBuffer outside a OpenGL context."); + } this.id = GL32.glGenBuffers(); this.bufferStorage = asBufferStorage; @@ -165,7 +167,10 @@ public class GLBuffer implements AutoCloseable { LodUtil.assertTrue(!uploadMethod.useEarlyMapping, "UploadMethod signal that this should use Mapping instead of uploadBuffer!"); int bbSize = bb.limit() - bb.position(); - LodUtil.assertTrue(bbSize <= maxExpansionSize, "maxExpansionSize is ["+maxExpansionSize+"] but buffer size is ["+bbSize+"]!"); + if (bbSize > maxExpansionSize) + { + LodUtil.assertNotReach("maxExpansionSize is [" + maxExpansionSize + "] but buffer size is [" + bbSize + "]!"); + } GLProxy.GL_LOGGER.debug("Uploading buffer with ["+new UnitBytes(bbSize)+"]."); // Don't upload an empty buffer @@ -219,6 +224,7 @@ public class GLBuffer implements AutoCloseable protected void uploadSubData(ByteBuffer bb, int maxExpansionSize, int bufferDataHint) { LodUtil.assertTrue(!this.bufferStorage, "Buffer is bufferStorage but its trying to use subData upload method!"); + int bbSize = bb.limit() - bb.position(); if (this.size < bbSize || this.size > bbSize * BUFFER_SHRINK_TRIGGER) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java index 010b03335..065d1b890 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java @@ -1,6 +1,7 @@ package com.seibel.distanthorizons.core.util; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; +import com.seibel.distanthorizons.coreapi.ModInfo; /** * Only for Legacy support
@@ -35,6 +36,8 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour */ public class FullDataPointUtilV1 { + public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD; + /** Represents the data held by an empty data point */ public static final int EMPTY_DATA_POINT = 0; @@ -69,9 +72,23 @@ public class FullDataPointUtilV1 @Deprecated public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) { - LodUtil.assertTrue(relMinY >= 0 && relMinY < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y["+relMinY+"] out of range!"); - LodUtil.assertTrue(height > 0 && height < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with height["+height+"] out of range!"); - LodUtil.assertTrue(relMinY + height <= RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y+depth["+(relMinY+height)+"] out of range!"); + if (RUN_VALIDATION) + { + // assertions are inside if-blocks to prevent unnecessary string concatenations + if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with y[" + relMinY + "] out of range!"); + } + if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!"); + } + if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with y+depth[" + (relMinY + height) + "] out of range!"); + } + } + long data = 0; data |= id & ID_MASK; @@ -80,11 +97,19 @@ public class FullDataPointUtilV1 data |= (long) blockLight << BLOCK_LIGHT_OFFSET; data |= (long) skyLight << SKY_LIGHT_OFFSET; - LodUtil.assertTrue(getId(data) == id && getHeight(data) == height && getBottomY(data) == relMinY && getBlockLight(data) == Byte.toUnsignedInt(blockLight) && getSkyLight(data) == Byte.toUnsignedInt(skyLight), - "Trying to create datapoint with " + - "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + - "but got " + - "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); + + if (RUN_VALIDATION) + { + if (getId(data) != id || getHeight(data) != height || getBottomY(data) != relMinY + || getBlockLight(data) != Byte.toUnsignedInt(blockLight) || getSkyLight(data) != Byte.toUnsignedInt(skyLight)) + { + LodUtil.assertNotReach( + "Trying to create datapoint with " + + "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + + "but got " + + "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); + } + } return data; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java index 0d9540567..491d0c256 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.util; +import com.seibel.distanthorizons.coreapi.ModInfo; import org.jetbrains.annotations.Contract; /** @@ -51,6 +52,8 @@ import org.jetbrains.annotations.Contract; */ public class FullDataPointUtilV2 { + public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD; + /** Represents the data held by an empty data point */ public static final int EMPTY_DATA_POINT = 0; @@ -82,9 +85,23 @@ public class FullDataPointUtilV2 */ public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) { - LodUtil.assertTrue(relMinY >= 0 && relMinY < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y["+relMinY+"] out of range!"); - LodUtil.assertTrue(height > 0 && height < RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with height["+height+"] out of range!"); - LodUtil.assertTrue(relMinY + height <= RenderDataPointUtil.MAX_WORLD_Y_SIZE, "Trying to create datapoint with y+depth["+(relMinY+height)+"] out of range!"); + if (RUN_VALIDATION) + { + // assertions are inside if-blocks to prevent unnecessary string concatenations + if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with y[" + relMinY + "] out of range!"); + } + if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!"); + } + if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with y+depth[" + (relMinY + height) + "] out of range!"); + } + } + long data = 0; data |= id & ID_MASK; @@ -93,11 +110,19 @@ public class FullDataPointUtilV2 data |= (long) blockLight << BLOCK_LIGHT_OFFSET; data |= (long) skyLight << SKY_LIGHT_OFFSET; - LodUtil.assertTrue(getId(data) == id && getHeight(data) == height && getBottomY(data) == relMinY && getBlockLight(data) == Byte.toUnsignedInt(blockLight) && getSkyLight(data) == Byte.toUnsignedInt(skyLight), - "Trying to create datapoint with " + - "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + - "but got " + - "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); + + if (RUN_VALIDATION) + { + if (getId(data) != id || getHeight(data) != height || getBottomY(data) != relMinY + || getBlockLight(data) != Byte.toUnsignedInt(blockLight) || getSkyLight(data) != Byte.toUnsignedInt(skyLight)) + { + LodUtil.assertNotReach( + "Trying to create datapoint with " + + "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + + "but got " + + "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); + } + } return data; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 316255595..85fd5ae1a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -23,6 +23,7 @@ import com.seibel.distanthorizons.core.level.AbstractDhLevel; import com.seibel.distanthorizons.core.logging.SpamReducedLogger; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView; +import com.seibel.distanthorizons.coreapi.ModInfo; /** @@ -63,7 +64,7 @@ public class RenderDataPointUtil // When converting to or from an int a 128 should be added or removed. // If there is a bug with color then it's probably caused by this. - private static final SpamReducedLogger warnLogger = new SpamReducedLogger(1); + public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD; public final static int EMPTY_DATA = 0; @@ -123,20 +124,56 @@ public class RenderDataPointUtil public static long createDataPoint(int alpha, int red, int green, int blue, int height, int depth, int lightSky, int lightBlock, int irisBlockMaterialId) { - LodUtil.assertTrue(height >= 0 && height < MAX_WORLD_Y_SIZE, "Trying to create datapoint with height[" + height + "] out of range!"); - LodUtil.assertTrue(depth >= 0 && depth < MAX_WORLD_Y_SIZE, "Trying to create datapoint with depth[" + depth + "] out of range!"); + if (RUN_VALIDATION) + { + // assertions are inside if-blocks to prevent unnecessary string concatenations + if (height < 0 || height >= MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!"); + } + if (depth < 0 || depth >= MAX_WORLD_Y_SIZE) + { + LodUtil.assertNotReach("Trying to create datapoint with depth[" + depth + "] out of range!"); + } + + if (lightSky < 0 || lightSky >= 16) + { + LodUtil.assertNotReach("Trying to create datapoint with lightSky[" + lightSky + "] out of range!"); + } + if (lightBlock < 0 || lightBlock >= 16) + { + LodUtil.assertNotReach("Trying to create datapoint with lightBlock[" + lightBlock + "] out of range!"); + } + + if (irisBlockMaterialId < 0 || irisBlockMaterialId >= 256) + { + LodUtil.assertNotReach("Trying to create datapoint with irisBlockMaterialId[" + irisBlockMaterialId + "] out of range!"); + } + + if (alpha < 0 || alpha >= 256) + { + LodUtil.assertNotReach("Trying to create datapoint with alpha[" + alpha + "] out of range!"); + } + if (red < 0 || red >= 256) + { + LodUtil.assertNotReach("Trying to create datapoint with red[" + red + "] out of range!"); + } + if (green < 0 || green >= 256) + { + LodUtil.assertNotReach("Trying to create datapoint with green[" + green + "] out of range!"); + } + if (blue < 0 || blue >= 256) + { + LodUtil.assertNotReach("Trying to create datapoint with blue[" + blue + "] out of range!"); + } + + + if (depth > height) + { + LodUtil.assertNotReach("Trying to create datapoint with depth[" + depth + "] greater than height[" + height + "]!"); + } + } - LodUtil.assertTrue(lightSky >= 0 && lightSky < 16, "Trying to create datapoint with lightSky[" + lightSky + "] out of range!"); - LodUtil.assertTrue(lightBlock >= 0 && lightBlock < 16, "Trying to create datapoint with lightBlock[" + lightBlock + "] out of range!"); - - LodUtil.assertTrue(irisBlockMaterialId >= 0 && irisBlockMaterialId < 256, "Trying to create datapoint with irisBlockMaterialId[" + irisBlockMaterialId + "] out of range!"); - - LodUtil.assertTrue(alpha >= 0 && alpha < 256, "Trying to create datapoint with alpha[" + alpha + "] out of range!"); - LodUtil.assertTrue(red >= 0 && red < 256, "Trying to create datapoint with red[" + red + "] out of range!"); - LodUtil.assertTrue(green >= 0 && green < 256, "Trying to create datapoint with green[" + green + "] out of range!"); - LodUtil.assertTrue(blue >= 0 && blue < 256, "Trying to create datapoint with blue[" + blue + "] out of range!"); - - LodUtil.assertTrue(depth <= height, "Trying to create datapoint with depth[" + depth + "] greater than height[" + height + "]!"); long out = (long) (alpha >>> ALPHA_DOWNSIZE_SHIFT) << ALPHA_SHIFT | (red & RED_MASK) << RED_SHIFT diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java index 1b4dc969b..49b67e736 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java @@ -143,7 +143,10 @@ public class QuadTree topQuadNode = new QuadNode(rootPos, this.treeMaxDetailLevel); boolean successfullyAdded = this.topRingList.set(ringListPosX, ringListPosZ, topQuadNode); - LodUtil.assertTrue(successfullyAdded, "Failed to add top quadTree node at position: " + rootPos); + if (!successfullyAdded) + { + LodUtil.assertNotReach("Failed to add top quadTree node at position: " + rootPos); + } } if (!topQuadNode.sectionPos.contains(pos)) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java index 275d606ea..2f31238ac 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/world/DhClientServerWorld.java @@ -90,7 +90,10 @@ public class DhClientServerWorld extends AbstractDhWorld implements IDhClientWor IClientLevelWrapper clientLevelWrapper = (IClientLevelWrapper) levelWrapper; IServerLevelWrapper serverLevelWrapper = clientLevelWrapper.tryGetServerSideWrapper(); LodUtil.assertTrue(serverLevelWrapper != null); - LodUtil.assertTrue(clientLevelWrapper.getDimensionType().equals(serverLevelWrapper.getDimensionType()), "tryGetServerSideWrapper returned a level for a different dimension. ClientLevelWrapper dim: " + clientLevelWrapper.getDimensionType().getDimensionName() + " ServerLevelWrapper dim: " + serverLevelWrapper.getDimensionType().getDimensionName()); + if (!clientLevelWrapper.getDimensionType().equals(serverLevelWrapper.getDimensionType())) + { + LodUtil.assertNotReach("tryGetServerSideWrapper returned a level for a different dimension. ClientLevelWrapper dim: " + clientLevelWrapper.getDimensionType().getDimensionName() + " ServerLevelWrapper dim: " + serverLevelWrapper.getDimensionType().getDimensionName()); + } DhClientServerLevel level = this.levelWrapperByDhLevel.get(serverLevelWrapper); From 3b9962d7dcb2ef78293c9643b6c4950541afb4d9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 19 Mar 2024 21:10:27 -0500 Subject: [PATCH 077/183] Fix transactionScript auto update variable flipped --- .../distanthorizons/core/sql/DatabaseUpdater.java | 2 +- core/src/test/java/tests/SqliteSetupTest.java | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java index ae5738e1d..6d5c70b61 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -106,7 +106,7 @@ public class DatabaseUpdater // split up each individual statement so Java can handle the script as a whole String[] fileUpdateSqlArray = resource.queryString.split(UPDATE_SCRIPT_BATCH_SEPARATOR); - boolean transactScript = resource.queryString.contains(UPDATE_SCRIPT_NO_TRANSACTION_FLAG); + boolean transactScript = !resource.queryString.contains(UPDATE_SCRIPT_NO_TRANSACTION_FLAG); diff --git a/core/src/test/java/tests/SqliteSetupTest.java b/core/src/test/java/tests/SqliteSetupTest.java index 350e8edbd..3e19bd973 100644 --- a/core/src/test/java/tests/SqliteSetupTest.java +++ b/core/src/test/java/tests/SqliteSetupTest.java @@ -43,12 +43,12 @@ public class SqliteSetupTest Assert.assertTrue("Unable to delete test database.", dbFile.delete()); } - @Test - public void testInMemorySqlite() - { - String databaseLocation = ":memory:"; - testSqliteDatabase(DATABASE_TYPE, databaseLocation); - } + //@Test + //public void testInMemorySqlite() + //{ + // String databaseLocation = ":memory:"; + // testSqliteDatabase(DATABASE_TYPE, databaseLocation); + //} From 4cdc6c96323097ad57cb0509f67267666dc2fd11 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 20 Mar 2024 07:25:38 -0500 Subject: [PATCH 078/183] Compress the column gen step in the database --- .../core/sql/dto/FullDataSourceV2DTO.java | 58 ++++++++++++++----- .../core/sql/repo/FullDataSourceV2Repo.java | 12 ++-- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index a9f5f77bb..51249a69c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -45,12 +45,12 @@ public class FullDataSourceV2DTO implements IBaseDTO /** only for the data array */ public int dataChecksum; - public byte[] dataByteArray; + public byte[] compressedDataByteArray; /** @see EDhApiWorldGenerationStep */ - public byte[] columnGenStepByteArray; + public byte[] compressedColumnGenStepByteArray; - public byte[] mappingByteArray; + public byte[] compressedMappingByteArray; public byte dataFormatVersion; public EDhApiDataCompressionMode compressionModeEnum; @@ -69,32 +69,34 @@ public class FullDataSourceV2DTO implements IBaseDTO public static FullDataSourceV2DTO CreateFromDataSource(FullDataSourceV2 dataSource, EDhApiDataCompressionMode compressionModeEnum) throws IOException { CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); + byte[] compressedWorldGenStepByteArray = writeGenerationStepsToBlob(dataSource.columnGenerationSteps, compressionModeEnum); byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum); return new FullDataSourceV2DTO( dataSource.getSectionPos(), - checkedDataPointArray.checksum, dataSource.columnGenerationSteps, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, compressedWorldGenStepByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY ); } + public FullDataSourceV2DTO( DhSectionPos pos, - int dataChecksum, byte[] columnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] dataByteArray, + int dataChecksum, byte[] compressedColumnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, - byte[] mappingByteArray, boolean applyToParent, + byte[] compressedMappingByteArray, boolean applyToParent, int levelMinY) { this.pos = pos; this.dataChecksum = dataChecksum; - this.columnGenStepByteArray = columnGenStepByteArray; + this.compressedColumnGenStepByteArray = compressedColumnGenStepByteArray; this.dataFormatVersion = dataFormatVersion; this.compressionModeEnum = compressionModeEnum; - this.dataByteArray = dataByteArray; - this.mappingByteArray = mappingByteArray; + this.compressedDataByteArray = compressedDataByteArray; + this.compressedMappingByteArray = compressedMappingByteArray; this.applyToParent = applyToParent; @@ -130,8 +132,8 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new IllegalStateException("There should only be one data format right now anyway."); } - dataSource.columnGenerationSteps = this.columnGenStepByteArray; - dataSource.dataPoints = readBlobToDataSourceDataArray(this.dataByteArray, this.compressionModeEnum); + dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); dataSource.mapping.clear(dataSource.getSectionPos()); // should only be null when used in a unit test @@ -142,7 +144,7 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); } - dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.mappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum)); + dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum)); } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; @@ -200,9 +202,9 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static long[][] readBlobToDataSourceDataArray(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static long[][] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); @@ -230,6 +232,30 @@ public class FullDataSourceV2DTO implements IBaseDTO } + private static byte[] writeGenerationStepsToBlob(byte[] columnGenStepByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); + + compressedOut.write(columnGenStepByteArray); + + compressedOut.flush(); + byteArrayOutputStream.close(); + + return byteArrayOutputStream.toByteArray(); + } + private static byte[] readBlobToGenerationSteps(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); + + byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + compressedIn.readFully(columnGenStepByteArray); + + return columnGenStepByteArray; + } + + private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -242,9 +268,9 @@ public class FullDataSourceV2DTO implements IBaseDTO return byteArrayOutputStream.toByteArray(); } - private static FullDataPointIdMap readBlobToDataMapping(byte[] dataByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + private static FullDataPointIdMap readBlobToDataMapping(byte[] compressedMappingByteArray, DhSectionPos pos, @NotNull ILevelWrapper levelWrapper, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException { - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedMappingByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); FullDataPointIdMap mapping = FullDataPointIdMap.deserialize(compressedIn, pos, levelWrapper); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 2a01b0528..20ecc7a92 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -116,9 +116,9 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Thu, 21 Mar 2024 17:11:04 -0500 Subject: [PATCH 079/183] change en_us "Horizontal Quality" -> "LOD Dropoff Distance" --- core/src/main/resources/assets/distanthorizons/lang/en_us.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index a8f26de75..81caf9e39 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -101,7 +101,7 @@ "distanthorizons.config.client.advanced.graphics.quality.horizontalScale.@tooltip": "How quickly LODs drop off in quality.\n\nLarger numbers will improve how distant terrain looks\nbut will increase memory and GPU usage.", "distanthorizons.config.client.advanced.graphics.quality.horizontalQuality": - "Horizontal Quality", + "LOD Dropoff Distance", "distanthorizons.config.client.advanced.graphics.quality.horizontalQuality.@tooltip": "How far apart drops in quality are.\n\nHigher settings will increase the distance between drops\nbut will increase memory and GPU usage.", "distanthorizons.config.client.advanced.graphics.quality.transparency": From f11e9a142f3b891aeee67c4c8be858f5ca25f1dd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 17:25:37 -0500 Subject: [PATCH 080/183] Add FullDataSourceV2 pooling and replace long[][] arrays with LongArrayList[] --- .../methods/data/DhApiTerrainDataRepo.java | 7 +- .../fullData/sources/FullDataSourceV1.java | 10 + .../fullData/sources/FullDataSourceV2.java | 230 ++++++++++-------- .../render/ColumnRenderSource.java | 7 +- .../FullDataToRenderDataTransformer.java | 15 +- .../transformers/LodDataBuilder.java | 11 +- .../file/AbstractNewDataSourceHandler.java | 27 +- .../core/file/IDataSource.java | 6 +- .../FullDataSourceProviderV2.java | 12 +- .../renderfile/RenderSourceFileHandler.java | 8 +- .../SubDimensionLevelMatcher.java | 13 +- .../core/sql/dto/FullDataSourceV2DTO.java | 62 +++-- 12 files changed, 248 insertions(+), 160 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 933b29210..d922064f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -46,6 +46,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.coreapi.util.math.Vec3d; import com.seibel.distanthorizons.coreapi.util.math.Vec3f; import com.seibel.distanthorizons.coreapi.util.math.Vec3i; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -220,10 +221,10 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.mapping; - long[] dataColumn = dataSource.get(relativePos.x, relativePos.z); + LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { - int dataColumnIndexCount = dataColumn.length; + int dataColumnIndexCount = dataColumn.size(); DhApiTerrainDataPoint[] returnArray = new DhApiTerrainDataPoint[dataColumnIndexCount]; long dataPoint; @@ -234,7 +235,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // search for a datapoint that contains the block y position for (int i = 0; i < dataColumnIndexCount; i++) { - dataPoint = dataColumn[i]; + dataPoint = dataColumn.getLong(i); if (!getSpecificYCoordinate) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 22b9db0c7..0558dfe4f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -387,6 +387,16 @@ public class FullDataSourceV1 implements IDataSource + //==================// + // override methods // + //==================// + + @Override + public void close() throws Exception + { /* not currently needed */ } + + + //================// // helper classes // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 9994c39be..54d2fff2d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -51,7 +52,7 @@ import java.util.concurrent.locks.ReentrantLock; * @see FullDataPointUtilV2 * @see FullDataSourceV1 */ -public class FullDataSourceV2 implements IDataSource +public class FullDataSourceV2 implements IDataSource, AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -98,7 +99,7 @@ public class FullDataSourceV2 implements IDataSource * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately * there's something in the render data logic that expects this order so we can't change it right now */ - public long[][] dataPoints; + public LongArrayList[] dataPoints; public boolean isEmpty; public boolean applyToParent = false; @@ -113,7 +114,7 @@ public class FullDataSourceV2 implements IDataSource private FullDataSourceV2(DhSectionPos pos) { this.pos = pos; - this.dataPoints = new long[WIDTH * WIDTH][]; + this.dataPoints = new LongArrayList[WIDTH * WIDTH]; this.mapping = new FullDataPointIdMap(pos); this.isEmpty = true; @@ -122,8 +123,8 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps = new byte[WIDTH * WIDTH]; } - public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } - private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -150,7 +151,7 @@ public class FullDataSourceV2 implements IDataSource // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; - long[][] dataPoints = new long[WIDTH * WIDTH][]; + LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH]; for (int x = 0; x < WIDTH; x++) { for (int z = 0; z < WIDTH; z++) @@ -159,7 +160,7 @@ public class FullDataSourceV2 implements IDataSource if (dataColumn != null && dataColumn.length != 0) { int index = relativePosToIndex(x, z); - dataPoints[index] = dataColumn; + dataPoints[index] = new LongArrayList(dataColumn); // convert the data point format @@ -212,13 +213,13 @@ public class FullDataSourceV2 implements IDataSource long[] legacyDataColumn = legacyData.get(x, z); if (legacyDataColumn != null && legacyDataColumn.length != 0) { - long[] newDataColumn = fullDataSource.get(x, z); + LongArrayList newDataColumn = fullDataSource.get(x, z); if (newDataColumn == null) { LodUtil.assertNotReach("Accessor column mismatch"); } - else if (legacyDataColumn.length != newDataColumn.length) + else if (legacyDataColumn.length != newDataColumn.size()) { LodUtil.assertNotReach("Accessor column length mismatch"); } @@ -226,7 +227,7 @@ public class FullDataSourceV2 implements IDataSource { for (int i = 0; i < legacyDataColumn.length; i++) { - if (legacyDataColumn[i] != newDataColumn[i]) + if (legacyDataColumn[i] != newDataColumn.getLong(i)) { LodUtil.assertNotReach("Data mismatch"); } @@ -247,7 +248,7 @@ public class FullDataSourceV2 implements IDataSource // data // //======// - public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } + public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } @@ -309,7 +310,7 @@ public class FullDataSourceV2 implements IDataSource { int index = relativePosToIndex(x, z); - long[] newDataArray = inputDataSource.dataPoints[index]; + LongArrayList newDataArray = inputDataSource.dataPoints[index]; if (newDataArray != null) { byte thisGenState = this.columnGenerationSteps[index]; @@ -318,11 +319,15 @@ public class FullDataSourceV2 implements IDataSource if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value && thisGenState <= inputGenState) { - long[] oldDataArray = this.dataPoints[index]; + LongArrayList oldDataArray = this.dataPoints[index]; // copy over the new data - this.dataPoints[index] = new long[newDataArray.length]; - System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); + if (this.dataPoints[index] == null) + { + this.dataPoints[index] = new LongArrayList(new long[newDataArray.size()]); + } + this.dataPoints[index].clear(); + this.dataPoints[index].addAll(newDataArray); this.remapDataColumn(index, remappedIds); if (RUN_DATA_ORDER_VALIDATION) @@ -383,8 +388,8 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps[recipientIndex] = inputGenStep; // data points - long[] oldDataArray = this.dataPoints[recipientIndex]; - long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); + LongArrayList oldDataArray = this.dataPoints[recipientIndex]; + LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); this.dataPoints[recipientIndex] = mergedInputDataArray; if (RUN_DATA_ORDER_VALIDATION) @@ -427,9 +432,13 @@ public class FullDataSourceV2 implements IDataSource } return minWorldGenStepValue; } - private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) + + private static ThreadLocal mergeTwoByTwoTempList = ThreadLocal.withInitial(() -> new LongArrayList(20)); + private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { - ArrayList newColumnList = new ArrayList<>(); + LongArrayList newColumnList = mergeTwoByTwoTempList.get(); + newColumnList.clear(); + // special numbers: // -2 = the column's height hasn't been determined yet @@ -466,8 +475,8 @@ public class FullDataSourceV2 implements IDataSource for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) { // TODO throw an assertion if the column isn't in top-down order or just fix it... - long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; - if (inputDataArray == null || inputDataArray.length == 0) + LongArrayList inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; + if (inputDataArray == null || inputDataArray.size() == 0) { currentDatapointIndex[colIndex] = -1; continue; @@ -476,7 +485,7 @@ public class FullDataSourceV2 implements IDataSource // determine the last index (the lowest data point) for each column if (currentDatapointIndex[colIndex] == -2) { - currentDatapointIndex[colIndex] = inputDataArray.length - 1; + currentDatapointIndex[colIndex] = inputDataArray.size() - 1; if (RUN_DATA_ORDER_VALIDATION) { @@ -491,7 +500,7 @@ public class FullDataSourceV2 implements IDataSource // went over the end continue; } - long datapoint = inputDataArray[dataPointIndex]; + long datapoint = inputDataArray.getLong(dataPointIndex); int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint); int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint); @@ -575,10 +584,10 @@ public class FullDataSourceV2 implements IDataSource // convert the arraylist to an array - long[]mergedInputDataArray = new long[newColumnList.size()]; - for (int i = 0; i < mergedInputDataArray.length; i++) + LongArrayList mergedInputDataArray = new LongArrayList(new long[newColumnList.size()]); + for (int i = 0; i < mergedInputDataArray.size(); i++) { - mergedInputDataArray[i] = newColumnList.get(i); + mergedInputDataArray.set(i, newColumnList.getLong(i)); } @@ -586,10 +595,10 @@ public class FullDataSourceV2 implements IDataSource // TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray // to sometimes be in a different order? Is it potentially related to what detail level is coming in? { - long firstDataPoint = mergedInputDataArray[0]; + long firstDataPoint = mergedInputDataArray.getLong(0); int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = mergedInputDataArray[mergedInputDataArray.length - 1]; + long lastDataPoint = mergedInputDataArray.getLong(mergedInputDataArray.size() - 1); int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -597,11 +606,11 @@ public class FullDataSourceV2 implements IDataSource // reverse the array so index 0 is the highest, // this is necessary for later logic // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < mergedInputDataArray.length / 2; i++) + for(int i = 0; i < mergedInputDataArray.size() / 2; i++) { - long temp = mergedInputDataArray[i]; - mergedInputDataArray[i] = mergedInputDataArray[mergedInputDataArray.length - i - 1]; - mergedInputDataArray[mergedInputDataArray.length - i - 1] = temp; + long temp = mergedInputDataArray.getLong(i); + mergedInputDataArray.set(i, mergedInputDataArray.getLong(mergedInputDataArray.size() - i - 1)); + mergedInputDataArray.set(mergedInputDataArray.size() - i - 1, temp); } } } @@ -615,15 +624,15 @@ public class FullDataSourceV2 implements IDataSource */ private void remapDataColumn(int dataPointIndex, int[] remappedIds) { - long[] dataColumn = this.dataPoints[dataPointIndex]; - for (int i = 0; i < dataColumn.length; i++) + LongArrayList dataColumn = this.dataPoints[dataPointIndex]; + for (int i = 0; i < dataColumn.size(); i++) { - dataColumn[i] = FullDataPointUtilV2.remap(remappedIds, dataColumn[i]); + dataColumn.set(i, FullDataPointUtilV2.remap(remappedIds, dataColumn.getLong(i))); } } - private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) + private static boolean areDataColumnsDifferent(LongArrayList oldDataArray, LongArrayList newDataArray) { - if (oldDataArray == null || oldDataArray.length != newDataArray.length) + if (oldDataArray == null || oldDataArray.size() != newDataArray.size()) { // new data was added/removed return true; @@ -631,8 +640,8 @@ public class FullDataSourceV2 implements IDataSource else { // check if the new column data is different - int oldArrayHash = Arrays.hashCode(oldDataArray); - int newArrayHash = Arrays.hashCode(newDataArray); + int oldArrayHash = oldDataArray.hashCode(); + int newArrayHash = newDataArray.hashCode(); return (newArrayHash != oldArrayHash); } } @@ -732,12 +741,12 @@ public class FullDataSourceV2 implements IDataSource * * @see FullDataSourceV2#dataPoints */ - public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, long[] dataArray) throws IllegalStateException + public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, LongArrayList dataArray) throws IllegalStateException { - long firstDataPoint = dataArray[0]; + long firstDataPoint = dataArray.getLong(0); int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = dataArray[dataArray.length - 1]; + long lastDataPoint = dataArray.getLong(dataArray.size() - 1); int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -767,7 +776,7 @@ public class FullDataSourceV2 implements IDataSource return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; @@ -778,9 +787,9 @@ public class FullDataSourceV2 implements IDataSource { // validate the incoming ID's int maxValidId = this.mapping.getMaxValidId(); - for (int i = 0; i < longArray.length; i++) + for (int i = 0; i < longArray.size(); i++) { - long dataPoint = longArray[i]; + long dataPoint = longArray.getLong(i); int id = FullDataPointUtilV2.getId(dataPoint); if (id > maxValidId) { @@ -858,64 +867,83 @@ public class FullDataSourceV2 implements IDataSource // pooling // //=========// - // TODO add pooled data sources - private static class Pooling + @Override + public void close() throws Exception { - /** used when pooling data sources */ - private final ArrayList cachedSources = new ArrayList<>(); - private final ReentrantLock cacheLock = new ReentrantLock(); - - - /** @return null if no pooled source exists */ - public FullDataSourceV1 tryGetPooledSource() - { - try - { - this.cacheLock.lock(); - - int index = this.cachedSources.size() - 1; - if (index == -1) - { - return null; - } - else - { - return this.cachedSources.remove(index); - } - } - finally - { - this.cacheLock.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. - */ - public void returnPooledDataSource(FullDataSourceV1 dataSource) - { - if (dataSource == null) - { - return; - } - else if (this.cachedSources.size() > 25) - { - return; - } - - try - { - this.cacheLock.lock(); - this.cachedSources.add(dataSource); - } - finally - { - this.cacheLock.unlock(); - } - } - + returnPooledDataSource(this); } + /** used when pooling data sources */ + private static final ArrayList CACHED_SOURCES = new ArrayList<>(); + private static final ReentrantLock CACHE_LOCK = new ReentrantLock(); + + + /** @return an empty data source if non are cached */ + public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData) + { + try + { + CACHE_LOCK.lock(); + + int index = CACHED_SOURCES.size() - 1; + if (index == -1) + { + return createEmpty(pos); + } + else + { + FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index);; + dataSource.pos = pos; + + if (clearData) + { + dataSource.mapping.clear(pos); + + for (int i = 0; i < dataSource.dataPoints.length; i++) + { + if (dataSource.dataPoints[i] != null) + { + dataSource.dataPoints[i].clear(); + } + } + + Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + } + + return dataSource; + } + } + finally + { + CACHE_LOCK.unlock(); + } + } + + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called. + */ + public static void returnPooledDataSource(FullDataSourceV2 dataSource) + { + if (dataSource == null) + { + return; + } + else if (CACHED_SOURCES.size() > 25) + { + return; + } + + try + { + CACHE_LOCK.lock(); + CACHED_SOURCES.add(dataSource); + } + finally + { + CACHE_LOCK.unlock(); + } + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 208aa4df5..fd2b8a011 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -37,6 +37,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.io.*; @@ -317,7 +318,7 @@ public class ColumnRenderSource implements IDataSource ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); - long[] dataColumn = inputFullDataSource.get(x, z); + LongArrayList dataColumn = inputFullDataSource.get(x, z); EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z); if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { @@ -476,6 +477,10 @@ public class ColumnRenderSource implements IDataSource return stringBuilder.toString(); } + @Override + public void close() throws Exception + { /* not currently needed */ } + //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index b4f12126c..28d7dd301 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.util.HashSet; @@ -117,7 +118,7 @@ public class FullDataToRenderDataTransformer throwIfThreadInterrupted(); ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - long[] dataColumn = fullDataSource.get(x, z); + LongArrayList dataColumn = fullDataSource.get(x, z); convertColumnData(level, fullDataSource.mapping, baseX + x, baseZ + z, columnArrayView, dataColumn); } } @@ -159,7 +160,7 @@ public class FullDataToRenderDataTransformer private static void iterateAndConvert( IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, - ColumnArrayView renderColumnData, long[] fullColumnData) + ColumnArrayView renderColumnData, LongArrayList fullColumnData) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); @@ -171,9 +172,9 @@ public class FullDataToRenderDataTransformer int columnOffset = 0; // goes from the top down - for (int i = 0; i < fullColumnData.length; i++) + for (int i = 0; i < fullColumnData.size(); i++) { - long fullData = fullColumnData[i]; + long fullData = fullColumnData.getLong(i); int bottomY = FullDataPointUtilV2.getBottomY(fullData); int blockHeight = FullDataPointUtilV2.getHeight(fullData); int id = FullDataPointUtilV2.getId(fullData); @@ -270,14 +271,14 @@ public class FullDataToRenderDataTransformer } // TODO what does this mean? - public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, long[] fullDataColumn) + public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, LongArrayList fullDataColumn) { - if (fullDataColumn == null || fullDataColumn.length == 0) + if (fullDataColumn == null || fullDataColumn.size() == 0) { return; } - int dataTotalLength = fullDataColumn.length; + int dataTotalLength = fullDataColumn.size(); if (dataTotalLength > columnArrayView.verticalSize()) { ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 2a3346f03..bcf79b56b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -131,7 +131,7 @@ public class LodDataBuilder { for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++) { - LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); + LongArrayList longs = new LongArrayList(new long[chunkWrapper.getHeight() / 4]); int lastY = chunkWrapper.getMaxBuildHeight(); IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; @@ -201,7 +201,7 @@ public class LodDataBuilder } longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - dataSource.setSingleColumn(longs.toLongArray(), + dataSource.setSingleColumn(longs, chunkX + chunkOffsetX, chunkZ + chunkOffsetZ, EDhApiWorldGenerationStep.LIGHT); @@ -230,7 +230,7 @@ public class LodDataBuilder // AND the below loop won't run. int size = (columnDataPoints != null) ? columnDataPoints.size() : 0; - long[] packedDataPoints = new long[size]; + LongArrayList packedDataPoints = new LongArrayList(new long[size]); for (int index = 0; index < size; index++) { DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index); @@ -240,13 +240,14 @@ public class LodDataBuilder (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); - packedDataPoints[index] = FullDataPointUtilV2.encode( + packedDataPoints.set(index, + FullDataPointUtilV2.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, (byte) (dataPoint.blockLightLevel), (byte) (dataPoint.skyLightLevel) - ); + )); } accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 3d804d013..f0489188c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -221,22 +221,23 @@ public abstract class AbstractNewDataSourceHandler } - // get or create the data source - TDataSource recipientDataSource = this.get(updatePos); - boolean dataModified = recipientDataSource.update(inputData, this.level); - - if (dataModified) + // try block allows for disposing of pooled data sources after the update is complete + try (TDataSource recipientDataSource = this.get(updatePos)) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(recipientDataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + boolean dataModified = recipientDataSource.update(inputData, this.level); + if (dataModified) { - if (listener != null) + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(recipientDataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - listener.OnDataSourceUpdated(recipientDataSource); + if (listener != null) + { + listener.OnDataSourceUpdated(recipientDataSource); + } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 11738fe7a..0da2c34b6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -11,11 +11,13 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import java.io.IOException; /** - * Base for all data sources. + * Base for all data sources.

+ * + * AutoCloseable Can be implemented to allow for disposing of pooled data sources.

* * @param there are times when we need specifically a client level vs a more generic level */ -public interface IDataSource extends IBaseDTO +public interface IDataSource extends IBaseDTO, AutoCloseable { DhSectionPos getSectionPos(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 70dd8bf1e..0e1b9f34d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -162,11 +162,11 @@ public class FullDataSourceProviderV2 { // TODO maybe just set children update flags to true? // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway - return FullDataSourceV2.createEmpty(pos); + return FullDataSourceV2.getPooledSource(pos, true); } @Override - protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); } + protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); } @@ -248,9 +248,11 @@ public class FullDataSourceProviderV2 childReadLock.lock(); this.lockedPosSet.add(childPos); - FullDataSourceV2 dataSource = this.get(childPos); - this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); - this.repo.setApplyToParent(childPos, false); + try (FullDataSourceV2 dataSource = this.get(childPos)) + { + this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); + this.repo.setApplyToParent(childPos, false); + } } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 46f971f16..003e67270 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -92,8 +92,12 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler { + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public DhSectionPos pos; public int levelMinY; @@ -113,7 +120,10 @@ public class FullDataSourceV2DTO implements IBaseDTO //========================// public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException - { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), levelWrapper); } + { + FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false); + return this.populateDataSource(dataSource, levelWrapper); + } public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } @@ -133,7 +143,7 @@ public class FullDataSourceV2DTO implements IBaseDTO } dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); - dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, this.compressionModeEnum); dataSource.mapping.clear(dataSource.getSectionPos()); // should only be null when used in a unit test @@ -163,7 +173,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // (de)serializing // //=================// - private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static CheckedByteArray writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -179,10 +189,10 @@ public class FullDataSourceV2DTO implements IBaseDTO int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; for (int xz = 0; xz < dataArrayLength; xz++) { - long[] dataColumn = dataArray[xz]; + LongArrayList dataColumn = dataArray[xz]; // write column length - short columnLength = (dataColumn != null) ? (short) dataColumn.length : 0; + short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; // a short is used instead of an int because at most we store 4096 vertical slices and a // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) compressedOut.writeShort(columnLength); @@ -190,7 +200,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) { - compressedOut.writeLong(dataColumn[y]); + compressedOut.writeLong(dataColumn.getLong(y)); } } @@ -202,33 +212,55 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static long[][] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, LongArrayList[] existingDataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); + int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; + if (existingDataArray == null + || existingDataArray.length != dataArrayLength) + { + existingDataArray = new LongArrayList[dataArrayLength]; + } + + // read the data - int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - long[][] dataArray = new long[dataArrayLength][]; - for (int xz = 0; xz < dataArray.length; xz++) + for (int xz = 0; xz < existingDataArray.length; xz++) { // read the column length - short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later - long[] dataColumn = new long[dataColumnLength]; + short dataColumnLength = compressedIn.readShort(); + + + // use the existing array if possible + LongArrayList dataColumn = existingDataArray[xz]; + if (dataColumn == null) + { + dataColumn = new LongArrayList(new long[dataColumnLength]); + existingDataArray[xz] = dataColumn; + } + + dataColumn.clear(); + dataColumn.ensureCapacity(dataColumnLength); + while (dataColumn.size() < dataColumnLength) + { + dataColumn.add(0); + } + // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) { long dataPoint = compressedIn.readLong(); - dataColumn[y] = dataPoint; + dataColumn.set(y, dataPoint); } - dataArray[xz] = dataColumn; + existingDataArray[xz] = dataColumn; } - return dataArray; + return existingDataArray; } From f3d8a749fdc5eed09ef843f048715ceb330e6646 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 18:12:37 -0500 Subject: [PATCH 081/183] Improve DhLodPos.getWidthAtDetail assertion --- .../java/com/seibel/distanthorizons/core/pos/DhLodPos.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java index cbc08fb64..2709fc6de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhLodPos.java @@ -72,7 +72,11 @@ public class DhLodPos implements Comparable // Get the width of this pos, measured in the target detail level. public int getWidthAtDetail(byte targetLevel) { - LodUtil.assertTrue(targetLevel <= this.detailLevel); + if (targetLevel > this.detailLevel) + { + LodUtil.assertNotReach("getWidthAtDetail for pos "+this+", given target detail level of bounds: ["+targetLevel+"], this: ["+this.detailLevel+"]"); + } + return BitShiftUtil.powerOfTwo(this.detailLevel - targetLevel); } From 7726335413f0f6d264a45185beb3ee161f30f52f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 19:59:12 -0500 Subject: [PATCH 082/183] Fix FullDataSourceV2Repo.getColumnGenerationStepForPos ignoring compression --- .../core/sql/repo/FullDataSourceV2Repo.java | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index 20ecc7a92..cea06e2bd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -20,9 +20,15 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; +import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; +import org.apache.logging.log4j.Logger; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; @@ -31,6 +37,9 @@ import java.util.Map; public class FullDataSourceV2Repo extends AbstractDhRepo { + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public FullDataSourceV2Repo(String databaseType, String databaseLocation) throws SQLException { super(databaseType, databaseLocation, FullDataSourceV2DTO.class); @@ -221,13 +230,33 @@ public class FullDataSourceV2Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( - "select ColumnGenerationStep " + + "select ColumnGenerationStep, CompressionMode " + "from "+this.getTableName()+" " + "WHERE DetailLevel = "+detailLevel+" AND PosX = "+pos.getX()+" AND PosZ = "+pos.getZ()); if (resultMap != null) { - return (byte[]) resultMap.get("ColumnGenerationStep"); + byte[] compressedByteArray = (byte[]) resultMap.get("ColumnGenerationStep"); + + byte compressionModeEnumValue = (byte) resultMap.get("CompressionMode"); + EDhApiDataCompressionMode compressionModeEnum = EDhApiDataCompressionMode.getFromValue(compressionModeEnumValue); + + try + { + // decompress the data + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); + + byte[] columnGenStepByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + compressedIn.readFully(columnGenStepByteArray); + + return columnGenStepByteArray; + } + catch (IOException e) + { + LOGGER.warn("Decompression issue when getting column gen steps for pos: "+pos, e); + return null; + } } else { From 6f7c46e0863c3774b99bae6d76eb8ad02db3d580 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 20:09:10 -0500 Subject: [PATCH 083/183] Remove unnecessary async render source updates --- .../core/file/renderfile/IRenderSourceProvider.java | 3 +-- .../core/file/renderfile/RenderSourceFileHandler.java | 3 +-- .../distanthorizons/core/level/ClientLevelModule.java | 6 ++---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java index 8db7634b8..ad044c2e4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core.file.renderfile; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; @@ -38,7 +37,7 @@ public interface IRenderSourceProvider extends AutoCloseable { CompletableFuture getAsync(DhSectionPos pos); - CompletableFuture updateDataSourceAsync(FullDataSourceV2 dataSource); + void updateDataSource(FullDataSourceV2 dataSource); /** Deletes any data stored in the render cache so it can be re-created */ void deleteRenderCache(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 003e67270..d0c18963a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -112,11 +112,10 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler updateDataSourceAsync(FullDataSourceV2 inputDataSource) + public void updateDataSource(FullDataSourceV2 inputDataSource) { // TODO once the legacy data provider has been replaced this can be removed this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); - return CompletableFuture.completedFuture(null); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 47ea5d305..7b88e473c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -216,10 +216,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { - ClientRenderState.renderSourceFileHandler - .updateDataSourceAsync(updatedFullDataSource) - // wait for the update to finish before triggering a reload to prevent holes in the world - .thenRun(() -> ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos())); + ClientRenderState.renderSourceFileHandler.updateDataSource(updatedFullDataSource); + ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos()); } } From 3ec9bfca1b66abd4fd03e397cf4159c7cd67abf1 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 20:11:14 -0500 Subject: [PATCH 084/183] comment out potenially unused world gen logic --- .../core/level/DhClientServerLevel.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 6510f3a89..1728d7825 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -105,14 +105,16 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev // create a new queue this.serverside.worldGenModule.startWorldGen(this.serverside.dataFileHandler, new ServerLevelModule.WorldGenState(this)); + // TODO I think this used to queue the world gen + // is it still needed? // populate the queue based on the current rendering tree - ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get(); - Iterator> iterator = renderState.quadtree.leafNodeIterator(); - while (iterator.hasNext()) - { - QuadNode node = iterator.next(); - this.serverside.dataFileHandler.getAsync(node.sectionPos); - } + //ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get(); + //Iterator> iterator = renderState.quadtree.leafNodeIterator(); + //while (iterator.hasNext()) + //{ + // QuadNode node = iterator.next(); + // //this.serverside.dataFileHandler.getAsync(node.sectionPos); + //} } else if (!shouldDoWorldGen && isWorldGenRunning) { From 4b98882b418b0d33bedf64c47d2087044c6ef9de Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 20:12:47 -0500 Subject: [PATCH 085/183] Revert "Add FullDataSourceV2 pooling and replace long[][] arrays with LongArrayList[]" This reverts commit f11e9a142f3b891aeee67c4c8be858f5ca25f1dd. --- .../methods/data/DhApiTerrainDataRepo.java | 7 +- .../fullData/sources/FullDataSourceV1.java | 10 - .../fullData/sources/FullDataSourceV2.java | 212 ++++++++---------- .../render/ColumnRenderSource.java | 7 +- .../FullDataToRenderDataTransformer.java | 15 +- .../transformers/LodDataBuilder.java | 11 +- .../file/AbstractNewDataSourceHandler.java | 27 ++- .../core/file/IDataSource.java | 6 +- .../FullDataSourceProviderV2.java | 12 +- .../renderfile/RenderSourceFileHandler.java | 8 +- .../SubDimensionLevelMatcher.java | 13 +- .../core/sql/dto/FullDataSourceV2DTO.java | 62 ++--- 12 files changed, 151 insertions(+), 239 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index d922064f4..933b29210 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -46,7 +46,6 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.coreapi.util.math.Vec3d; import com.seibel.distanthorizons.coreapi.util.math.Vec3f; import com.seibel.distanthorizons.coreapi.util.math.Vec3i; -import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -221,10 +220,10 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.mapping; - LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z); + long[] dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { - int dataColumnIndexCount = dataColumn.size(); + int dataColumnIndexCount = dataColumn.length; DhApiTerrainDataPoint[] returnArray = new DhApiTerrainDataPoint[dataColumnIndexCount]; long dataPoint; @@ -235,7 +234,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // search for a datapoint that contains the block y position for (int i = 0; i < dataColumnIndexCount; i++) { - dataPoint = dataColumn.getLong(i); + dataPoint = dataColumn[i]; if (!getSpecificYCoordinate) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 0558dfe4f..22b9db0c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -387,16 +387,6 @@ public class FullDataSourceV1 implements IDataSource - //==================// - // override methods // - //==================// - - @Override - public void close() throws Exception - { /* not currently needed */ } - - - //================// // helper classes // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 54d2fff2d..9994c39be 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -35,7 +35,6 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; -import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -52,7 +51,7 @@ import java.util.concurrent.locks.ReentrantLock; * @see FullDataPointUtilV2 * @see FullDataSourceV1 */ -public class FullDataSourceV2 implements IDataSource, AutoCloseable +public class FullDataSourceV2 implements IDataSource { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ @@ -99,7 +98,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately * there's something in the render data logic that expects this order so we can't change it right now */ - public LongArrayList[] dataPoints; + public long[][] dataPoints; public boolean isEmpty; public boolean applyToParent = false; @@ -114,7 +113,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable private FullDataSourceV2(DhSectionPos pos) { this.pos = pos; - this.dataPoints = new LongArrayList[WIDTH * WIDTH]; + this.dataPoints = new long[WIDTH * WIDTH][]; this.mapping = new FullDataPointIdMap(pos); this.isEmpty = true; @@ -123,8 +122,8 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable this.columnGenerationSteps = new byte[WIDTH * WIDTH]; } - public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } - private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -151,7 +150,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; - LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH]; + long[][] dataPoints = new long[WIDTH * WIDTH][]; for (int x = 0; x < WIDTH; x++) { for (int z = 0; z < WIDTH; z++) @@ -160,7 +159,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable if (dataColumn != null && dataColumn.length != 0) { int index = relativePosToIndex(x, z); - dataPoints[index] = new LongArrayList(dataColumn); + dataPoints[index] = dataColumn; // convert the data point format @@ -213,13 +212,13 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable long[] legacyDataColumn = legacyData.get(x, z); if (legacyDataColumn != null && legacyDataColumn.length != 0) { - LongArrayList newDataColumn = fullDataSource.get(x, z); + long[] newDataColumn = fullDataSource.get(x, z); if (newDataColumn == null) { LodUtil.assertNotReach("Accessor column mismatch"); } - else if (legacyDataColumn.length != newDataColumn.size()) + else if (legacyDataColumn.length != newDataColumn.length) { LodUtil.assertNotReach("Accessor column length mismatch"); } @@ -227,7 +226,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable { for (int i = 0; i < legacyDataColumn.length; i++) { - if (legacyDataColumn[i] != newDataColumn.getLong(i)) + if (legacyDataColumn[i] != newDataColumn[i]) { LodUtil.assertNotReach("Data mismatch"); } @@ -248,7 +247,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // data // //======// - public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } + public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } @@ -310,7 +309,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable { int index = relativePosToIndex(x, z); - LongArrayList newDataArray = inputDataSource.dataPoints[index]; + long[] newDataArray = inputDataSource.dataPoints[index]; if (newDataArray != null) { byte thisGenState = this.columnGenerationSteps[index]; @@ -319,15 +318,11 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value && thisGenState <= inputGenState) { - LongArrayList oldDataArray = this.dataPoints[index]; + long[] oldDataArray = this.dataPoints[index]; // copy over the new data - if (this.dataPoints[index] == null) - { - this.dataPoints[index] = new LongArrayList(new long[newDataArray.size()]); - } - this.dataPoints[index].clear(); - this.dataPoints[index].addAll(newDataArray); + this.dataPoints[index] = new long[newDataArray.length]; + System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); this.remapDataColumn(index, remappedIds); if (RUN_DATA_ORDER_VALIDATION) @@ -388,8 +383,8 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable this.columnGenerationSteps[recipientIndex] = inputGenStep; // data points - LongArrayList oldDataArray = this.dataPoints[recipientIndex]; - LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); + long[] oldDataArray = this.dataPoints[recipientIndex]; + long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); this.dataPoints[recipientIndex] = mergedInputDataArray; if (RUN_DATA_ORDER_VALIDATION) @@ -432,13 +427,9 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable } return minWorldGenStepValue; } - - private static ThreadLocal mergeTwoByTwoTempList = ThreadLocal.withInitial(() -> new LongArrayList(20)); - private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) + private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { - LongArrayList newColumnList = mergeTwoByTwoTempList.get(); - newColumnList.clear(); - + ArrayList newColumnList = new ArrayList<>(); // special numbers: // -2 = the column's height hasn't been determined yet @@ -475,8 +466,8 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) { // TODO throw an assertion if the column isn't in top-down order or just fix it... - LongArrayList inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; - if (inputDataArray == null || inputDataArray.size() == 0) + long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; + if (inputDataArray == null || inputDataArray.length == 0) { currentDatapointIndex[colIndex] = -1; continue; @@ -485,7 +476,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // determine the last index (the lowest data point) for each column if (currentDatapointIndex[colIndex] == -2) { - currentDatapointIndex[colIndex] = inputDataArray.size() - 1; + currentDatapointIndex[colIndex] = inputDataArray.length - 1; if (RUN_DATA_ORDER_VALIDATION) { @@ -500,7 +491,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // went over the end continue; } - long datapoint = inputDataArray.getLong(dataPointIndex); + long datapoint = inputDataArray[dataPointIndex]; int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint); int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint); @@ -584,10 +575,10 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // convert the arraylist to an array - LongArrayList mergedInputDataArray = new LongArrayList(new long[newColumnList.size()]); - for (int i = 0; i < mergedInputDataArray.size(); i++) + long[]mergedInputDataArray = new long[newColumnList.size()]; + for (int i = 0; i < mergedInputDataArray.length; i++) { - mergedInputDataArray.set(i, newColumnList.getLong(i)); + mergedInputDataArray[i] = newColumnList.get(i); } @@ -595,10 +586,10 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray // to sometimes be in a different order? Is it potentially related to what detail level is coming in? { - long firstDataPoint = mergedInputDataArray.getLong(0); + long firstDataPoint = mergedInputDataArray[0]; int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = mergedInputDataArray.getLong(mergedInputDataArray.size() - 1); + long lastDataPoint = mergedInputDataArray[mergedInputDataArray.length - 1]; int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -606,11 +597,11 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // reverse the array so index 0 is the highest, // this is necessary for later logic // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < mergedInputDataArray.size() / 2; i++) + for(int i = 0; i < mergedInputDataArray.length / 2; i++) { - long temp = mergedInputDataArray.getLong(i); - mergedInputDataArray.set(i, mergedInputDataArray.getLong(mergedInputDataArray.size() - i - 1)); - mergedInputDataArray.set(mergedInputDataArray.size() - i - 1, temp); + long temp = mergedInputDataArray[i]; + mergedInputDataArray[i] = mergedInputDataArray[mergedInputDataArray.length - i - 1]; + mergedInputDataArray[mergedInputDataArray.length - i - 1] = temp; } } } @@ -624,15 +615,15 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable */ private void remapDataColumn(int dataPointIndex, int[] remappedIds) { - LongArrayList dataColumn = this.dataPoints[dataPointIndex]; - for (int i = 0; i < dataColumn.size(); i++) + long[] dataColumn = this.dataPoints[dataPointIndex]; + for (int i = 0; i < dataColumn.length; i++) { - dataColumn.set(i, FullDataPointUtilV2.remap(remappedIds, dataColumn.getLong(i))); + dataColumn[i] = FullDataPointUtilV2.remap(remappedIds, dataColumn[i]); } } - private static boolean areDataColumnsDifferent(LongArrayList oldDataArray, LongArrayList newDataArray) + private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) { - if (oldDataArray == null || oldDataArray.size() != newDataArray.size()) + if (oldDataArray == null || oldDataArray.length != newDataArray.length) { // new data was added/removed return true; @@ -640,8 +631,8 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable else { // check if the new column data is different - int oldArrayHash = oldDataArray.hashCode(); - int newArrayHash = newDataArray.hashCode(); + int oldArrayHash = Arrays.hashCode(oldDataArray); + int newArrayHash = Arrays.hashCode(newDataArray); return (newArrayHash != oldArrayHash); } } @@ -741,12 +732,12 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable * * @see FullDataSourceV2#dataPoints */ - public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, LongArrayList dataArray) throws IllegalStateException + public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, long[] dataArray) throws IllegalStateException { - long firstDataPoint = dataArray.getLong(0); + long firstDataPoint = dataArray[0]; int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = dataArray.getLong(dataArray.size() - 1); + long lastDataPoint = dataArray[dataArray.length - 1]; int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -776,7 +767,7 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; @@ -787,9 +778,9 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable { // validate the incoming ID's int maxValidId = this.mapping.getMaxValidId(); - for (int i = 0; i < longArray.size(); i++) + for (int i = 0; i < longArray.length; i++) { - long dataPoint = longArray.getLong(i); + long dataPoint = longArray[i]; int id = FullDataPointUtilV2.getId(dataPoint); if (id > maxValidId) { @@ -867,83 +858,64 @@ public class FullDataSourceV2 implements IDataSource, AutoCloseable // pooling // //=========// - @Override - public void close() throws Exception + // TODO add pooled data sources + private static class Pooling { - returnPooledDataSource(this); - } - - - /** used when pooling data sources */ - private static final ArrayList CACHED_SOURCES = new ArrayList<>(); - private static final ReentrantLock CACHE_LOCK = new ReentrantLock(); - - - /** @return an empty data source if non are cached */ - public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData) - { - try + /** used when pooling data sources */ + private final ArrayList cachedSources = new ArrayList<>(); + private final ReentrantLock cacheLock = new ReentrantLock(); + + + /** @return null if no pooled source exists */ + public FullDataSourceV1 tryGetPooledSource() { - CACHE_LOCK.lock(); - - int index = CACHED_SOURCES.size() - 1; - if (index == -1) + try { - return createEmpty(pos); - } - else - { - FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index);; - dataSource.pos = pos; + this.cacheLock.lock(); - if (clearData) + int index = this.cachedSources.size() - 1; + if (index == -1) { - dataSource.mapping.clear(pos); - - for (int i = 0; i < dataSource.dataPoints.length; i++) - { - if (dataSource.dataPoints[i] != null) - { - dataSource.dataPoints[i].clear(); - } - } - - Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + return null; + } + else + { + return this.cachedSources.remove(index); } - - return dataSource; } - } - finally - { - CACHE_LOCK.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called. - */ - public static void returnPooledDataSource(FullDataSourceV2 dataSource) - { - if (dataSource == null) - { - return; - } - else if (CACHED_SOURCES.size() > 25) - { - return; + finally + { + this.cacheLock.unlock(); + } } - try + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. + */ + public void returnPooledDataSource(FullDataSourceV1 dataSource) { - CACHE_LOCK.lock(); - CACHED_SOURCES.add(dataSource); - } - finally - { - CACHE_LOCK.unlock(); + if (dataSource == null) + { + return; + } + else if (this.cachedSources.size() > 25) + { + return; + } + + try + { + this.cacheLock.lock(); + this.cachedSources.add(dataSource); + } + finally + { + this.cacheLock.unlock(); + } } + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index fd2b8a011..208aa4df5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -37,7 +37,6 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; -import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.io.*; @@ -318,7 +317,7 @@ public class ColumnRenderSource implements IDataSource ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); - LongArrayList dataColumn = inputFullDataSource.get(x, z); + long[] dataColumn = inputFullDataSource.get(x, z); EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z); if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { @@ -477,10 +476,6 @@ public class ColumnRenderSource implements IDataSource return stringBuilder.toString(); } - @Override - public void close() throws Exception - { /* not currently needed */ } - //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 28d7dd301..b4f12126c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -36,7 +36,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; -import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.util.HashSet; @@ -118,7 +117,7 @@ public class FullDataToRenderDataTransformer throwIfThreadInterrupted(); ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - LongArrayList dataColumn = fullDataSource.get(x, z); + long[] dataColumn = fullDataSource.get(x, z); convertColumnData(level, fullDataSource.mapping, baseX + x, baseZ + z, columnArrayView, dataColumn); } } @@ -160,7 +159,7 @@ public class FullDataToRenderDataTransformer private static void iterateAndConvert( IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, - ColumnArrayView renderColumnData, LongArrayList fullColumnData) + ColumnArrayView renderColumnData, long[] fullColumnData) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); @@ -172,9 +171,9 @@ public class FullDataToRenderDataTransformer int columnOffset = 0; // goes from the top down - for (int i = 0; i < fullColumnData.size(); i++) + for (int i = 0; i < fullColumnData.length; i++) { - long fullData = fullColumnData.getLong(i); + long fullData = fullColumnData[i]; int bottomY = FullDataPointUtilV2.getBottomY(fullData); int blockHeight = FullDataPointUtilV2.getHeight(fullData); int id = FullDataPointUtilV2.getId(fullData); @@ -271,14 +270,14 @@ public class FullDataToRenderDataTransformer } // TODO what does this mean? - public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, LongArrayList fullDataColumn) + public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, long[] fullDataColumn) { - if (fullDataColumn == null || fullDataColumn.size() == 0) + if (fullDataColumn == null || fullDataColumn.length == 0) { return; } - int dataTotalLength = fullDataColumn.size(); + int dataTotalLength = fullDataColumn.length; if (dataTotalLength > columnArrayView.verticalSize()) { ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index bcf79b56b..2a3346f03 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -131,7 +131,7 @@ public class LodDataBuilder { for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++) { - LongArrayList longs = new LongArrayList(new long[chunkWrapper.getHeight() / 4]); + LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); int lastY = chunkWrapper.getMaxBuildHeight(); IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); IBlockStateWrapper blockState = AIR; @@ -201,7 +201,7 @@ public class LodDataBuilder } longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - dataSource.setSingleColumn(longs, + dataSource.setSingleColumn(longs.toLongArray(), chunkX + chunkOffsetX, chunkZ + chunkOffsetZ, EDhApiWorldGenerationStep.LIGHT); @@ -230,7 +230,7 @@ public class LodDataBuilder // AND the below loop won't run. int size = (columnDataPoints != null) ? columnDataPoints.size() : 0; - LongArrayList packedDataPoints = new LongArrayList(new long[size]); + long[] packedDataPoints = new long[size]; for (int index = 0; index < size; index++) { DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index); @@ -240,14 +240,13 @@ public class LodDataBuilder (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); - packedDataPoints.set(index, - FullDataPointUtilV2.encode( + packedDataPoints[index] = FullDataPointUtilV2.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, (byte) (dataPoint.blockLightLevel), (byte) (dataPoint.skyLightLevel) - )); + ); } accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index f0489188c..3d804d013 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -221,23 +221,22 @@ public abstract class AbstractNewDataSourceHandler } - // try block allows for disposing of pooled data sources after the update is complete - try (TDataSource recipientDataSource = this.get(updatePos)) + // get or create the data source + TDataSource recipientDataSource = this.get(updatePos); + boolean dataModified = recipientDataSource.update(inputData, this.level); + + if (dataModified) { - boolean dataModified = recipientDataSource.update(inputData, this.level); - if (dataModified) + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(recipientDataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(recipientDataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + if (listener != null) { - if (listener != null) - { - listener.OnDataSourceUpdated(recipientDataSource); - } + listener.OnDataSourceUpdated(recipientDataSource); } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 0da2c34b6..11738fe7a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -11,13 +11,11 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import java.io.IOException; /** - * Base for all data sources.

- * - * AutoCloseable Can be implemented to allow for disposing of pooled data sources.

+ * Base for all data sources. * * @param there are times when we need specifically a client level vs a more generic level */ -public interface IDataSource extends IBaseDTO, AutoCloseable +public interface IDataSource extends IBaseDTO { DhSectionPos getSectionPos(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 0e1b9f34d..70dd8bf1e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -162,11 +162,11 @@ public class FullDataSourceProviderV2 { // TODO maybe just set children update flags to true? // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway - return FullDataSourceV2.getPooledSource(pos, true); + return FullDataSourceV2.createEmpty(pos); } @Override - protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); } + protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); } @@ -248,11 +248,9 @@ public class FullDataSourceProviderV2 childReadLock.lock(); this.lockedPosSet.add(childPos); - try (FullDataSourceV2 dataSource = this.get(childPos)) - { - this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); - this.repo.setApplyToParent(childPos, false); - } + FullDataSourceV2 dataSource = this.get(childPos); + this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); + this.repo.setApplyToParent(childPos, false); } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index d0c18963a..2b3fba7b9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -92,12 +92,8 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler { - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - public DhSectionPos pos; public int levelMinY; @@ -120,10 +113,7 @@ public class FullDataSourceV2DTO implements IBaseDTO //========================// public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException - { - FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false); - return this.populateDataSource(dataSource, levelWrapper); - } + { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), levelWrapper); } public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } @@ -143,7 +133,7 @@ public class FullDataSourceV2DTO implements IBaseDTO } dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); - dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, dataSource.dataPoints, this.compressionModeEnum); + dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); dataSource.mapping.clear(dataSource.getSectionPos()); // should only be null when used in a unit test @@ -173,7 +163,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // (de)serializing // //=================// - private static CheckedByteArray writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -189,10 +179,10 @@ public class FullDataSourceV2DTO implements IBaseDTO int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; for (int xz = 0; xz < dataArrayLength; xz++) { - LongArrayList dataColumn = dataArray[xz]; + long[] dataColumn = dataArray[xz]; // write column length - short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; + short columnLength = (dataColumn != null) ? (short) dataColumn.length : 0; // a short is used instead of an int because at most we store 4096 vertical slices and a // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) compressedOut.writeShort(columnLength); @@ -200,7 +190,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) { - compressedOut.writeLong(dataColumn.getLong(y)); + compressedOut.writeLong(dataColumn[y]); } } @@ -212,55 +202,33 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, LongArrayList[] existingDataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static long[][] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); - int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - if (existingDataArray == null - || existingDataArray.length != dataArrayLength) - { - existingDataArray = new LongArrayList[dataArrayLength]; - } - - // read the data - for (int xz = 0; xz < existingDataArray.length; xz++) + int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; + long[][] dataArray = new long[dataArrayLength][]; + for (int xz = 0; xz < dataArray.length; xz++) { // read the column length - short dataColumnLength = compressedIn.readShort(); - - - // use the existing array if possible - LongArrayList dataColumn = existingDataArray[xz]; - if (dataColumn == null) - { - dataColumn = new LongArrayList(new long[dataColumnLength]); - existingDataArray[xz] = dataColumn; - } - - dataColumn.clear(); - dataColumn.ensureCapacity(dataColumnLength); - while (dataColumn.size() < dataColumnLength) - { - dataColumn.add(0); - } - + short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later + long[] dataColumn = new long[dataColumnLength]; // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) { long dataPoint = compressedIn.readLong(); - dataColumn.set(y, dataPoint); + dataColumn[y] = dataPoint; } - existingDataArray[xz] = dataColumn; + dataArray[xz] = dataColumn; } - return existingDataArray; + return dataArray; } From 18ad241025db12f7b0e4ba524883def9b1a479da Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 21:53:15 -0500 Subject: [PATCH 086/183] Replace datasource v2 long[][] with LongArrayList[] and fix previous implementation --- .../methods/data/DhApiTerrainDataRepo.java | 7 +- .../fullData/sources/FullDataSourceV2.java | 159 +++++++++++------- .../render/ColumnRenderSource.java | 3 +- .../FullDataToRenderDataTransformer.java | 15 +- .../transformers/LodDataBuilder.java | 8 +- .../SubDimensionLevelMatcher.java | 13 +- .../core/sql/dto/FullDataSourceV2DTO.java | 17 +- 7 files changed, 136 insertions(+), 86 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index 933b29210..d922064f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -46,6 +46,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.coreapi.util.math.Vec3d; import com.seibel.distanthorizons.coreapi.util.math.Vec3f; import com.seibel.distanthorizons.coreapi.util.math.Vec3i; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -220,10 +221,10 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo { // attempt to get the LOD data from the data source FullDataPointIdMap mapping = dataSource.mapping; - long[] dataColumn = dataSource.get(relativePos.x, relativePos.z); + LongArrayList dataColumn = dataSource.get(relativePos.x, relativePos.z); if (dataColumn != null) { - int dataColumnIndexCount = dataColumn.length; + int dataColumnIndexCount = dataColumn.size(); DhApiTerrainDataPoint[] returnArray = new DhApiTerrainDataPoint[dataColumnIndexCount]; long dataPoint; @@ -234,7 +235,7 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo // search for a datapoint that contains the block y position for (int i = 0; i < dataColumnIndexCount; i++) { - dataPoint = dataColumn[i]; + dataPoint = dataColumn.getLong(i); if (!getSpecificYCoordinate) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 9994c39be..10a1837fe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -98,7 +99,7 @@ public class FullDataSourceV2 implements IDataSource * TODO that ordering feels weird, it'd be nice to reverse that order, unfortunately * there's something in the render data logic that expects this order so we can't change it right now */ - public long[][] dataPoints; + public LongArrayList[] dataPoints; public boolean isEmpty; public boolean applyToParent = false; @@ -113,7 +114,7 @@ public class FullDataSourceV2 implements IDataSource private FullDataSourceV2(DhSectionPos pos) { this.pos = pos; - this.dataPoints = new long[WIDTH * WIDTH][]; + this.dataPoints = new LongArrayList[WIDTH * WIDTH]; this.mapping = new FullDataPointIdMap(pos); this.isEmpty = true; @@ -122,8 +123,8 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps = new byte[WIDTH * WIDTH]; } - public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } - private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, long[][] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -150,7 +151,7 @@ public class FullDataSourceV2 implements IDataSource // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; - long[][] dataPoints = new long[WIDTH * WIDTH][]; + LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH]; for (int x = 0; x < WIDTH; x++) { for (int z = 0; z < WIDTH; z++) @@ -159,7 +160,7 @@ public class FullDataSourceV2 implements IDataSource if (dataColumn != null && dataColumn.length != 0) { int index = relativePosToIndex(x, z); - dataPoints[index] = dataColumn; + dataPoints[index] = new LongArrayList(dataColumn); // convert the data point format @@ -212,13 +213,12 @@ public class FullDataSourceV2 implements IDataSource long[] legacyDataColumn = legacyData.get(x, z); if (legacyDataColumn != null && legacyDataColumn.length != 0) { - long[] newDataColumn = fullDataSource.get(x, z); - + LongArrayList newDataColumn = fullDataSource.get(x, z); if (newDataColumn == null) { LodUtil.assertNotReach("Accessor column mismatch"); } - else if (legacyDataColumn.length != newDataColumn.length) + else if (legacyDataColumn.length != newDataColumn.size()) { LodUtil.assertNotReach("Accessor column length mismatch"); } @@ -226,7 +226,7 @@ public class FullDataSourceV2 implements IDataSource { for (int i = 0; i < legacyDataColumn.length; i++) { - if (legacyDataColumn[i] != newDataColumn[i]) + if (legacyDataColumn[i] != newDataColumn.getLong(i)) { LodUtil.assertNotReach("Data mismatch"); } @@ -247,7 +247,7 @@ public class FullDataSourceV2 implements IDataSource // data // //======// - public long[] get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } + public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } @@ -309,8 +309,8 @@ public class FullDataSourceV2 implements IDataSource { int index = relativePosToIndex(x, z); - long[] newDataArray = inputDataSource.dataPoints[index]; - if (newDataArray != null) + LongArrayList inputDataArray = inputDataSource.dataPoints[index]; + if (inputDataArray != null) { byte thisGenState = this.columnGenerationSteps[index]; byte inputGenState = inputDataSource.columnGenerationSteps[index]; @@ -318,11 +318,31 @@ public class FullDataSourceV2 implements IDataSource if (inputGenState != EDhApiWorldGenerationStep.EMPTY.value && thisGenState <= inputGenState) { - long[] oldDataArray = this.dataPoints[index]; + // check if the data changed + if (this.dataPoints[index] == null) + { + // no data was present previously + this.dataPoints[index] = new LongArrayList(inputDataArray); + dataChanged = true; + } + else if (this.dataPoints[index].size() != inputDataArray.size()) + { + // data is present, but the size is different + dataChanged = true; + } + + int oldDataHash = 0; + if (!dataChanged) + { + // some old data existed with the same length, + // we'll have to compare the caches + oldDataHash = this.dataPoints[index].hashCode(); + } + // copy over the new data - this.dataPoints[index] = new long[newDataArray.length]; - System.arraycopy(newDataArray, 0, this.dataPoints[index], 0, newDataArray.length); + this.dataPoints[index].clear(); + this.dataPoints[index].addAll(inputDataArray); this.remapDataColumn(index, remappedIds); if (RUN_DATA_ORDER_VALIDATION) @@ -330,11 +350,17 @@ public class FullDataSourceV2 implements IDataSource throwIfDataColumnInWrongOrder(inputDataSource.pos, this.dataPoints[index]); } - // we only need to see if the data was changed in one column + + if (!dataChanged) { - // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same - dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[index]); + // check if the identical length data column hashes are the same + // hashes need to be compared after the ID's have been remapped otherwise the ID's won't match even if the data is the same + if (oldDataHash != this.dataPoints[index].hashCode()) + { + // the hashes are different, something was changed + dataChanged = true; + } } this.columnGenerationSteps[index] = inputGenState; @@ -378,29 +404,55 @@ public class FullDataSourceV2 implements IDataSource int recipientIndex = relativePosToIndex(recipientX, recipientZ); - // world gen + // world gen // byte inputGenStep = determineMinWorldGenStepForTwoByTwoColumn(inputDataSource.columnGenerationSteps, x, z); this.columnGenerationSteps[recipientIndex] = inputGenStep; - // data points - long[] oldDataArray = this.dataPoints[recipientIndex]; - long[] mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); + + + // data points // + LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); + + // check if the data changed + if (this.dataPoints[recipientIndex] == null) + { + // no data was present previously + dataChanged = true; + } + else if (this.dataPoints[recipientIndex].size() != mergedInputDataArray.size()) + { + // data is present, but the size is different + dataChanged = true; + } + + int oldDataHash = 0; + if (!dataChanged) + { + // some old data existed with the same length, + // we'll have to compare the caches + oldDataHash = this.dataPoints[recipientIndex].hashCode(); + } + + this.dataPoints[recipientIndex] = mergedInputDataArray; + this.remapDataColumn(recipientIndex, remappedIds); if (RUN_DATA_ORDER_VALIDATION) { throwIfDataColumnInWrongOrder(inputDataSource.pos, this.dataPoints[recipientIndex]); } - // mapping - this.remapDataColumn(recipientIndex, remappedIds); - // we only need to see if the data was changed in one column if (!dataChanged) { - // needs to be done after the ID's have been remapped otherwise the ID's won't match even if the data is the same - dataChanged = areDataColumnsDifferent(oldDataArray, this.dataPoints[recipientIndex]); + // check if the identical length data column hashes are the same + // hashes need to be compared after the ID's have been remapped otherwise the ID's won't match even if the data is the same + if (oldDataHash != this.dataPoints[recipientIndex].hashCode()) + { + // the hashes are different, something was changed + dataChanged = true; + } } this.isEmpty = false; @@ -427,9 +479,9 @@ public class FullDataSourceV2 implements IDataSource } return minWorldGenStepValue; } - private static long[] mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) + private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { - ArrayList newColumnList = new ArrayList<>(); + LongArrayList newColumnList = new LongArrayList(); // special numbers: // -2 = the column's height hasn't been determined yet @@ -466,8 +518,8 @@ public class FullDataSourceV2 implements IDataSource for (int inputZ = z; inputZ < z + 2; inputZ++, colIndex++) { // TODO throw an assertion if the column isn't in top-down order or just fix it... - long[] inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; - if (inputDataArray == null || inputDataArray.length == 0) + LongArrayList inputDataArray = inputDataSource.dataPoints[relativePosToIndex(inputX, inputZ)]; + if (inputDataArray == null || inputDataArray.size() == 0) { currentDatapointIndex[colIndex] = -1; continue; @@ -476,7 +528,7 @@ public class FullDataSourceV2 implements IDataSource // determine the last index (the lowest data point) for each column if (currentDatapointIndex[colIndex] == -2) { - currentDatapointIndex[colIndex] = inputDataArray.length - 1; + currentDatapointIndex[colIndex] = inputDataArray.size() - 1; if (RUN_DATA_ORDER_VALIDATION) { @@ -491,7 +543,7 @@ public class FullDataSourceV2 implements IDataSource // went over the end continue; } - long datapoint = inputDataArray[dataPointIndex]; + long datapoint = inputDataArray.getLong(dataPointIndex); int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint); int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint); @@ -574,22 +626,15 @@ public class FullDataSourceV2 implements IDataSource } - // convert the arraylist to an array - long[]mergedInputDataArray = new long[newColumnList.size()]; - for (int i = 0; i < mergedInputDataArray.length; i++) - { - mergedInputDataArray[i] = newColumnList.get(i); - } - // flip the array if necessary // TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray // to sometimes be in a different order? Is it potentially related to what detail level is coming in? { - long firstDataPoint = mergedInputDataArray[0]; + long firstDataPoint = newColumnList.getLong(0); int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = mergedInputDataArray[mergedInputDataArray.length - 1]; + long lastDataPoint = newColumnList.getLong(newColumnList.size() - 1); int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -597,16 +642,16 @@ public class FullDataSourceV2 implements IDataSource // reverse the array so index 0 is the highest, // this is necessary for later logic // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < mergedInputDataArray.length / 2; i++) + for(int i = 0; i < newColumnList.size() / 2; i++) { - long temp = mergedInputDataArray[i]; - mergedInputDataArray[i] = mergedInputDataArray[mergedInputDataArray.length - i - 1]; - mergedInputDataArray[mergedInputDataArray.length - i - 1] = temp; + long temp = newColumnList.getLong(i); + newColumnList.set(i, newColumnList.getLong(newColumnList.size() - i - 1)); + newColumnList.set(newColumnList.size() - i - 1, temp); } } } - return mergedInputDataArray; + return newColumnList; } /** * Only update the ID once it's been added to this data source. @@ -615,10 +660,10 @@ public class FullDataSourceV2 implements IDataSource */ private void remapDataColumn(int dataPointIndex, int[] remappedIds) { - long[] dataColumn = this.dataPoints[dataPointIndex]; - for (int i = 0; i < dataColumn.length; i++) + LongArrayList dataColumn = this.dataPoints[dataPointIndex]; + for (int i = 0; i < dataColumn.size(); i++) { - dataColumn[i] = FullDataPointUtilV2.remap(remappedIds, dataColumn[i]); + dataColumn.set(i, FullDataPointUtilV2.remap(remappedIds, dataColumn.getLong(i))); } } private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) @@ -732,12 +777,12 @@ public class FullDataSourceV2 implements IDataSource * * @see FullDataSourceV2#dataPoints */ - public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, long[] dataArray) throws IllegalStateException + public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, LongArrayList dataArray) throws IllegalStateException { - long firstDataPoint = dataArray[0]; + long firstDataPoint = dataArray.getLong(0); int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - long lastDataPoint = dataArray[dataArray.length - 1]; + long lastDataPoint = dataArray.getLong(dataArray.size() - 1); int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) @@ -767,7 +812,7 @@ public class FullDataSourceV2 implements IDataSource return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public void setSingleColumn(long[] longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; @@ -778,9 +823,9 @@ public class FullDataSourceV2 implements IDataSource { // validate the incoming ID's int maxValidId = this.mapping.getMaxValidId(); - for (int i = 0; i < longArray.length; i++) + for (int i = 0; i < longArray.size(); i++) { - long dataPoint = longArray[i]; + long dataPoint = longArray.getLong(i); int id = FullDataPointUtilV2.getId(dataPoint); if (id > maxValidId) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 208aa4df5..b1282c73a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -37,6 +37,7 @@ import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.io.*; @@ -317,7 +318,7 @@ public class ColumnRenderSource implements IDataSource ColumnArrayView columnArrayView = this.getVerticalDataPointView(x, z); int columnHash = columnArrayView.getDataHash(); - long[] dataColumn = inputFullDataSource.get(x, z); + LongArrayList dataColumn = inputFullDataSource.get(x, z); EDhApiWorldGenerationStep worldGenStep = inputFullDataSource.getWorldGenStepAtRelativePos(x, z); if (dataColumn != null && worldGenStep != EDhApiWorldGenerationStep.EMPTY) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index b4f12126c..28d7dd301 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -36,6 +36,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import java.util.HashSet; @@ -117,7 +118,7 @@ public class FullDataToRenderDataTransformer throwIfThreadInterrupted(); ColumnArrayView columnArrayView = columnSource.getVerticalDataPointView(x, z); - long[] dataColumn = fullDataSource.get(x, z); + LongArrayList dataColumn = fullDataSource.get(x, z); convertColumnData(level, fullDataSource.mapping, baseX + x, baseZ + z, columnArrayView, dataColumn); } } @@ -159,7 +160,7 @@ public class FullDataToRenderDataTransformer private static void iterateAndConvert( IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, - ColumnArrayView renderColumnData, long[] fullColumnData) + ColumnArrayView renderColumnData, LongArrayList fullColumnData) { boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); @@ -171,9 +172,9 @@ public class FullDataToRenderDataTransformer int columnOffset = 0; // goes from the top down - for (int i = 0; i < fullColumnData.length; i++) + for (int i = 0; i < fullColumnData.size(); i++) { - long fullData = fullColumnData[i]; + long fullData = fullColumnData.getLong(i); int bottomY = FullDataPointUtilV2.getBottomY(fullData); int blockHeight = FullDataPointUtilV2.getHeight(fullData); int id = FullDataPointUtilV2.getId(fullData); @@ -270,14 +271,14 @@ public class FullDataToRenderDataTransformer } // TODO what does this mean? - public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, long[] fullDataColumn) + public static void convertColumnData(IDhClientLevel level, FullDataPointIdMap fullDataMapping, int blockX, int blockZ, ColumnArrayView columnArrayView, LongArrayList fullDataColumn) { - if (fullDataColumn == null || fullDataColumn.length == 0) + if (fullDataColumn == null || fullDataColumn.size() == 0) { return; } - int dataTotalLength = fullDataColumn.length; + int dataTotalLength = fullDataColumn.size(); if (dataTotalLength > columnArrayView.verticalSize()) { ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 2a3346f03..601281505 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -201,7 +201,7 @@ public class LodDataBuilder } longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - dataSource.setSingleColumn(longs.toLongArray(), + dataSource.setSingleColumn(longs, chunkX + chunkOffsetX, chunkZ + chunkOffsetZ, EDhApiWorldGenerationStep.LIGHT); @@ -230,7 +230,7 @@ public class LodDataBuilder // AND the below loop won't run. int size = (columnDataPoints != null) ? columnDataPoints.size() : 0; - long[] packedDataPoints = new long[size]; + LongArrayList packedDataPoints = new LongArrayList(new long[size]); for (int index = 0; index < size; index++) { DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index); @@ -240,13 +240,13 @@ public class LodDataBuilder (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); - packedDataPoints[index] = FullDataPointUtilV2.encode( + packedDataPoints.set(index, FullDataPointUtilV2.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, (byte) (dataPoint.blockLightLevel), (byte) (dataPoint.skyLightLevel) - ); + )); } accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 61e0e0ffa..d4ef7bd55 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -41,6 +41,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftCli import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.LogManager; import java.io.File; @@ -233,8 +234,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable { for (int z = 0; z < FullDataSourceV1.WIDTH; z++) { - long[] newColumn = newDataSource.get(x, z); - long[] testColumn = testFullDataSource.get(x, z); + LongArrayList newColumn = newDataSource.get(x, z); + LongArrayList testColumn = testFullDataSource.get(x, z); if (newColumn != null && testColumn != null) { @@ -244,11 +245,11 @@ public class SubDimensionLevelMatcher implements AutoCloseable FullDataPointIdMap testDataMap = testFullDataSource.mapping; // use min to prevent going out of bounds - int minColumnIndex = Math.min(newColumn.length, testColumn.length); + int minColumnIndex = Math.min(newColumn.size(), testColumn.size()); for (int i = 0; i < minColumnIndex; i++) { - long newDataPoint = newColumn[i]; - long testDataPoint = testColumn[i]; + long newDataPoint = newColumn.getLong(i); + long testDataPoint = testColumn.getLong(i); int newId = FullDataPointUtilV2.getId(newDataPoint); int testId = FullDataPointUtilV2.getId(testDataPoint); @@ -297,7 +298,7 @@ public class SubDimensionLevelMatcher implements AutoCloseable else if (newColumn != null) { // missing test column - totalDataPointCount += newColumn.length; + totalDataPointCount += newColumn.size(); } else { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 51249a69c..3cbcd96f4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import it.unimi.dsi.fastutil.longs.LongArrayList; import org.jetbrains.annotations.NotNull; import java.io.ByteArrayInputStream; @@ -163,7 +164,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // (de)serializing // //=================// - private static CheckedByteArray writeDataSourceDataArrayToBlob(long[][] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static CheckedByteArray writeDataSourceDataArrayToBlob(LongArrayList[] dataArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { // write the outputs to a stream to prep for writing to the database ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -179,10 +180,10 @@ public class FullDataSourceV2DTO implements IBaseDTO int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; for (int xz = 0; xz < dataArrayLength; xz++) { - long[] dataColumn = dataArray[xz]; + LongArrayList dataColumn = dataArray[xz]; // write column length - short columnLength = (dataColumn != null) ? (short) dataColumn.length : 0; + short columnLength = (dataColumn != null) ? (short) dataColumn.size() : 0; // a short is used instead of an int because at most we store 4096 vertical slices and a // short fits that with less wasted spaces vs an int (short has max value of 32,767 vs int's max of 2 billion) compressedOut.writeShort(columnLength); @@ -190,7 +191,7 @@ public class FullDataSourceV2DTO implements IBaseDTO // write column data (will be skipped if no data was present) for (int y = 0; y < columnLength; y++) { - compressedOut.writeLong(dataColumn[y]); + compressedOut.writeLong(dataColumn.getLong(y)); } } @@ -202,7 +203,7 @@ public class FullDataSourceV2DTO implements IBaseDTO return new CheckedByteArray(checksum, byteArrayOutputStream.toByteArray()); } - private static long[][] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + private static LongArrayList[] readBlobToDataSourceDataArray(byte[] compressedDataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedDataByteArray); DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); @@ -210,18 +211,18 @@ public class FullDataSourceV2DTO implements IBaseDTO // read the data int dataArrayLength = FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH; - long[][] dataArray = new long[dataArrayLength][]; + LongArrayList[] dataArray = new LongArrayList[dataArrayLength]; for (int xz = 0; xz < dataArray.length; xz++) { // read the column length short dataColumnLength = compressedIn.readShort(); // separate variables are used for debugging and in case validation wants to be added later - long[] dataColumn = new long[dataColumnLength]; + LongArrayList dataColumn = new LongArrayList(new long[dataColumnLength]); // read column data (will be skipped if no data was present) for (int y = 0; y < dataColumnLength; y++) { long dataPoint = compressedIn.readLong(); - dataColumn[y] = dataPoint; + dataColumn.set(y, dataPoint); } dataArray[xz] = dataColumn; From 5d50775932933eb205ec6478c041df873f33831d Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 21 Mar 2024 22:07:35 -0500 Subject: [PATCH 087/183] re-implement data source pooling --- .../fullData/sources/FullDataSourceV1.java | 9 ++ .../fullData/sources/FullDataSourceV2.java | 117 +++++++++++------- .../render/ColumnRenderSource.java | 4 + .../file/AbstractNewDataSourceHandler.java | 24 ++-- .../core/file/IDataSource.java | 6 +- .../FullDataSourceProviderV2.java | 14 ++- .../renderfile/RenderSourceFileHandler.java | 8 +- .../core/sql/dto/FullDataSourceV2DTO.java | 7 +- 8 files changed, 118 insertions(+), 71 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 22b9db0c7..cac5aee09 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -386,6 +386,15 @@ public class FullDataSourceV1 implements IDataSource public void setIdMapping(FullDataPointIdMap mappings) { this.mapping.mergeAndReturnRemappedEntityIds(mappings); } + //==================// + // override methods // + //==================// + + @Override + public void close() throws Exception + { /* not currently needed */ } + + //================// // helper classes // diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 10a1837fe..843f21550 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -903,63 +903,84 @@ public class FullDataSourceV2 implements IDataSource // pooling // //=========// - // TODO add pooled data sources - private static class Pooling + @Override + public void close() throws Exception { - /** used when pooling data sources */ - private final ArrayList cachedSources = new ArrayList<>(); - private final ReentrantLock cacheLock = new ReentrantLock(); - - - /** @return null if no pooled source exists */ - public FullDataSourceV1 tryGetPooledSource() + returnPooledDataSource(this); + } + + + /** used when pooling data sources */ + private static final ArrayList CACHED_SOURCES = new ArrayList<>(); + private static final ReentrantLock CACHE_LOCK = new ReentrantLock(); + + + /** @return an empty data source if non are cached */ + public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData) + { + try { - try - { - this.cacheLock.lock(); - - int index = this.cachedSources.size() - 1; - if (index == -1) - { - return null; - } - else - { - return this.cachedSources.remove(index); - } - } - finally - { - this.cacheLock.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link Pooling#tryGetPooledSource} is called. - */ - public void returnPooledDataSource(FullDataSourceV1 dataSource) - { - if (dataSource == null) - { - return; - } - else if (this.cachedSources.size() > 25) - { - return; - } + CACHE_LOCK.lock(); - try + int index = CACHED_SOURCES.size() - 1; + if (index == -1) { - this.cacheLock.lock(); - this.cachedSources.add(dataSource); + // no pooled sources exist + return createEmpty(pos); } - finally + else { - this.cacheLock.unlock(); + FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index); + dataSource.pos = pos; + + if (clearData) + { + dataSource.mapping.clear(pos); + + for (int i = 0; i < dataSource.dataPoints.length; i++) + { + if (dataSource.dataPoints[i] != null) + { + dataSource.dataPoints[i].clear(); + } + } + + Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + } + + return dataSource; } } + finally + { + CACHE_LOCK.unlock(); + } + } + + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called. + */ + public static void returnPooledDataSource(FullDataSourceV2 dataSource) + { + if (dataSource == null) + { + return; + } + else if (CACHED_SOURCES.size() > 25) + { + return; + } + try + { + CACHE_LOCK.lock(); + CACHED_SOURCES.add(dataSource); + } + finally + { + CACHE_LOCK.unlock(); + } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index b1282c73a..fd2b8a011 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -477,6 +477,10 @@ public class ColumnRenderSource implements IDataSource return stringBuilder.toString(); } + @Override + public void close() throws Exception + { /* not currently needed */ } + //==============// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 3d804d013..048c88ecc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -222,21 +222,23 @@ public abstract class AbstractNewDataSourceHandler // get or create the data source - TDataSource recipientDataSource = this.get(updatePos); - boolean dataModified = recipientDataSource.update(inputData, this.level); - - if (dataModified) + try (TDataSource recipientDataSource = this.get(updatePos)) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(recipientDataSource); - this.repo.save(dto); + boolean dataModified = recipientDataSource.update(inputData, this.level); - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + if (dataModified) { - if (listener != null) + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(recipientDataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - listener.OnDataSourceUpdated(recipientDataSource); + if (listener != null) + { + listener.OnDataSourceUpdated(recipientDataSource); + } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 11738fe7a..0da2c34b6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -11,11 +11,13 @@ import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStre import java.io.IOException; /** - * Base for all data sources. + * Base for all data sources.

+ * + * AutoCloseable Can be implemented to allow for disposing of pooled data sources.

* * @param there are times when we need specifically a client level vs a more generic level */ -public interface IDataSource extends IBaseDTO +public interface IDataSource extends IBaseDTO, AutoCloseable { DhSectionPos getSectionPos(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 70dd8bf1e..94ce359b7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -156,17 +156,17 @@ public class FullDataSourceProviderV2 @Override protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException - { return dto.createDataSource(this.level.getLevelWrapper()); } + { return dto.createPooledDataSource(this.level.getLevelWrapper()); } @Override protected FullDataSourceV2 createNewDataSourceFromExistingDtos(DhSectionPos pos) { // TODO maybe just set children update flags to true? // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway - return FullDataSourceV2.createEmpty(pos); + return FullDataSourceV2.getPooledSource(pos, true); } @Override - protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.createEmpty(pos); } + protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); } @@ -248,9 +248,11 @@ public class FullDataSourceProviderV2 childReadLock.lock(); this.lockedPosSet.add(childPos); - FullDataSourceV2 dataSource = this.get(childPos); - this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); - this.repo.setApplyToParent(childPos, false); + try (FullDataSourceV2 dataSource = this.get(childPos)) + { + this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); + this.repo.setApplyToParent(childPos, false); + } } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java index 2b3fba7b9..d0c18963a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java @@ -92,8 +92,12 @@ public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler // data source population // //========================// - public FullDataSourceV2 createDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException - { return this.populateDataSource(FullDataSourceV2.createEmpty(this.pos), levelWrapper); } + public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException + { + FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false); + return this.populateDataSource(dataSource, levelWrapper); + } public FullDataSourceV2 populateDataSource(FullDataSourceV2 dataSource, @NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { return this.internalPopulateDataSource(dataSource, levelWrapper, false); } From cbfd4f9de35e36ce6cc2ed7823c7ba31c6b12c44 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 23 Mar 2024 15:49:40 -0500 Subject: [PATCH 088/183] Pool FullDataPointIdMap Entries and improve hash logic --- .../fullData/FullDataPointIdMap.java | 100 ++++++++++++++++-- .../core/sql/repo/FullDataSourceV2Repo.java | 9 ++ 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index 468d57bee..56d682a42 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -27,12 +27,12 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrappe import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; +import it.unimi.dsi.fastutil.ints.Int2ReferenceOpenHashMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.*; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.*; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -131,7 +131,7 @@ public class FullDataPointIdMap * If an entry with the given values already exists nothing will * be added but the existing item's ID will still be returned. */ - public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(new Entry(biome, blockState), true); } + public int addIfNotPresentAndGetId(IBiomeWrapper biome, IBlockStateWrapper blockState) { return this.addIfNotPresentAndGetId(Entry.getEntry(biome, blockState), true); } /** @param useWriteLocks should only be false if this method is already in a write lock to prevent unlocking at the wrong time */ private int addIfNotPresentAndGetId(Entry biomeBlockStateEntry, boolean useWriteLocks) { @@ -322,15 +322,81 @@ public class FullDataPointIdMap { private static final IWrapperFactory WRAPPER_FACTORY = SingletonInjector.INSTANCE.get(IWrapperFactory.class); + private static final Int2ReferenceOpenHashMap> ENTRY_POOL = new Int2ReferenceOpenHashMap<>(); + /** lock is necessary since {@link Int2ReferenceOpenHashMap} isn't concurrent and concurrent threads can cause infinite loops */ + private static final ReentrantReadWriteLock ENTRY_POOL_LOCK = new ReentrantReadWriteLock(); + public final IBiomeWrapper biome; public final IBlockStateWrapper blockState; private Integer hashCode = null; - // constructor // - public Entry(IBiomeWrapper biome, IBlockStateWrapper blockState) + //=============// + // constructor // + //=============// + + public static Entry getEntry(IBiomeWrapper biome, IBlockStateWrapper blockState) + { + int entryHash = getHashCode(biome, blockState); + + // try getting the existing entry + try + { + ENTRY_POOL_LOCK.readLock().lock(); + + // check if an entry already exists + ArrayList entryList = ENTRY_POOL.get(entryHash); + if (entryList != null) + { + // at least one entry exists with this hash code + for (int i = 0; i < entryList.size(); i++) + { + Entry entry = entryList.get(i); + if (entry.biome.equals(biome) && entry.blockState.equals(blockState)) + { + return entry; + } + } + + // if we got here, then there was a hash collision and this entry wasn't present in the array + } + } + finally + { + ENTRY_POOL_LOCK.readLock().unlock(); + } + + + // no entry exists, + // create a new one + try + { + ENTRY_POOL_LOCK.writeLock().lock(); + + ArrayList entryList = ENTRY_POOL.get(entryHash); + if (entryList == null) + { + // no entries exist for this hash code + + // we assume that hash collisions should basically never happen, + // so the array starts with an initial capacity of 1. + // However, since collisions will eventually happen, using an arrayList prevents unexpected bugs caused by collisions. + entryList = new ArrayList<>(1); + ENTRY_POOL.put(entryHash, entryList); + } + + Entry newEntry = new Entry(biome, blockState); + entryList.add(newEntry); + return newEntry; + } + finally + { + ENTRY_POOL_LOCK.writeLock().unlock(); + } + } + private Entry(IBiomeWrapper biome, IBlockStateWrapper blockState) { this.biome = biome; this.blockState = blockState; @@ -338,15 +404,29 @@ public class FullDataPointIdMap - // methods // + //===========// + // overrides // + //===========// + public static int getHashCode(Entry entry) { return getHashCode(entry.biome, entry.blockState); } + public static int getHashCode(IBiomeWrapper biome, IBlockStateWrapper blockState) + { + final int prime = 31; + + int result = 1; + // the biome and blockstate hashcode should be already calculated by the time + // we get here, so this operation should be very fast + result = prime * result + (biome == null ? 0 : biome.hashCode()); + result = prime * result + (blockState == null ? 0 : blockState.hashCode()); + return result; + } @Override public int hashCode() { // cache the hash code to improve speed if (this.hashCode == null) { - this.hashCode = this.serialize().hashCode(); + this.hashCode = getHashCode(this); } return this.hashCode; @@ -371,6 +451,10 @@ public class FullDataPointIdMap + //=================// + // (de)serializing // + //=================// + public String serialize() { return this.biome.getSerialString() + BLOCK_STATE_SEPARATOR_STRING + this.blockState.getSerialString(); } public static Entry deserialize(String str, ILevelWrapper levelWrapper) throws IOException, InterruptedException @@ -389,7 +473,7 @@ public class FullDataPointIdMap IBiomeWrapper biome = WRAPPER_FACTORY.deserializeBiomeWrapper(stringArray[0], levelWrapper); IBlockStateWrapper blockState = WRAPPER_FACTORY.deserializeBlockStateWrapper(stringArray[1], levelWrapper); - return new Entry(biome, blockState); + return Entry.getEntry(biome, blockState); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index cea06e2bd..b325aacd7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -40,6 +40,11 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Sat, 23 Mar 2024 16:14:34 -0500 Subject: [PATCH 089/183] Reduce some GC pressure in FullDataSourceV2.mergeInputTwoByTwoDataColumn --- .../fullData/sources/FullDataSourceV2.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 843f21550..bb86518ff 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -495,6 +495,13 @@ public class FullDataSourceV2 implements IDataSource int minY = 0; + // these arrays will be reused quite often, so re-using them helps reduce some GC pressure + long[] datapointsForYSlice = new long[4]; + int[] mergeIds = new int[4]; + int[] mergeBlockLights = new int[4]; + int[] mergeSkyLights = new int[4]; + + for (int blockY = 0; blockY < RenderDataPointUtil.MAX_WORLD_Y_SIZE; blockY++, height++) { // if each column has reached the end of their data, nothing more needs to be done @@ -511,7 +518,7 @@ public class FullDataSourceV2 implements IDataSource // scary double loop but, // this will only ever loop 4 times, // once for each of the 4 input columns - long[] datapointsForYSlice = new long[4]; + Arrays.fill(datapointsForYSlice, 0L); int colIndex = 0; for (int inputX = x; inputX < x + 2; inputX++) { @@ -584,9 +591,9 @@ public class FullDataSourceV2 implements IDataSource - int[] mergeIds = new int[4]; - int[] mergeBlockLights = new int[4]; - int[] mergeSkyLights = new int[4]; + Arrays.fill(mergeIds, 0); + Arrays.fill(mergeBlockLights, 0); + Arrays.fill(mergeSkyLights, 0); for (int i = 0; i < 4; i++) { mergeIds[i] = FullDataPointUtilV2.getId(datapointsForYSlice[i]); From a1c85d91fc4c2aa2398f11ffc0f78f1305ad01ab Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 23 Mar 2024 16:18:21 -0500 Subject: [PATCH 090/183] Comment out trace logs These logs aren't printed and will just increase GC pressure for strings --- .../distanthorizons/core/api/internal/SharedApi.java | 2 +- .../core/dataObjects/fullData/FullDataPointIdMap.java | 8 ++++---- .../render/bufferBuilding/ColumnRenderBufferBuilder.java | 7 ++++--- .../distanthorizons/core/generation/DhLightingEngine.java | 2 +- .../core/generation/WorldGenerationQueue.java | 6 +++--- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java index f72484df5..f04bd98c1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java @@ -293,7 +293,7 @@ public class SharedApi executor.execute(() -> { - LOGGER.trace(chunkWrapper.getChunkPos() + " " + executor.getActiveCount() + " / " + executor.getQueue().size() + " - " + executor.getCompletedTaskCount()); + //LOGGER.trace(chunkWrapper.getChunkPos() + " " + executor.getActiveCount() + " / " + executor.getQueue().size() + " - " + executor.getCompletedTaskCount()); try { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index 56d682a42..4dd9ef5b9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -182,7 +182,7 @@ public class FullDataPointIdMap { try { - LOGGER.trace("merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); + //LOGGER.trace("merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); inputMap.readWriteLock.readLock().lock(); this.readWriteLock.writeLock().lock(); @@ -203,7 +203,7 @@ public class FullDataPointIdMap this.readWriteLock.writeLock().unlock(); inputMap.readWriteLock.readLock().unlock(); - LOGGER.trace("finished merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); + //LOGGER.trace("finished merging {" + this.pos + ", " + this.entryList.size() + "} and {" + inputMap.pos + ", " + inputMap.entryList.size() + "}"); } } @@ -254,7 +254,7 @@ public class FullDataPointIdMap finally { this.readWriteLock.readLock().unlock(); - LOGGER.trace("serialize " + this.pos + " " + this.entryList.size()); + //LOGGER.trace("serialize " + this.pos + " " + this.entryList.size()); } } @@ -287,7 +287,7 @@ public class FullDataPointIdMap } } - LOGGER.trace("deserialized " + pos + " " + newMap.entryList.size() + "-" + entityCount); + //LOGGER.trace("deserialized " + pos + " " + newMap.entryList.size() + "-" + entityCount); return newMap; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 4c9bd2b43..0761b66bb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -85,7 +85,7 @@ public class ColumnRenderBufferBuilder { boolean enableTransparency = Config.Client.Advanced.Graphics.Quality.transparency.get().transparencyEnabled; - EVENT_LOGGER.trace("RenderRegion start QuadBuild @ " + renderSource.sectionPos); + //EVENT_LOGGER.trace("RenderRegion start QuadBuild @ " + renderSource.sectionPos); boolean enableSkyLightCulling = ( // dimensions with a ceiling will be all caves so we don't want cave culling @@ -126,7 +126,7 @@ public class ColumnRenderBufferBuilder { try { - EVENT_LOGGER.trace("RenderRegion start Upload @ " + renderSource.sectionPos); + //EVENT_LOGGER.trace("RenderRegion start Upload @ " + renderSource.sectionPos); ColumnRenderBuffer buffer = renderBufferRef.swap(null); if (buffer == null) @@ -138,7 +138,8 @@ public class ColumnRenderBufferBuilder { buffer.uploadBuffer(quadBuilder, GLProxy.getInstance().getGpuUploadMethod()); LodUtil.assertTrue(buffer.buffersUploaded); - EVENT_LOGGER.trace("RenderRegion end Upload @ " + renderSource.sectionPos); + + //EVENT_LOGGER.trace("RenderRegion end Upload @ " + renderSource.sectionPos); return buffer; } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java index c319c3b07..0e648ef33 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/DhLightingEngine.java @@ -207,7 +207,7 @@ public class DhLightingEngine long endTimeNs = System.nanoTime(); float totalTimeMs = (endTimeNs - startTimeNs) / 1_000_000.0f; - LOGGER.trace("Finished generating lighting for chunk: [" + centerChunkPos + "] in ["+totalTimeMs+"] milliseconds"); + //LOGGER.trace("Finished generating lighting for chunk: [" + centerChunkPos + "] in ["+totalTimeMs+"] milliseconds"); } /** Applies each {@link LightPos} from the queue to the given set of {@link IChunkWrapper}'s. */ diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index e946d2698..05d75ec8c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -292,7 +292,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb boolean taskStarted = this.tryStartingWorldGenTaskGroup(newTaskGroup); if (!taskStarted) { - LOGGER.trace("Unable to start task: "+closestTask.pos+", skipping. Task position may have already been generated."); + //LOGGER.trace("Unable to start task: "+closestTask.pos+", skipping. Task position may have already been generated."); } } else @@ -302,7 +302,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb // the newly selected task, we cannot use it, // as some chunks may have already been written into. - LOGGER.trace("A task already exists for this position, todo: "+closestTask.pos); + //LOGGER.trace("A task already exists for this position, todo: "+closestTask.pos); } // a task has been started @@ -349,7 +349,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb if (this.alreadyGeneratedPosHashSet.containsKey(newTaskGroup.group.pos)) { // temporary solution to prevent generating the same section multiple times - LOGGER.trace("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping..."); + //LOGGER.trace("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping..."); // sending a success result is necessary to make sure the render sections are reloaded correctly newTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(new DhSectionPos(granularity, taskPos.getX(), taskPos.getZ())))); From b64df318ce4c369e57bce7b301d7d08aca7593e2 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 23 Mar 2024 17:18:05 -0500 Subject: [PATCH 091/183] Improve error handling in DebugRenderer --- .../core/render/renderer/DebugRenderer.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java index 7f53323f0..92fec9b71 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java @@ -56,10 +56,9 @@ public class DebugRenderer { public static DebugRenderer INSTANCE = new DebugRenderer(); - public static final ConfigBasedLogger logger = new ConfigBasedLogger( - LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT); - public static final ConfigBasedSpamLogger spamLogger = new ConfigBasedSpamLogger( - LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT, 1); + public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT); + public static final ConfigBasedSpamLogger SPAM_LOGGER = new ConfigBasedSpamLogger(LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT, 1); + private static final IMinecraftRenderWrapper MC_RENDER = SingletonInjector.INSTANCE.get(IMinecraftRenderWrapper.class); @@ -480,18 +479,25 @@ public class DebugRenderer { synchronized (this) { - Iterator> iterator = rendererList.iterator(); - while (iterator.hasNext()) + try { - WeakReference ref = iterator.next(); - IDebugRenderable renderable = ref.get(); - if (renderable == null) + Iterator> iterator = rendererList.iterator(); + while (iterator.hasNext()) { - iterator.remove(); - continue; + WeakReference ref = iterator.next(); + IDebugRenderable renderable = ref.get(); + if (renderable == null) + { + iterator.remove(); + continue; + } + + renderable.debugRender(debugRenderer); } - - renderable.debugRender(debugRenderer); + } + catch (Exception e) + { + SPAM_LOGGER.error("Unexpected Debug renderer error, Error: "+e.getMessage(), e); } } } From 8c5cd7f11f7a7083b88580f2cfc155cb16a321d3 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 23 Mar 2024 17:55:59 -0500 Subject: [PATCH 092/183] Remove render data file handling and related code --- .../fullData/sources/FullDataSourceV1.java | 15 +- .../fullData/sources/FullDataSourceV2.java | 13 - .../render/ColumnRenderSource.java | 105 +---- .../render/ColumnRenderSourceLoader.java | 6 +- .../file/AbstractLegacyDataSourceHandler.java | 420 ------------------ .../core/file/IDataSource.java | 10 - .../FullDataSourceProviderV1.java | 154 ++++--- .../renderfile/IRenderSourceProvider.java | 45 -- .../renderfile/RenderSourceFileHandler.java | 173 -------- .../core/level/ClientLevelModule.java | 70 +-- .../core/render/LodQuadTree.java | 17 +- .../core/render/LodRenderSection.java | 56 +-- ...ourceDTO.java => FullDataSourceV1DTO.java} | 14 +- ...rceRepo.java => FullDataSourceV1Repo.java} | 97 +++- .../core/sql/repo/LegacyFullDataRepo.java | 102 ----- .../core/sql/repo/RenderDataRepo.java | 43 -- 16 files changed, 281 insertions(+), 1059 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/IRenderSourceProvider.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java rename core/src/main/java/com/seibel/distanthorizons/core/sql/dto/{LegacyDataSourceDTO.java => FullDataSourceV1DTO.java} (81%) rename core/src/main/java/com/seibel/distanthorizons/core/sql/repo/{AbstractLegacyDataSourceRepo.java => FullDataSourceV1Repo.java} (64%) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index cac5aee09..33e89daaa 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -24,7 +24,7 @@ import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; @@ -157,7 +157,7 @@ public class FullDataSourceV1 implements IDataSource * Clears and then overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an existing {@link FullDataSourceV1} and can be used in place of a constructor to reuse an existing {@link FullDataSourceV1} object. */ - public void repopulateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void repopulateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { // clear/overwrite the old data this.resizeDataStructuresForRepopulation(dto.pos); @@ -171,7 +171,7 @@ public class FullDataSourceV1 implements IDataSource * Overwrites any data in this object with the data from the given file and stream. * This is expected to be used with an empty {@link FullDataSourceV1} and functions similar to a constructor. */ - public void populateFromStream(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException + public void populateFromStream(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException, InterruptedException { FullDataSourceSummaryData summaryData = this.readSourceSummaryInfo(dto, inputStream, level); this.setSourceSummaryData(summaryData); @@ -193,13 +193,6 @@ public class FullDataSourceV1 implements IDataSource // low level stream methods // - @Deprecated - @Override - public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) throws IOException - { - throw new UnsupportedOperationException("Deprecated"); - } - /** unused, just here for reference as to how the data was written */ @Deprecated public void writeSourceSummaryInfo(IDhLevel level, DhDataOutputStream outputStream) throws IOException @@ -210,7 +203,7 @@ public class FullDataSourceV1 implements IDataSource outputStream.writeByte(this.worldGenStep.value); } - public FullDataSourceSummaryData readSourceSummaryInfo(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public FullDataSourceSummaryData readSourceSummaryInfo(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataDetail = inputStream.readInt(); if (dataDetail != dto.dataDetailLevel) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index bb86518ff..5a5dff219 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -893,19 +893,6 @@ public class FullDataSourceV2 implements IDataSource - //============// - // deprecated // - //============// - - @Deprecated - @Override - public void writeToStream(DhDataOutputStream outputStream, IDhLevel level) - { - throw new UnsupportedOperationException("deprecated"); - } - - - //=========// // pooling // //=========// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index fd2b8a011..0d40f9e1a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -211,83 +211,9 @@ public class ColumnRenderSource implements IDataSource - //========================// - // data update and output // - //========================// - - @Override - public void writeToStream(DhDataOutputStream outputStream, IDhClientLevel level) throws IOException { this.writeToStream(outputStream); } - public void writeToStream(DhDataOutputStream outputStream) throws IOException - { - outputStream.flush(); - - outputStream.writeByte(this.getDataDetailLevel()); - outputStream.writeInt(this.verticalDataCount); - - if (this.isEmpty) - { - // no data is present - outputStream.writeByte(NO_DATA_FLAG_BYTE); - } - else - { - // data is present - outputStream.writeByte(DATA_GUARD_BYTE); - outputStream.writeInt(this.yOffset); - - // write the data for each column - for (int xz = 0; xz < SECTION_SIZE * SECTION_SIZE; xz++) - { - for (int y = 0; y < this.verticalDataCount; y++) - { - long currentDatapoint = this.renderDataContainer[xz * this.verticalDataCount + y]; - outputStream.writeLong(Long.reverseBytes(currentDatapoint)); // the reverse bytes is necessary to ensure the data is read in correctly - } - } - } - - outputStream.writeByte(DATA_GUARD_BYTE); - outputStream.writeByte(this.worldGenStep.value); - - outputStream.flush(); - } - - /** Overrides any data that has not been written directly using write(). Skips empty source dataPoints. */ - public void updateFromRenderSource(ColumnRenderSource renderSource) - { - // validate we are writing for the same location - LodUtil.assertTrue(renderSource.sectionPos.equals(this.sectionPos)); - - // change the vertical size if necessary (this can happen if the vertical quality was changed in the config) - this.clearAndChangeVerticalSize(renderSource.verticalDataCount); - // validate both objects have the same number of dataPoints - LodUtil.assertTrue(renderSource.verticalDataCount == this.verticalDataCount); - - - if (renderSource.isEmpty) - { - // the source is empty, don't attempt to update anything - return; - } - // the source isn't empty, this object won't be empty after the method finishes - this.isEmpty = false; - - localVersion.incrementAndGet(); - } - /** - * If the newVerticalSize is different than the current verticalSize, - * this will delete any data currently in this object and re-size it.
- * Otherwise this method will do nothing. - */ - private void clearAndChangeVerticalSize(int newVerticalSize) - { - if (newVerticalSize != this.verticalDataCount) - { - this.verticalDataCount = newVerticalSize; - this.renderDataContainer = new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]; - this.localVersion.incrementAndGet(); - } - } + //=============// + // data update // + //=============// @Override public boolean update(FullDataSourceV2 inputFullDataSource, IDhClientLevel level) @@ -355,37 +281,12 @@ public class ColumnRenderSource implements IDataSource // data helper methods // //=====================// - public boolean doesDataPointExist(int posX, int posZ) { return RenderDataPointUtil.doesDataPointExist(this.getFirstDataPoint(posX, posZ)); } - - public void generateData(ColumnRenderSource lowerDataContainer, int posX, int posZ) - { - ColumnArrayView outputView = this.getVerticalDataPointView(posX, posZ); - ColumnQuadView quadView = lowerDataContainer.getQuadViewOverRange(posX * 2, posZ * 2, 2, 2); - outputView.mergeMultiDataFrom(quadView); - } - - public int getMaxLodCount() { return SECTION_SIZE * SECTION_SIZE * this.getVerticalSize(); } - - public long getRoughRamUsageInBytes() { return (long) this.renderDataContainer.length * Long.BYTES; } - public DhSectionPos getSectionPos() { return this.sectionPos; } @Override public DhSectionPos getKey() { return this.sectionPos; } public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - /** @return how many data points wide this {@link ColumnRenderSource} is. */ - public int getWidthInDataPoints() { return BitShiftUtil.powerOfTwo(this.getDetailOffset()); } - public byte getDetailOffset() { return SECTION_SIZE_OFFSET; } - - public byte getRenderDataFormatVersion() { return DATA_FORMAT_VERSION; } - - /** - * Whether this object is still valid. If not, a new one should be created. - * TODO this will be necessary for dedicated multiplayer support, if the server has newer data this section should no longer be valid - */ - public boolean isValid() { return true; } - public boolean isEmpty() { return this.isEmpty; } public void markNotEmpty() { this.isEmpty = false; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java index 94caa4724..1e816ce0f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java @@ -22,7 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import org.apache.logging.log4j.Logger; @@ -31,7 +31,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; /** - * Handles loading and parsing {@link LegacyDataSourceDTO}s to create {@link ColumnRenderSource}s.

+ * Handles loading and parsing {@link FullDataSourceV1DTO}s to create {@link ColumnRenderSource}s.

* * Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what * file versions this class can handle. @@ -48,7 +48,7 @@ public class ColumnRenderSourceLoader - public ColumnRenderSource loadRenderSource(LegacyDataSourceDTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException + public ColumnRenderSource loadRenderSource(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException { int dataFileVersion = dto.binaryDataFormatVersion; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java deleted file mode 100644 index 18b82fe15..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractLegacyDataSourceHandler.java +++ /dev/null @@ -1,420 +0,0 @@ -package com.seibel.distanthorizons.core.file; - -import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.TimerUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; -import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.Nullable; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.channels.ClosedChannelException; -import java.util.Enumeration; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; -import java.util.zip.Adler32; -import java.util.zip.CheckedOutputStream; - -@Deprecated -public abstract class AbstractLegacyDataSourceHandler, TDhLevel extends IDhLevel> - implements AutoCloseable -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private static final Timer DELAYED_SAVE_TIMER = TimerUtil.CreateTimer("DataSourceSaveTimer"); - /** How long a data source must remain un-modified before being written to disk. */ - private static final int SAVE_DELAY_IN_MS = 4_000; - - /** - * The highest numerical detail level known about. - * Used when determining which positions to update. - */ - protected final AtomicInteger topSectionDetailLevelRef; - protected final int minDetailLevel = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; - - protected final ConcurrentHashMap unsavedDataSourceBySectionPos = new ConcurrentHashMap<>(); - protected final ConcurrentHashMap saveTimerTasksBySectionPos = new ConcurrentHashMap<>(); - - protected final ReentrantLock[] updateLockArray; - protected final ReentrantLock[] queueSaveLockArray; - protected final ReentrantLock closeLock = new ReentrantLock(); - protected volatile boolean isShutdown = false; - - protected final TDhLevel level; - protected final File saveDir; - - public final AbstractLegacyDataSourceRepo repo; - - - - //=============// - // constructor // - //=============// - - public AbstractLegacyDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public AbstractLegacyDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) - { - this.level = level; - this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; - if (!this.saveDir.exists() && !this.saveDir.mkdirs()) - { - LOGGER.warn("Unable to create full data folder, file saving may fail."); - } - - // the lock arrays' length is double the number of CPU cores so the number of collisions - // should be relatively low without having too many extra locks - int lockCount = Runtime.getRuntime().availableProcessors() * 2; - this.updateLockArray = new ReentrantLock[lockCount]; - this.queueSaveLockArray = new ReentrantLock[lockCount]; - for (int i = 0; i < lockCount; i++) - { - this.updateLockArray[i] = new ReentrantLock(); - this.queueSaveLockArray[i] = new ReentrantLock(); - } - - this.repo = this.createRepo(); - - // determine the top detail level currently in the database - int maxSectionDetailLevel = this.repo.getMaxSectionDetailLevel(); - this.topSectionDetailLevelRef = new AtomicInteger(maxSectionDetailLevel); - } - - - - - //==================// - // abstract methods // - //==================// - - /** When this is called the parent folders should be created */ - protected abstract AbstractLegacyDataSourceRepo createRepo(); - - protected abstract TDataSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException; - /** - * Creates a new data source using any DTOs already present in the database. - * Can return null if there was an issue, but in general should return at least an empty data source. - */ - @Nullable - protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos); - - protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); - - - - //==============// - // data reading // - //==============// - - /** - * Returns the {@link TDataSource} for the given section position.
- * The returned data source may be null if there was a problem.

- * - * This call is concurrent. I.e. it supports being called by multiple threads at the same time. - */ - public CompletableFuture getAsync(DhSectionPos pos) - { - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); - } - /** - * Should only be used in internal file handler methods where we are already running on a file handler thread. - * Can return null if there was a problem. - * @see AbstractLegacyDataSourceHandler#getAsync(DhSectionPos) - */ - @Nullable - public TDataSource get(DhSectionPos pos) - { - // used the unsaved data source if present - if (this.unsavedDataSourceBySectionPos.containsKey(pos)) - { - return this.unsavedDataSourceBySectionPos.get(pos); - } - // an unsaved data source isn't present - // check the database - - - // increase the top detail level if necessary - this.topSectionDetailLevelRef.updateAndGet(oldDetailLevel -> Math.max(oldDetailLevel, pos.getDetailLevel())); - - - TDataSource dataSource = null; - try - { - LegacyDataSourceDTO dto = this.repo.getByKey(pos); - if (dto != null) - { - // load from file - dataSource = this.createDataSourceFromDto(dto); - } - else - { - // attempt to create from any existing files - dataSource = this.createNewDataSourceFromExistingDtos(pos); - } - } - catch (InterruptedException ignore) { } - catch (IOException e) - { - LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); - } - - return dataSource; - } - - - - //===============// - // data updating // - //===============// - - public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) - { - ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); - if (executor == null || executor.isTerminated()) - { - return CompletableFuture.completedFuture(null); - } - - - try - { - // run file handling on a separate thread - return CompletableFuture.runAsync(() -> - { - DhSectionPos bottomPos = inputDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); - - bottomPos.forEachPosUpToDetailLevel( - this.topSectionDetailLevelRef.byteValue(), - (pos) -> this.updateDataSourceAtPos(pos, inputDataSource) ); - - }, executor); - } - catch (RejectedExecutionException ignore) - { - // can happen if the executor was shutdown while this task was queued - return CompletableFuture.completedFuture(null); - } - } - protected void updateDataSourceAtPos(DhSectionPos pos, FullDataSourceV2 newDataSource) - { - // a lock is necessary to prevent two threads from writing to the same position at once, - // if that happens only the second update will apply and the LOD will end up with hole(s) - ReentrantLock updateLock = this.getUpdateLockForPos(pos); - - try - { - updateLock.lock(); - - // get or create the data source - TDataSource dataSource = this.get(pos); - if (dataSource == null) - { - dataSource = this.makeEmptyDataSource(pos); - } - dataSource.update(newDataSource, this.level); - - this.queueDelayedSave(dataSource); - } - catch (Exception e) - { - LOGGER.error("Error updating pos ["+pos+"], error: "+e.getMessage(), e); - } - finally - { - updateLock.unlock(); - } - } - /** - * Queues the given data source to save after {@link AbstractLegacyDataSourceHandler#SAVE_DELAY_IN_MS} - * milliseconds have passed without any additional modifications.

- * - * This prevents repeatedly reading/writing the same data source to/from disk if said - * source is currently being updated via world gen or chunk modifications. - * This drastically reduces disk usage and improves performance. - */ - protected void queueDelayedSave(TDataSource dataSource) - { - // a lock is necessary to prevent two threads from queuing a save at the same time, - // which can cause the timer to queue canceled tasks - DhSectionPos pos = dataSource.getSectionPos(); - ReentrantLock saveQueueLock = this.getSaveQueueLockForPos(pos); - - - // done to prevent queueing saves while the current queue is being cleared - if (this.isShutdown) - { - return; - } - - - try - { - saveQueueLock.lock(); - - // put the data source in memory until it can be flushed to disk - this.unsavedDataSourceBySectionPos.put(pos, dataSource); - - TimerTask task = new TimerTask() - { - @Override - public void run() - { - - // remove this task from the queue - AbstractLegacyDataSourceHandler.this.saveTimerTasksBySectionPos.remove(pos); - - try - { - final TDataSource finalDataSource = AbstractLegacyDataSourceHandler.this.unsavedDataSourceBySectionPos.remove(pos); - - // this can rarely happen due to imperfect concurrency handling, - // if the data source is null that just means it has already been saved so nothing needs to be done - if (finalDataSource != null) - { - AbstractLegacyDataSourceHandler.this.writeDataSourceToFile(finalDataSource); - } - } - catch (Exception e) // this can throw errors (not exceptions) when installed in Iris' dev environment for some reason due to an issue with LZ4's compression library - { - LOGGER.error("Failed to save updated data for section ["+pos+"], error: ["+e.getMessage()+"]", e); - } - } - }; - try - { - DELAYED_SAVE_TIMER.schedule(task, SAVE_DELAY_IN_MS); - } - catch (IllegalStateException ignore) - { - // James isn't sure why this is possible since this logic is inside a lock, - // maybe the timer is just async enough that there can be problems? - LOGGER.warn("Attempted to queue an already canceled task. Pos: ["+pos+"], task already queued for pos: ["+this.saveTimerTasksBySectionPos.containsKey(pos)+"]"); - } - - - // cancel the old save timer if present - // (this is equivalent to restarting the timer) - TimerTask oldTask = this.saveTimerTasksBySectionPos.put(pos, task); - if (oldTask != null) - { - oldTask.cancel(); - } - } - finally - { - saveQueueLock.unlock(); - } - } - protected void writeDataSourceToFile(TDataSource dataSource) throws IOException - { - LodUtil.assertTrue(dataSource != null); - - try - { - // write the outputs to a stream to prep for writing to the database - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - - // the order of these streams is important, otherwise the checksum won't be calculated - CheckedOutputStream checkedOut = new CheckedOutputStream(byteArrayOutputStream, new Adler32()); - // normally a DhStream should be the topmost stream to prevent closing the stream accidentally, - // but since this stream will be closed immediately after writing anyway, it won't be an issue - DhDataOutputStream compressedOut = new DhDataOutputStream(checkedOut, EDhApiDataCompressionMode.LZ4); - - dataSource.writeToStream(compressedOut, AbstractLegacyDataSourceHandler.this.level); - - compressedOut.flush(); - int checksum = (int) checkedOut.getChecksum().getValue(); - byteArrayOutputStream.close(); - - - // save the DTO - LegacyDataSourceDTO newDto = new LegacyDataSourceDTO( - dataSource.getSectionPos(), checksum, - dataSource.getDataDetailLevel(), EDhApiWorldGenerationStep.EMPTY, ColumnRenderSource.DATA_NAME, - dataSource.getDataFormatVersion(), - byteArrayOutputStream.toByteArray()); - this.repo.save(newDto); - } - catch (ClosedChannelException e) // includes ClosedByInterruptException - { - // expected if the file handler is shut down, the exception can be ignored - } - } - - - - //================// - // helper methods // - //================// - - /** Based on the stack overflow post: https://stackoverflow.com/a/45909920 */ - protected ReentrantLock getUpdateLockForPos(DhSectionPos pos) { return this.updateLockArray[Math.abs(pos.hashCode()) % this.updateLockArray.length]; } - protected ReentrantLock getSaveQueueLockForPos(DhSectionPos pos) { return this.queueSaveLockArray[Math.abs(pos.hashCode()) % this.queueSaveLockArray.length]; } - - - - //=========// - // cleanup // - //=========// - - @Override - public void close() - { - try - { - this.closeLock.lock(); - this.isShutdown = true; - - // wait a moment so any queued saves can finish queuing, - // otherwise we might not see everything that needs saving and attempt to use a closed repo - Thread.sleep(200); - - LOGGER.info("Closing [" + this.getClass().getSimpleName() + "] for level: [" + this.level + "], saving [" + this.saveTimerTasksBySectionPos.size() + "] positions."); - - - Enumeration list = this.saveTimerTasksBySectionPos.keys(); - while (list.hasMoreElements()) - { - DhSectionPos pos = list.nextElement(); - TimerTask saveTask = this.saveTimerTasksBySectionPos.remove(pos); - if (saveTask != null) - { - saveTask.run(); - // canceling the task doesn't need to be done since the it has internal logic to prevent running more than once - } - } - - LOGGER.info("[" + this.getClass().getSimpleName() + "] saving complete, closing repo."); - this.repo.close(); - } - catch (InterruptedException ignore) { } - finally - { - this.closeLock.unlock(); - } - } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 0da2c34b6..edcb69407 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -21,19 +21,9 @@ public interface IDataSource extends IBaseDTO. - */ - package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; -import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.sql.repo.LegacyFullDataRepo; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; +import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV1Repo; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -35,34 +15,53 @@ import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.locks.ReentrantLock; -public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler +public class FullDataSourceProviderV1 + implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + protected final ReentrantLock closeLock = new ReentrantLock(); + protected volatile boolean isShutdown = false; + + protected final TDhLevel level; + protected final File saveDir; + + public final FullDataSourceV1Repo repo; + //=============// // constructor // //=============// - public FullDataSourceProviderV1(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + public FullDataSourceProviderV1(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { - super(level, saveStructure, saveDirOverride); + this.level = level; + this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; + if (!this.saveDir.exists() && !this.saveDir.mkdirs()) + { + LOGGER.warn("Unable to create full data folder, file saving may fail."); + } + + this.repo = this.createRepo(); } - //====================// - // Abstract overrides // - //====================// + //==================// + // abstract methods // + //==================// - @Override - protected AbstractLegacyDataSourceRepo createRepo() + /** When this is called the parent folders should be created */ + protected FullDataSourceV1Repo createRepo() { try { - return new LegacyFullDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); + return new FullDataSourceV1Repo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); } catch (SQLException e) { @@ -72,32 +71,61 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler + * The returned data source may be null if there was a problem.

+ * + * This call is concurrent. I.e. it supports being called by multiple threads at the same time. + */ + public CompletableFuture getAsync(DhSectionPos pos) + { + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return CompletableFuture.completedFuture(null); + } + + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + /** + * Should only be used in internal file handler methods where we are already running on a file handler thread. + * Can return null. + * @see FullDataSourceProviderV1#getAsync(DhSectionPos) + */ + @Nullable + public FullDataSourceV1 get(DhSectionPos pos) + { + FullDataSourceV1 dataSource = null; + try + { + FullDataSourceV1DTO dto = this.repo.getByKey(pos); + if (dto != null) + { + // load from file + dataSource = this.createDataSourceFromDto(dto); + } + } + catch (InterruptedException ignore) { } + catch (IOException e) + { + LOGGER.warn("File read Error for pos ["+pos+"], error: "+e.getMessage(), e); + } + + return dataSource; + } @@ -105,13 +133,13 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler getDataSourcesToMigrate(int limit) { ArrayList dataSourceList = new ArrayList<>(); - ArrayList migrationPosList = ((LegacyFullDataRepo) this.repo).getPositionsToMigrate(limit); + ArrayList migrationPosList = ((FullDataSourceV1Repo) this.repo).getPositionsToMigrate(limit); for (int i = 0; i < migrationPosList.size(); i++) { DhSectionPos pos = migrationPosList.get(i); @@ -125,7 +153,29 @@ public class FullDataSourceProviderV1 extends AbstractLegacyDataSourceHandler. - */ - -package com.seibel.distanthorizons.core.file.renderfile; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; - -import java.util.concurrent.CompletableFuture; - -/** - * Handles reading, writing, and updating {@link ColumnRenderSource}'s.
- * Should be backed by a database handled by a {@link RenderDataRepo}. - * - * @deprecated an interface isn't necessary for the single render source provider we have - */ -@Deprecated -public interface IRenderSourceProvider extends AutoCloseable -{ - CompletableFuture getAsync(DhSectionPos pos); - - void updateDataSource(FullDataSourceV2 dataSource); - - /** Deletes any data stored in the render cache so it can be re-created */ - void deleteRenderCache(); - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java deleted file mode 100644 index d0c18963a..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderSourceFileHandler.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.file.renderfile; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSourceLoader; -import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; -import com.seibel.distanthorizons.core.file.AbstractLegacyDataSourceHandler; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.logging.f3.F3Screen; -import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; -import com.seibel.distanthorizons.core.level.IDhClientLevel; -import com.seibel.distanthorizons.core.sql.repo.AbstractLegacyDataSourceRepo; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; -import com.seibel.distanthorizons.core.sql.repo.RenderDataRepo; -import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.*; -import java.util.concurrent.*; - -public class RenderSourceFileHandler extends AbstractLegacyDataSourceHandler implements IRenderSourceProvider -{ - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - private final F3Screen.NestedMessage threadPoolMsg; - - public final FullDataSourceProviderV2 fullDataSourceProvider; - - - - //=============// - // constructor // - //=============// - - public RenderSourceFileHandler(FullDataSourceProviderV2 sourceProvider, IDhClientLevel clientLevel, AbstractSaveStructure saveStructure) - { - super(clientLevel, saveStructure); - - this.fullDataSourceProvider = sourceProvider; - this.threadPoolMsg = new F3Screen.NestedMessage(this::f3Log); - } - - - - //====================// - // Abstract overrides // - //====================// - - @Override - protected AbstractLegacyDataSourceRepo createRepo() - { - try - { - return new RenderDataRepo("jdbc:sqlite", this.saveDir.getPath() + "/" + AbstractSaveStructure.DATABASE_NAME); - } - catch (SQLException e) - { - // should only happen if there is an issue with the database (it's locked or can't be created if missing) - // or the database update failed - throw new RuntimeException(e); - } - } - - @Override - protected ColumnRenderSource createDataSourceFromDto(LegacyDataSourceDTO dto) throws InterruptedException, IOException - { return ColumnRenderSourceLoader.INSTANCE.loadRenderSource(dto, dto.getInputStream(), this.level); } - @Override - protected ColumnRenderSource createNewDataSourceFromExistingDtos(DhSectionPos pos) - { - ColumnRenderSource renderDataSource; - - try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos)) - { - renderDataSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - } - catch (Exception e) { throw new RuntimeException(e); } - - return renderDataSource; - } - - @Override - protected ColumnRenderSource makeEmptyDataSource(DhSectionPos pos) - { return ColumnRenderSource.createEmptyRenderSource(pos); } - - - - //=====================// - // extension overrides // - //=====================// - - @Override - public void updateDataSource(FullDataSourceV2 inputDataSource) - { - // TODO once the legacy data provider has been replaced this can be removed - this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource); - } - - - - //=========// - // F3 menu // - //=========// - - /** Returns what should be displayed in Minecraft's F3 debug menu */ - private String[] f3Log() - { - ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor(); - String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-"; - String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-"; - - ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); - String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-"; - String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; - - int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); - - - - ArrayList lines = new ArrayList<>(); - lines.add("File Handler [" + this.level.getLevelWrapper().getDimensionType().getDimensionName() + "]"); - lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); - lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); - lines.add(" Level Unsaved #: " + this.level.getUnsavedDataSourceCount()); - if (unsavedDataSourceCount != -1) - { - lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount); - } - lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size()); - lines.add(" Unsaved render sources: " + this.unsavedDataSourceBySectionPos.size()); - - - - return lines.toArray(new String[0]); - } - - - - //=====================// - // shutdown / clearing // - //=====================// - - public void close() - { - super.close(); - this.threadPoolMsg.close(); - } - - public void deleteRenderCache() { this.repo.deleteAll(); } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 7b88e473c..139e2d73f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -26,7 +26,6 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.renderfile.RenderSourceFileHandler; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -36,13 +35,16 @@ import com.seibel.distanthorizons.core.render.LodQuadTree; import com.seibel.distanthorizons.core.render.RenderBufferHandler; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; import com.seibel.distanthorizons.core.util.LodUtil; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import org.apache.logging.log4j.Logger; import java.io.Closeable; +import java.util.ArrayList; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc @@ -50,7 +52,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); - private final IDhClientLevel parentClientLevel; + private final IDhClientLevel clientLevel; public final FullDataSourceProviderV2 fullDataSourceProvider; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); @@ -63,12 +65,12 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // constructor // //=============// - public ClientLevelModule(IDhClientLevel parentClientLevel) + public ClientLevelModule(IDhClientLevel clientLevel) { - this.parentClientLevel = parentClientLevel; + this.clientLevel = clientLevel; this.f3Message = new F3Screen.NestedMessage(this::f3Log); - this.fullDataSourceProvider = this.parentClientLevel.getFullDataProvider(); + this.fullDataSourceProvider = this.clientLevel.getFullDataProvider(); this.fullDataSourceProvider.dateSourceUpdateListeners.add(this); } @@ -102,14 +104,14 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle return; } - IClientLevelWrapper clientLevelWrapper = this.parentClientLevel.getClientLevelWrapper(); + IClientLevelWrapper clientLevelWrapper = this.clientLevel.getClientLevelWrapper(); if (clientLevelWrapper == null) { return; } clientRenderState.close(); - clientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); + clientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, clientRenderState)) { //FIXME: How to handle this? @@ -141,7 +143,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle /** @return if the {@link ClientRenderState} was successfully swapped */ public boolean startRenderer(IClientLevelWrapper clientLevelWrapper) { - ClientRenderState ClientRenderState = new ClientRenderState(this.parentClientLevel, clientLevelWrapper, this.parentClientLevel.getFullDataProvider(), this.parentClientLevel.getSaveStructure()); + // TODO why are we passing in a level wrapper? Our client level is already defined. + ClientRenderState ClientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure()); if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState)) { LOGGER.warn("Failed to start renderer due to concurrency"); @@ -208,15 +211,14 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // data handling // //===============// - public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.parentClientLevel.getFullDataProvider().updateDataSourceAsync(data); } + public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientLevel.getFullDataProvider().updateDataSourceAsync(data); } @Override public void OnDataSourceUpdated(FullDataSourceV2 updatedFullDataSource) { - // if rendering also update the render sources + // if rendering, also update the render sources ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { - ClientRenderState.renderSourceFileHandler.updateDataSource(updatedFullDataSource); ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos()); } } @@ -254,19 +256,40 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // misc helper functions // //=======================// - /** Returns what should be displayed in Minecraft's F3 debug menu */ - protected String[] f3Log() + private String[] f3Log() { - String dimName = this.parentClientLevel.getClientLevelWrapper().getDimensionType().getDimensionName(); - ClientRenderState renderState = this.ClientRenderStateRef.get(); - if (renderState == null) + String dimName = this.clientLevel.getLevelWrapper().getDimensionType().getDimensionName(); + boolean rendererActive = this.ClientRenderStateRef.get() != null; + + ThreadPoolExecutor fileExecutor = ThreadPoolUtil.getFileHandlerExecutor(); + String fileQueueSize = (fileExecutor != null) ? fileExecutor.getQueue().size()+"" : "-"; + String fileCompletedTaskSize = (fileExecutor != null) ? fileExecutor.getCompletedTaskCount()+"" : "-"; + + ThreadPoolExecutor updateExecutor = ThreadPoolUtil.getUpdatePropagatorExecutor(); + String updateQueueSize = (updateExecutor != null) ? updateExecutor.getQueue().size()+"" : "-"; + String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; + + int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); + + + ArrayList lines = new ArrayList<>(); + lines.add(""); + lines.add("level [" + dimName + "] rendering: " + (rendererActive ? "Active" : "Inactive")); + // TODO a lot of these items only need to be rendered once, but we don't currently have a way of doing that, so only add them for the rendered level + if (rendererActive) { - return new String[]{"level @ " + dimName + ": Inactive"}; - } - else - { - return new String[]{"level @ " + dimName + ": Active"}; + lines.add("File Handler [" + dimName + "]"); + lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); + lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); + lines.add(" Level Unsaved #: " + this.clientLevel.getUnsavedDataSourceCount()); + if (unsavedDataSourceCount != -1) + { + lines.add(" File Handler Unsaved #: " + unsavedDataSourceCount); + } + lines.add(" Parent Update #: " + this.fullDataSourceProvider.parentUpdatingPosSet.size()); } + + return lines.toArray(new String[0]); } public void clearRenderCache() @@ -299,7 +322,6 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle public final IClientLevelWrapper clientLevelWrapper; public final LodQuadTree quadtree; - public final RenderSourceFileHandler renderSourceFileHandler; public final LodRenderer renderer; public ClientRenderState( @@ -307,12 +329,11 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle AbstractSaveStructure saveStructure) { this.clientLevelWrapper = clientLevelWrapper; - this.renderSourceFileHandler = new RenderSourceFileHandler(fullDataSourceProvider, dhClientLevel, saveStructure); this.quadtree = new LodQuadTree(dhClientLevel, Config.Client.Advanced.Graphics.Quality.lodChunkRenderDistanceRadius.get() * LodUtil.CHUNK_WIDTH * 2, // initial position is (0,0) just in case the player hasn't loaded in yet, the tree will be moved once the level starts ticking 0, 0, - this.renderSourceFileHandler.fullDataSourceProvider, this.renderSourceFileHandler); + fullDataSourceProvider); RenderBufferHandler renderBufferHandler = new RenderBufferHandler(this.quadtree); this.renderer = new LodRenderer(renderBufferHandler); @@ -326,7 +347,6 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle this.renderer.close(); this.quadtree.close(); - this.renderSourceFileHandler.close(); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index e2f512aa5..ac69e5126 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -24,7 +24,6 @@ import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; @@ -60,7 +59,6 @@ public class LodQuadTree extends QuadTree implements AutoClose public final int blockRenderDistanceDiameter; private final FullDataSourceProviderV2 fullDataSourceProvider; - private final IRenderSourceProvider renderSourceProvider; /** * This holds every {@link DhSectionPos} that should be reloaded next tick.
@@ -91,14 +89,12 @@ public class LodQuadTree extends QuadTree implements AutoClose public LodQuadTree( IDhClientLevel level, int viewDiameterInBlocks, int initialPlayerBlockX, int initialPlayerBlockZ, - FullDataSourceProviderV2 fullDataSourceProvider, - IRenderSourceProvider renderSourceProvider) + FullDataSourceProviderV2 fullDataSourceProvider) { super(viewDiameterInBlocks, new DhBlockPos2D(initialPlayerBlockX, initialPlayerBlockZ), TREE_LOWEST_DETAIL_LEVEL); this.level = level; this.fullDataSourceProvider = fullDataSourceProvider; - this.renderSourceProvider = renderSourceProvider; this.blockRenderDistanceDiameter = viewDiameterInBlocks; this.horizontalScaleChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, (newHorizontalScale) -> this.onHorizontalQualityChange()); @@ -164,7 +160,7 @@ public class LodQuadTree extends QuadTree implements AutoClose LodRenderSection renderSection = this.getValue(pos); if (renderSection != null) { - renderSection.reload(this.renderSourceProvider); + renderSection.reload(this.fullDataSourceProvider); } } catch (IndexOutOfBoundsException e) @@ -313,7 +309,7 @@ public class LodQuadTree extends QuadTree implements AutoClose { // prepare this section for rendering // TODO this should fire for the lowest detail level first to improve loading speed - renderSection.loadRenderSource(this.renderSourceProvider, this.level); + renderSection.loadRenderSource(this.fullDataSourceProvider, this.level); // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections if (!parentRenderSectionIsEnabled && renderSection.canRenderNow()) @@ -435,10 +431,7 @@ public class LodQuadTree extends QuadTree implements AutoClose { try { - LOGGER.info("Clearing render cache..."); - - // delete the cache first so the nodes won't accidentally try re-loading the old data - this.renderSourceProvider.deleteRenderCache(); + LOGGER.info("Disposing render data..."); // clear the tree Iterator> nodeIterator = this.nodeIterator(); @@ -452,7 +445,7 @@ public class LodQuadTree extends QuadTree implements AutoClose } } - LOGGER.info("Render cache invalidated, please wait a moment for everything to reload..."); + LOGGER.info("Render data cleared, please wait a moment for everything to reload..."); } catch (Exception e) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 9063ec741..e70f20e94 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -20,11 +20,12 @@ package com.seibel.distanthorizons.core.render; import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBufferBuilder; +import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.renderfile.IRenderSourceProvider; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -57,13 +58,13 @@ public class LodRenderSection implements IDebugRenderable private boolean isRenderingEnabled = false; /** - * If this is true, then {@link LodRenderSection#reload(IRenderSourceProvider)} was called while - * a {@link IRenderSourceProvider} was already being loaded. + * If this is true, then {@ link LodRenderSection#reload(IRenderSourceProvider)} was called while + * a {@ link IRenderSourceProvider} was already being loaded. */ private boolean reloadRenderSourceOnceLoaded = false; - private IRenderSourceProvider renderSourceProvider = null; - private CompletableFuture renderSourceLoadFuture; + private FullDataSourceProviderV2 fullDataSourceProvider = null; + private CompletableFuture fullDataSourceLoadFuture; private ColumnRenderSource renderSource; private IDhClientLevel level = null; @@ -120,17 +121,17 @@ public class LodRenderSection implements IDebugRenderable //=============// /** does nothing if a render source is already loaded or in the process of loading */ - public void loadRenderSource(IRenderSourceProvider renderDataProvider, IDhClientLevel level) + public void loadRenderSource(FullDataSourceProviderV2 fullDataSourceProvider, IDhClientLevel level) { - this.renderSourceProvider = renderDataProvider; + this.fullDataSourceProvider = fullDataSourceProvider; this.level = level; - if (this.renderSourceProvider == null) + if (this.fullDataSourceProvider == null) { LOGGER.warn("LodRenderSection [" + this.pos + "] called loadRenderSource with a empty source provider"); return; } // don't re-load or double load the render source - if (this.renderSource != null || this.renderSourceLoadFuture != null) + if (this.renderSource != null || this.fullDataSourceLoadFuture != null) { // since the render source has been loaded, make sure the render buffers are populated // FIXME this is a duck tape solution, since the renderBufferRef should be populated elsewhere, but this does fix empty LODs when moving around the world @@ -145,7 +146,7 @@ public class LodRenderSection implements IDebugRenderable this.startLoadRenderSourceAsync(); } - public void reload(IRenderSourceProvider renderDataProvider) + public void reload(FullDataSourceProviderV2 fullDataSourceProvider) { // debug rendering boolean showRenderSectionStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get(); @@ -160,8 +161,8 @@ public class LodRenderSection implements IDebugRenderable } - this.renderSourceProvider = renderDataProvider; - if (this.renderSourceProvider == null) + this.fullDataSourceProvider = fullDataSourceProvider; + if (this.fullDataSourceProvider == null) { LOGGER.warn("LodRenderSection [" + this.pos + "] called reload with a empty source provider"); return; @@ -173,7 +174,7 @@ public class LodRenderSection implements IDebugRenderable return; } // wait for the current load future to finish before re-loading - if (this.renderSourceLoadFuture != null) + if (this.fullDataSourceLoadFuture != null) { this.reloadRenderSourceOnceLoaded = true; return; @@ -184,19 +185,22 @@ public class LodRenderSection implements IDebugRenderable private void startLoadRenderSourceAsync() { - this.renderSourceLoadFuture = this.renderSourceProvider.getAsync(this.pos); - this.renderSourceLoadFuture.whenComplete((renderSource, ex) -> + this.fullDataSourceLoadFuture = this.fullDataSourceProvider.getAsync(this.pos); + this.fullDataSourceLoadFuture.whenComplete((fullDataSource, ex) -> { - this.renderSource = renderSource; + // this runs on the a file handler thread, so transforming the data + // here shouldn't cause any stutters + // (Although it might be good to have it on a separate thread anyway) + this.renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); this.lastNs = -1; this.markBufferDirty(); if (this.reloadRenderSourceOnceLoaded) { this.reloadRenderSourceOnceLoaded = false; - this.reload(this.renderSourceProvider); + this.reload(this.fullDataSourceProvider); } - this.renderSourceLoadFuture = null; + this.fullDataSourceLoadFuture = null; }); } @@ -213,7 +217,7 @@ public class LodRenderSection implements IDebugRenderable public boolean canRenderNow() { - if (this.renderSourceLoadFuture != null || this.buildRenderBufferFuture != null) + if (this.fullDataSourceLoadFuture != null || this.buildRenderBufferFuture != null) { // wait for loading to finish return false; @@ -278,7 +282,7 @@ public class LodRenderSection implements IDebugRenderable } /** @return true if this section is loaded and set to render */ - public boolean canBuildBuffer() { return this.renderSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); } + public boolean canBuildBuffer() { return this.fullDataSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); } private boolean isBufferOutdated() { return this.neighborUpdated || this.renderSource.localVersion.get() != this.lastSwapLocalVersion; } /** @return true if this section is loaded and set to render */ @@ -296,10 +300,10 @@ public class LodRenderSection implements IDebugRenderable this.disposeActiveBuffer = true; this.renderSource = null; - if (this.renderSourceLoadFuture != null) + if (this.fullDataSourceLoadFuture != null) { - this.renderSourceLoadFuture.cancel(true); - this.renderSourceLoadFuture = null; + this.fullDataSourceLoadFuture.cancel(true); + this.fullDataSourceLoadFuture = null; } } @@ -473,7 +477,7 @@ public class LodRenderSection implements IDebugRenderable return "LodRenderSection{" + "pos=" + this.pos + ", lodRenderSource=" + this.renderSource + - ", loadFuture=" + this.renderSourceLoadFuture + + ", loadFuture=" + this.fullDataSourceLoadFuture + ", isRenderEnabled=" + this.isRenderingEnabled + '}'; } @@ -496,11 +500,11 @@ public class LodRenderSection implements IDebugRenderable public void debugRender(DebugRenderer debugRenderer) { Color color = Color.red; - if (this.renderSourceProvider == null) + if (this.fullDataSourceProvider == null) { color = Color.black; } - else if (this.renderSourceLoadFuture != null) + else if (this.fullDataSourceLoadFuture != null) { color = Color.yellow; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java similarity index 81% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java index e89a3c7c7..6c5d1d7ce 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/LegacyDataSourceDTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV1DTO.java @@ -32,20 +32,12 @@ import java.io.InputStream; import java.util.concurrent.atomic.AtomicLong; /** - * Handles storing {@link ColumnRenderSource}'s and {@link FullDataSourceV1}'s in the database. - * - * @deprecated {@link ColumnRenderSource} should store the actual GPU buffer data, - * at that point this DTO should be just used for {@link FullDataSourceV1} - * and could be renamed to FullDataSourceV1DTO + * Handles storing{@link FullDataSourceV1}'s in the database. */ -@Deprecated -public class LegacyDataSourceDTO implements IBaseDTO +public class FullDataSourceV1DTO implements IBaseDTO { public DhSectionPos pos; public int checksum; - /** @deprecated the database now has a last modified date time that should be used instead */ - @Deprecated - public AtomicLong dataVersion = new AtomicLong(Long.MAX_VALUE); public byte dataDetailLevel; public EDhApiWorldGenerationStep worldGenStep; @@ -62,7 +54,7 @@ public class LegacyDataSourceDTO implements IBaseDTO // constructor // //=============// - public LegacyDataSourceDTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) + public FullDataSourceV1DTO(DhSectionPos pos, int checksum, byte dataDetailLevel, EDhApiWorldGenerationStep worldGenStep, String dataType, byte binaryDataFormatVersion, byte[] dataArray) { this.pos = pos; this.checksum = checksum; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java similarity index 64% rename from core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java rename to core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index 32d67d19b..521163695 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractLegacyDataSourceRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -21,34 +21,55 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.sql.dto.LegacyDataSourceDTO; +import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo +public class FullDataSourceV1Repo extends AbstractDhRepo { - public AbstractLegacyDataSourceRepo(String databaseType, String databaseLocation) throws SQLException + public static final String TABLE_NAME = "Legacy_FullData_V1"; + + + + //=============// + // constructor // + //=============// + + public FullDataSourceV1Repo(String databaseType, String databaseLocation) throws SQLException { - super(databaseType, databaseLocation, LegacyDataSourceDTO.class); + super(databaseType, databaseLocation, FullDataSourceV1DTO.class); } + //===========// + // overrides // + //===========// + + @Override + public String getTableName() { return TABLE_NAME; } + + @Override + public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } + + + //=======================// // repo required methods // //=======================// @Override - public LegacyDataSourceDTO convertDictionaryToDto(Map objectMap) throws ClassCastException + public FullDataSourceV1DTO convertDictionaryToDto(Map objectMap) throws ClassCastException { String posString = (String) objectMap.get("DhSectionPos"); DhSectionPos pos = DhSectionPos.deserialize(posString); // meta data int checksum = (Integer) objectMap.get("Checksum"); - long dataVersion = (Long) objectMap.get("DataVersion"); byte dataDetailLevel = (Byte) objectMap.get("DataDetailLevel"); String worldGenStepString = (String) objectMap.get("WorldGenStep"); EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromName(worldGenStepString); @@ -59,7 +80,7 @@ public abstract class AbstractLegacyDataSourceRepo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( + "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); + + if (resultMap == null) + { + return 0; + } + else + { + int count = (int) resultMap.get("itemCount"); + return count; + } + } + + /** Returns the new "returnCount" positions that need to be migrated */ + public ArrayList getPositionsToMigrate(int returnCount) + { + ArrayList list = new ArrayList<>(); + + List> resultMapList = this.queryDictionary( + "select DhSectionPos " + + "from "+this.getTableName()+" " + + "WHERE MigrationFailed <> 1 " + + "LIMIT "+returnCount+";"); + + for (Map resultMap : resultMapList) + { + // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] + DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos")); + list.add(sectionPos); + } + + return list; + } + + public void markMigrationFailed(DhSectionPos pos) + { + String sql = + "UPDATE "+this.getTableName()+" \n" + + "SET MigrationFailed = 1 \n" + + "WHERE DhSectionPos = '"+pos.serialize()+"'"; + + this.queryDictionaryFirst(sql); + } + + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java deleted file mode 100644 index 2d191da4e..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/LegacyFullDataRepo.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.sql.repo; - -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class LegacyFullDataRepo extends AbstractLegacyDataSourceRepo -{ - public static final String TABLE_NAME = "Legacy_FullData_V1"; - - - public LegacyFullDataRepo(String databaseType, String databaseLocation) throws SQLException - { - super(databaseType, databaseLocation); - } - - - @Override - public String getTableName() { return TABLE_NAME; } - - @Override - public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } - - - - //===========// - // migration // - //===========// - - /** Returns how many positions need to be migrated over to the new version */ - public int getMigrationCount() - { - Map resultMap = this.queryDictionaryFirst( - "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); - - if (resultMap == null) - { - return 0; - } - else - { - int count = (int) resultMap.get("itemCount"); - return count; - } - } - - /** Returns the new "returnCount" positions that need to be migrated */ - public ArrayList getPositionsToMigrate(int returnCount) - { - ArrayList list = new ArrayList<>(); - - List> resultMapList = this.queryDictionary( - "select DhSectionPos " + - "from "+this.getTableName()+" " + - "WHERE MigrationFailed <> 1 " + - "LIMIT "+returnCount+";"); - - for (Map resultMap : resultMapList) - { - // returned in the format [sectionDetailLevel,x,z] IE [6,0,0] - DhSectionPos sectionPos = DhSectionPos.deserialize((String) resultMap.get("DhSectionPos")); - list.add(sectionPos); - } - - return list; - } - - public void markMigrationFailed(DhSectionPos pos) - { - String sql = - "UPDATE "+this.getTableName()+" \n" + - "SET MigrationFailed = 1 \n" + - "WHERE DhSectionPos = '"+pos.serialize()+"'"; - - this.queryDictionaryFirst(sql); - } - - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java deleted file mode 100644 index fa43e81ea..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/RenderDataRepo.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.sql.repo; - -import com.seibel.distanthorizons.core.pos.DhSectionPos; - -import java.sql.SQLException; - -public class RenderDataRepo extends AbstractLegacyDataSourceRepo -{ - public static final String TABLE_NAME = "DhRenderData"; - - - public RenderDataRepo(String databaseType, String databaseLocation) throws SQLException - { - super(databaseType, databaseLocation); - } - - - @Override - public String getTableName() { return TABLE_NAME; } - - @Override - public String createWhereStatement(DhSectionPos pos) { return "DhSectionPos = '"+pos.serialize()+"'"; } - -} From 661f286b77228fb051d07edd93abb904f4501795 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 23 Mar 2024 18:08:35 -0500 Subject: [PATCH 093/183] Fix accidentally marking DTOs to update past the max parent detail level --- .../dataObjects/fullData/sources/FullDataSourceV2.java | 8 +++----- .../core/file/AbstractNewDataSourceHandler.java | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 5a5dff219..0d4a35e35 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; +import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -278,11 +279,8 @@ public class FullDataSourceV2 implements IDataSource throw new UnsupportedOperationException("Unsupported data source update. Expected input detail level of ["+thisDetailLevel+"] or ["+(thisDetailLevel+1)+"], received detail level ["+inputDetailLevel+"]."); } - if (dataChanged && this.pos.getDetailLevel() < FullDataSourceProviderV2.TOP_SECTION_DETAIL_LEVEL) - { - // mark that this data source should be applied to its parent - this.applyToParent = true; - } + // determine if this data source should be applied to its parent + this.applyToParent = (dataChanged && this.pos.getDetailLevel() < AbstractNewDataSourceHandler.TOP_SECTION_DETAIL_LEVEL); if (dataChanged) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 048c88ecc..542723e80 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -20,6 +20,8 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; +// TODO is there a reason this is separate from FullDataSourceProviderV2? +// We shouldn't need multiple data source handlers public abstract class AbstractNewDataSourceHandler , TDTO extends IBaseDTO, From ed9cf526cdcfd56263281545485681b46d090951 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 24 Mar 2024 11:38:30 -0500 Subject: [PATCH 094/183] rename example unit tests to prevent naming issues --- .../test/java/tests/{ExampleTest.java => ExampleApiTest.java} | 2 +- .../test/java/tests/{ExampleTest.java => ExampleCoreTest.java} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename api/src/test/java/tests/{ExampleTest.java => ExampleApiTest.java} (97%) rename core/src/test/java/tests/{ExampleTest.java => ExampleCoreTest.java} (97%) diff --git a/api/src/test/java/tests/ExampleTest.java b/api/src/test/java/tests/ExampleApiTest.java similarity index 97% rename from api/src/test/java/tests/ExampleTest.java rename to api/src/test/java/tests/ExampleApiTest.java index 63e1ecf89..ebf7c52a1 100644 --- a/api/src/test/java/tests/ExampleTest.java +++ b/api/src/test/java/tests/ExampleApiTest.java @@ -28,7 +28,7 @@ import org.junit.Test; * @author James Seibel * @version 2022-9-5 */ -public class ExampleTest +public class ExampleApiTest { @Test diff --git a/core/src/test/java/tests/ExampleTest.java b/core/src/test/java/tests/ExampleCoreTest.java similarity index 97% rename from core/src/test/java/tests/ExampleTest.java rename to core/src/test/java/tests/ExampleCoreTest.java index 63e1ecf89..8e51284d0 100644 --- a/core/src/test/java/tests/ExampleTest.java +++ b/core/src/test/java/tests/ExampleCoreTest.java @@ -28,7 +28,7 @@ import org.junit.Test; * @author James Seibel * @version 2022-9-5 */ -public class ExampleTest +public class ExampleCoreTest { @Test From 63b636512876711904ac59e3a6f9ce3d83b136b5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 24 Mar 2024 11:50:45 -0500 Subject: [PATCH 095/183] Add swing headless setup to Initializer --- .../com/seibel/distanthorizons/core/Initializer.java | 11 +++++++++++ .../core/config/gui/JavaScreenHandlerScreen.java | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index 0684e4cc3..4aa4fe286 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -33,6 +33,7 @@ import net.jpountz.lz4.LZ4Compressor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.awt.*; import java.io.InputStream; /** Handles first time Core setup. */ @@ -73,6 +74,16 @@ public class Initializer } + // attempt to setup Swing so we can display dialogs (popup windows) + System.setProperty("java.awt.headless", "false"); + if (GraphicsEnvironment.isHeadless()) + { + LOGGER.warn("Java.awt.headless is false. This means Distant Horizons can't display error and info dialog windows."); + } + else + { + LOGGER.info("Java.awt.headless set to true. Distant Horizons can correctly display error and info dialog windows."); + } // link Core's config to the API DhApi.Delayed.configs = DhApiConfig.INSTANCE; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/gui/JavaScreenHandlerScreen.java b/core/src/main/java/com/seibel/distanthorizons/core/config/gui/JavaScreenHandlerScreen.java index 48c6b2077..06c626c1f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/gui/JavaScreenHandlerScreen.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/gui/JavaScreenHandlerScreen.java @@ -37,7 +37,8 @@ public class JavaScreenHandlerScreen extends AbstractScreen static { - // Required to run this + // Needs to be called before any Swing code is called, otherwise + // Swing will get stuck thinking it's headless System.setProperty("java.awt.headless", "false"); } From bbe5ae9b7c6dd1ec29ad17b2e0a92563e8fd132e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 24 Mar 2024 11:52:32 -0500 Subject: [PATCH 096/183] Fix potential issues when java.awt.headless is true --- .../distanthorizons/core/config/Config.java | 20 ++++++++++--- .../core/jar/updater/SelfUpdater.java | 29 +++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index b9c1a0e24..180ab4aeb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -37,7 +37,9 @@ import com.seibel.distanthorizons.coreapi.util.StringUtil; import org.apache.logging.log4j.Logger; import javax.swing.*; +import java.awt.*; import java.util.*; +import java.util.List; /** @@ -1423,10 +1425,20 @@ public class Config .set(new HashMap()) .build(); - public static ConfigUIButton uiButtonTest = new ConfigUIButton(() -> { new Thread(() -> { - System.setProperty("java.awt.headless", "false"); // Required to make it work - JOptionPane.showMessageDialog(null, "Button pressed!", "UITester dialog", JOptionPane.INFORMATION_MESSAGE); - });}); + public static ConfigUIButton uiButtonTest = new ConfigUIButton(() -> + { + new Thread(() -> + { + if (!GraphicsEnvironment.isHeadless()) + { + JOptionPane.showMessageDialog(null, "Button pressed!", "UITester dialog", JOptionPane.INFORMATION_MESSAGE); + } + else + { + LOGGER.info("button pressed!"); + } + }); + }); public static ConfigCategory categoryTest = new ConfigCategory.Builder().set(CategoryTest.class).build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java index 1bd11960a..16df88e2b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/jar/updater/SelfUpdater.java @@ -33,6 +33,7 @@ import com.seibel.distanthorizons.coreapi.util.jar.DeleteOnUnlock; import org.apache.logging.log4j.Logger; import javax.swing.*; +import java.awt.*; import java.io.*; import java.net.URLEncoder; import java.nio.file.Files; @@ -222,9 +223,17 @@ public class SelfUpdater deleteOldJarOnJvmShutdown = true; LOGGER.info(ModInfo.READABLE_NAME + " successfully updated. It will apply on game's relaunch"); - new Thread(() -> { - System.setProperty("java.awt.headless", "false"); // Required to make it work - JOptionPane.showMessageDialog(null, ModInfo.READABLE_NAME + " updated, this will be applied on game restart.", ModInfo.READABLE_NAME, JOptionPane.INFORMATION_MESSAGE); + new Thread(() -> + { + String message = ModInfo.READABLE_NAME + " updated, this will be applied on game restart."; + if (!GraphicsEnvironment.isHeadless()) + { + JOptionPane.showMessageDialog(null, message, ModInfo.READABLE_NAME, JOptionPane.INFORMATION_MESSAGE); + } + else + { + LOGGER.info(message); + } }).start(); return true; } @@ -270,9 +279,17 @@ public class SelfUpdater deleteOldJarOnJvmShutdown = true; LOGGER.info(ModInfo.READABLE_NAME + " successfully updated. It will apply on game's relaunch"); - new Thread(() -> { - System.setProperty("java.awt.headless", "false"); // Required to make it work - JOptionPane.showMessageDialog(null, ModInfo.READABLE_NAME + " updated, this will be applied on game restart.", ModInfo.READABLE_NAME, JOptionPane.INFORMATION_MESSAGE); + new Thread(() -> + { + String message = ModInfo.READABLE_NAME + " updated, this will be applied on game restart."; + if (!GraphicsEnvironment.isHeadless()) + { + JOptionPane.showMessageDialog(null, message, ModInfo.READABLE_NAME, JOptionPane.INFORMATION_MESSAGE); + } + else + { + LOGGER.info(message); + } }).start(); zis.close(); From 74627fdf902d06763f56db4a006789cc6814daf8 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 24 Mar 2024 12:00:51 -0500 Subject: [PATCH 097/183] Use automatica Zstd buffer pooling --- .../core/util/objects/dataStreams/DhDataInputStream.java | 3 ++- .../core/util/objects/dataStreams/DhDataOutputStream.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java index c78c3a486..1c0fb0114 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataInputStream.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; +import com.github.luben.zstd.RecyclingBufferPool; import com.github.luben.zstd.ZstdInputStream; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import net.jpountz.lz4.LZ4FrameInputStream; @@ -51,7 +52,7 @@ public class DhDataInputStream extends DataInputStream case LZ4: return new LZ4FrameInputStream(stream); case Z_STD: - return new ZstdInputStream(stream); + return new ZstdInputStream(stream, RecyclingBufferPool.INSTANCE); case LZMA2: // Note: all LZMA/XZ compressors can be decompressed using this same InputStream return new XZInputStream(stream); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java index ac184e7fd..aab1dbd0a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; +import com.github.luben.zstd.RecyclingBufferPool; import com.github.luben.zstd.ZstdOutputStream; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; import net.jpountz.lz4.LZ4FrameOutputStream; @@ -47,7 +48,7 @@ public class DhDataOutputStream extends DataOutputStream case LZ4: return new LZ4FrameOutputStream(stream); case Z_STD: - return new ZstdOutputStream(stream); + return new ZstdOutputStream(stream, RecyclingBufferPool.INSTANCE); case LZMA2: // in James' testing preset 4 has the best balance between compression ratio and speed // 5 is slightly more compressed 0.128 vs 0.139, but is roughly 60% slower From b3a20fb938f3488b99ee0b4586592c75fc48c8f0 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 24 Mar 2024 12:17:48 -0500 Subject: [PATCH 098/183] Use native bindings for LZ4 output compression to reduce GC load --- .../util/objects/dataStreams/DhDataOutputStream.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java index aab1dbd0a..fe5665024 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java @@ -22,7 +22,9 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; import com.github.luben.zstd.RecyclingBufferPool; import com.github.luben.zstd.ZstdOutputStream; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FrameOutputStream; +import net.jpountz.xxhash.XXHashFactory; import org.tukaani.xz.LZMA2Options; import org.tukaani.xz.XZOutputStream; @@ -46,7 +48,12 @@ public class DhDataOutputStream extends DataOutputStream case UNCOMPRESSED: return stream; case LZ4: - return new LZ4FrameOutputStream(stream); + return new LZ4FrameOutputStream(stream, + LZ4FrameOutputStream.BLOCKSIZE.SIZE_64KB, -1L, + // using native instances reduces GC pressures + LZ4Factory.nativeInstance().fastCompressor(), + XXHashFactory.nativeInstance().hash32(), + LZ4FrameOutputStream.FLG.Bits.BLOCK_INDEPENDENCE); case Z_STD: return new ZstdOutputStream(stream, RecyclingBufferPool.INSTANCE); case LZMA2: From 524937719933e5b1c0002d3ae7e908e30a8bc380 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 24 Mar 2024 15:11:42 -0500 Subject: [PATCH 099/183] Add data caching to Lzma data compression --- .../dataStreams/DhDataOutputStream.java | 21 ++- .../objects/dataStreams/LzmaArrayCache.java | 135 ++++++++++++++++++ 2 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java index fe5665024..3337cb26f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/DhDataOutputStream.java @@ -22,11 +22,12 @@ package com.seibel.distanthorizons.core.util.objects.dataStreams; import com.github.luben.zstd.RecyclingBufferPool; import com.github.luben.zstd.ZstdOutputStream; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FrameOutputStream; import net.jpountz.xxhash.XXHashFactory; -import org.tukaani.xz.LZMA2Options; -import org.tukaani.xz.XZOutputStream; +import org.apache.logging.log4j.Logger; +import org.tukaani.xz.*; import java.io.*; @@ -37,6 +38,10 @@ import java.io.*; */ public class DhDataOutputStream extends DataOutputStream { + private static final ThreadLocal LZMA_RESETTABLE_ARRAY_CACHE_GETTER = ThreadLocal.withInitial(() -> new ResettableArrayCache(new LzmaArrayCache())); + + + public DhDataOutputStream(OutputStream stream, EDhApiDataCompressionMode compressionMode) throws IOException { super(warpStream(new BufferedOutputStream(stream), compressionMode)); @@ -50,16 +55,20 @@ public class DhDataOutputStream extends DataOutputStream case LZ4: return new LZ4FrameOutputStream(stream, LZ4FrameOutputStream.BLOCKSIZE.SIZE_64KB, -1L, - // using native instances reduces GC pressures + // using native instances reduce GC pressure LZ4Factory.nativeInstance().fastCompressor(), XXHashFactory.nativeInstance().hash32(), LZ4FrameOutputStream.FLG.Bits.BLOCK_INDEPENDENCE); case Z_STD: return new ZstdOutputStream(stream, RecyclingBufferPool.INSTANCE); case LZMA2: - // in James' testing preset 4 has the best balance between compression ratio and speed - // 5 is slightly more compressed 0.128 vs 0.139, but is roughly 60% slower - return new XZOutputStream(stream, new LZMA2Options(4)); + // using an array cache significantly reduces GC pressure + ResettableArrayCache arrayCache = LZMA_RESETTABLE_ARRAY_CACHE_GETTER.get(); + arrayCache.reset(); + // Note: if the LZMA2Options are changed the array cache may need to be re-tested. + // the array cache was specifically tested and tuned for LZMA preset 4 + return new XZOutputStream(stream, new LZMA2Options(4), + XZ.CHECK_CRC64, arrayCache); default: throw new IllegalArgumentException("No compressor defined for ["+compressionMode+"]"); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java new file mode 100644 index 000000000..e932e9936 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/dataStreams/LzmaArrayCache.java @@ -0,0 +1,135 @@ +package com.seibel.distanthorizons.core.util.objects.dataStreams; + +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import it.unimi.dsi.fastutil.ints.Int2ReferenceArrayMap; +import it.unimi.dsi.fastutil.ints.IntUnaryOperator; +import org.apache.logging.log4j.Logger; +import org.tukaani.xz.ArrayCache; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * LZMA requires a custom object to cache it's backend arrays. + * + * TODO there's a lot of duplicate code in this class since it has logic for both + * int[]'s and byte[]'s. + */ +public class LzmaArrayCache extends ArrayCache +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + /** + * In James' testing the byte and int caches only ever had to store 2 and 4 arrays respectively. + * With the in mind we could take a few shortcuts, but if that changes then we need to be + * notified as it might cause issues with the current logic. + */ + public static final int WARN_CACHE_LENGTH_EXCEEDED = 10; + + public static final AtomicInteger MAX_INT_CACHE_LENGTH_REF = new AtomicInteger(WARN_CACHE_LENGTH_EXCEEDED); + public static final AtomicInteger MAX_BYTE_CACHE_LENGTH_REF = new AtomicInteger(WARN_CACHE_LENGTH_EXCEEDED); + + + public final IntUnaryOperator maxByteCacheSizeUnaryOperator = (x) -> Math.max(this.byteCache.size(), x); + public final IntUnaryOperator maxIntCacheSizeUnaryOperator = (x) -> Math.max(this.intCache.size(), x); + + /** + * generally only 2 items long
+ * {@link Int2ReferenceArrayMap} can be used since the cache should only be a few items long. + * If the array ends up being longer then this design will need to be changed. + */ + public final Int2ReferenceArrayMap> byteCache = new Int2ReferenceArrayMap<>(); + /** generally only 4 items long */ + public final Int2ReferenceArrayMap> intCache = new Int2ReferenceArrayMap<>(); + + + + //=============// + // byte arrays // + //=============// + + @Override + public byte[] getByteArray(int size, boolean fillWithZeros) + { + ArrayList cacheList = this.byteCache.computeIfAbsent(size, (newSize) -> new ArrayList<>(4)); + if (cacheList.size() == 0) + { + return new byte[size]; + } + + byte[] array = cacheList.remove(cacheList.size()-1); + if (array == null) + { + return new byte[size]; + } + // the array needs to be cleared to prevent accidentally sending dirty data + Arrays.fill(array, (byte)0); + return array; + } + + @Override + public void putArray(byte[] array) + { + int size = array.length; + this.byteCache.computeIfAbsent(size, (newSize) -> new ArrayList<>()); + this.byteCache.get(size).add(array); + + + if (this.byteCache.size() > WARN_CACHE_LENGTH_EXCEEDED) + { + int previousMax = MAX_BYTE_CACHE_LENGTH_REF.getAndUpdate(this.maxByteCacheSizeUnaryOperator); + int newMax = MAX_BYTE_CACHE_LENGTH_REF.get(); + if (newMax > previousMax) + { + LOGGER.warn("LZMA byte array cache expected size exceeded. Expected max length ["+WARN_CACHE_LENGTH_EXCEEDED+"], actual length ["+this.byteCache.size()+"]."); + } + } + } + + + + //============// + // int arrays // + //============// + + @Override + public int[] getIntArray(int size, boolean fillWithZeros) + { + ArrayList cacheList = this.intCache.computeIfAbsent(size, (newSize) -> new ArrayList<>(4)); + if (cacheList.size() == 0) + { + return new int[size]; + } + + int[] array = cacheList.remove(cacheList.size()-1); + if (array == null) + { + return new int[size]; + } + // the array needs to be cleared to prevent accidentally sending dirty data + Arrays.fill(array, (byte)0); + return array; + } + + @Override + public void putArray(int[] array) + { + int size = array.length; + this.intCache.computeIfAbsent(size, (newSize) -> new ArrayList<>()); + this.intCache.get(size).add(array); + + + if (this.intCache.size() > WARN_CACHE_LENGTH_EXCEEDED) + { + int previousMax = MAX_INT_CACHE_LENGTH_REF.getAndUpdate(this.maxIntCacheSizeUnaryOperator); + int newMax = MAX_INT_CACHE_LENGTH_REF.get(); + if (newMax > previousMax) + { + LOGGER.warn("LZMA int array cache expected size exceeded. Expected max length ["+WARN_CACHE_LENGTH_EXCEEDED+"], actual length ["+this.intCache.size()+"]."); + } + } + } + + +} From 5722a92dc15f5b37115e45573b9196b345e12382 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 27 Mar 2024 15:54:24 -0600 Subject: [PATCH 100/183] Add broken lossy chunk compression --- .../transformers/LodDataBuilder.java | 81 +++++++++++++++---- .../block/IBlockStateWrapper.java | 3 + 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 601281505..372692f29 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -26,7 +26,9 @@ import com.seibel.distanthorizons.api.objects.data.DhApiChunk; import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; @@ -127,13 +129,13 @@ public class LodDataBuilder //==========================// int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); - for (int chunkX = 0; chunkX < LodUtil.CHUNK_WIDTH; chunkX++) + for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) { - for (int chunkZ = 0; chunkZ < LodUtil.CHUNK_WIDTH; chunkZ++) + for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++) { LongArrayList longs = new LongArrayList(chunkWrapper.getHeight() / 4); int lastY = chunkWrapper.getMaxBuildHeight(); - IBiomeWrapper biome = chunkWrapper.getBiome(chunkX, lastY, chunkZ); + IBiomeWrapper biome = chunkWrapper.getBiome(relBlockX, lastY, relBlockZ); IBlockStateWrapper blockState = AIR; int mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); @@ -143,8 +145,8 @@ public class LodDataBuilder if (lastY < chunkWrapper.getMaxBuildHeight()) { // FIXME: The lastY +1 offset is to reproduce the old behavior. Remove this when we get per-face lighting - blockLight = (byte) chunkWrapper.getBlockLight(chunkX, lastY + 1, chunkZ); - skyLight = (byte) chunkWrapper.getSkyLight(chunkX, lastY + 1, chunkZ); + blockLight = (byte) chunkWrapper.getBlockLight(relBlockX, lastY + 1, relBlockZ); + skyLight = (byte) chunkWrapper.getSkyLight(relBlockX, lastY + 1, relBlockZ); } else { @@ -155,9 +157,9 @@ public class LodDataBuilder // determine the starting Y Pos - int y = chunkWrapper.getLightBlockingHeightMapValue(chunkX,chunkZ); + int y = chunkWrapper.getLightBlockingHeightMapValue(relBlockX,relBlockZ); // go up until we reach open air or the world limit - IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); + IBlockStateWrapper topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); while (!topBlockState.isAir() && y < chunkWrapper.getMaxBuildHeight()) { try @@ -165,13 +167,13 @@ public class LodDataBuilder // This is necessary in some edge cases with snow layers and some other blocks that may not appear in the height map but do block light. // Interestingly this doesn't appear to be the case in the DhLightingEngine, if this same logic is added there the lighting breaks for the affected blocks. y++; - topBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); + topBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); } catch (Exception e) { if (!getTopErrorLogged) { - LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + chunkX + "," + y + "," + chunkZ + "] error: " + e.getMessage(), e); + LOGGER.warn("Unexpected issue in LodDataBuilder, future errors won't be logged. Chunk [" + chunkWrapper.getChunkPos() + "] with max height: [" + chunkWrapper.getMaxBuildHeight() + "] had issue getting block at pos [" + relBlockX + "," + y + "," + relBlockZ + "] error: " + e.getMessage(), e); getTopErrorLogged = true; } @@ -183,12 +185,15 @@ public class LodDataBuilder for (; y >= minBuildHeight; y--) { - IBiomeWrapper newBiome = chunkWrapper.getBiome(chunkX, y, chunkZ); - IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(chunkX, y, chunkZ); - byte newBlockLight = (byte) chunkWrapper.getBlockLight(chunkX, y + 1, chunkZ); - byte newSkyLight = (byte) chunkWrapper.getSkyLight(chunkX, y + 1, chunkZ); + IBiomeWrapper newBiome = chunkWrapper.getBiome(relBlockX, y, relBlockZ); + IBlockStateWrapper newBlockState = chunkWrapper.getBlockState(relBlockX, y, relBlockZ); + byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ); + byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ); - if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) + if ( + (!newBiome.equals(biome) || !newBlockState.equals(blockState)) + && !blockState.isAir() && !blockVisible(chunkWrapper, relBlockX, y, relBlockZ) + ) { longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); biome = newBiome; @@ -202,8 +207,8 @@ public class LodDataBuilder longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); dataSource.setSingleColumn(longs, - chunkX + chunkOffsetX, - chunkZ + chunkOffsetZ, + relBlockX + chunkOffsetX, + relBlockZ + chunkOffsetZ, EDhApiWorldGenerationStep.LIGHT); } } @@ -211,6 +216,50 @@ public class LodDataBuilder LodUtil.assertTrue(!dataSource.isEmpty); return dataSource; } + private static boolean blockVisible(IChunkWrapper chunkWrapper, int relBlockX, int blockY, int relBlockZ) + { + DhBlockPos originalBlockPos = new DhBlockPos(relBlockX,blockY,relBlockZ); + DhBlockPos testBlockPos = new DhBlockPos(relBlockX,blockY,relBlockZ); + + // up/down + if (blockInDirectionVisible(chunkWrapper, EDhDirection.UP, originalBlockPos, testBlockPos)) + { + return true; + } + if (blockInDirectionVisible(chunkWrapper, EDhDirection.DOWN, originalBlockPos, testBlockPos)) + { + return true; + } + + // north/south + if (blockInDirectionVisible(chunkWrapper, EDhDirection.NORTH, originalBlockPos, testBlockPos)) + { + return true; + } + if (blockInDirectionVisible(chunkWrapper, EDhDirection.SOUTH, originalBlockPos, testBlockPos)) + { + return true; + } + + // east/west + if (blockInDirectionVisible(chunkWrapper, EDhDirection.EAST, originalBlockPos, testBlockPos)) + { + return true; + } + if (blockInDirectionVisible(chunkWrapper, EDhDirection.WEST, originalBlockPos, testBlockPos)) + { + return true; + } + + + return false; + } + private static boolean blockInDirectionVisible(IChunkWrapper chunkWrapper, EDhDirection direction, DhBlockPos originalBlockPos, DhBlockPos testBlockPos) + { + originalBlockPos.mutateOffset(direction, testBlockPos); + IBlockStateWrapper blockState = chunkWrapper.getBlockState(testBlockPos); + return blockState.isAir() || blockState.getOpacity() != IBlockStateWrapper.FULLY_OPAQUE; + } /** @throws ClassCastException if an API user returns the wrong object type(s) */ diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java index 85bcd10cd..f6f7f1dce 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java @@ -67,6 +67,9 @@ public interface IBlockStateWrapper extends IDhApiBlockStateWrapper /** * Returning a value of 0 means the block is completely transparent. Date: Wed, 27 Mar 2024 19:09:09 -0500 Subject: [PATCH 101/183] Add world compression config --- .../config/EDhApiWorldCompressionMode.java | 67 +++++++++++++++++++ .../distanthorizons/core/config/Config.java | 16 +++++ .../transformers/LodDataBuilder.java | 50 +++++++++++--- .../assets/distanthorizons/lang/en_us.json | 9 +++ 4 files changed, 131 insertions(+), 11 deletions(-) create mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java new file mode 100644 index 000000000..2d9534787 --- /dev/null +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java @@ -0,0 +1,67 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.api.enums.config; + +/** + * UNCOMPRESSED
+ * VISUALLY_EQUAL

+ * + * @version 2024-3-27 + * @since API 1.1.0 + */ +public enum EDhApiWorldCompressionMode +{ + // Reminder: + // when adding items up the API minor version + // when removing items up the API major version + + /** Every block/biome change is recorded in the database. */ + UNCOMPRESSED(0), + + /** + * Only visible block/biome changes are recorded in the database.
+ * Hidden blocks (IE ores) are ignored. + */ + VISUALLY_EQUAL(1); + + + + /** More stable than using the ordinal of the enum */ + public final byte value; + + EDhApiWorldCompressionMode(int value) { this.value = (byte) value; } + + + public static EDhApiWorldCompressionMode getFromValue(byte value) + { + EDhApiWorldCompressionMode[] enumList = EDhApiWorldCompressionMode.values(); + for (int i = 0; i < enumList.length; i++) + { + if (enumList[i].value == value) + { + return enumList[i]; + } + } + + throw new IllegalArgumentException("No lossy compression mode with the value ["+value+"]"); + } + + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 180ab4aeb..046a99677 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -822,6 +822,22 @@ public class Config + "") .build(); + public static ConfigEntry worldCompression = new ConfigEntry.Builder() + .set(EDhApiWorldCompressionMode.VISUALLY_EQUAL) + .comment("" + //+ "What algorithm should be used to compress new LOD data? \n" + //+ "This setting will only affect new or updated LOD data, \n" + //+ "any data already generated when this setting is changed will be\n" + //+ "unaffected until it needs to be re-written to the database.\n" + //+ "\n" + //+ EDhApiDataCompressionMode.UNCOMPRESSED + " \n" + //+ "Should only be used for testing, is worse in every way vs ["+EDhApiDataCompressionMode.LZ4+"].\n" + //+ "Expected Compression Ratio: 1.0\n" + //+ "Estimated average DTO read speed: 1.64 milliseconds\n" + //+ "Estimated average DTO write speed: 12.44 milliseconds\n" + + "") + .build(); + } public static class Multiplayer diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 372692f29..430b85495 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -21,9 +21,11 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; import java.util.List; +import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.api.objects.data.DhApiChunk; import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; @@ -128,6 +130,9 @@ public class LodDataBuilder // populate the data source // //==========================// + EDhApiWorldCompressionMode worldCompressionMode = Config.Client.Advanced.LodBuilding.worldCompression.get(); + boolean ignoreHiddenBlocks = (worldCompressionMode != EDhApiWorldCompressionMode.UNCOMPRESSED); + int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) { @@ -190,18 +195,25 @@ public class LodDataBuilder byte newBlockLight = (byte) chunkWrapper.getBlockLight(relBlockX, y + 1, relBlockZ); byte newSkyLight = (byte) chunkWrapper.getSkyLight(relBlockX, y + 1, relBlockZ); - if ( - (!newBiome.equals(biome) || !newBlockState.equals(blockState)) - && !blockState.isAir() && !blockVisible(chunkWrapper, relBlockX, y, relBlockZ) - ) + // save the biome/block change + if (!newBiome.equals(biome) || !newBlockState.equals(blockState)) { - longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); - biome = newBiome; - blockState = newBlockState; - mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); - blockLight = newBlockLight; - skyLight = newSkyLight; - lastY = y; + // if we ignore hidden blocks, don't save this biome/block change + // wait until the block is visible and then save the new datapoint + if (!ignoreHiddenBlocks + // if the last block is air, this block will always be visible + || blockState.isAir() + // check if this block is visible from any direction + || blockVisible(chunkWrapper, relBlockX, y, relBlockZ)) + { + longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + biome = newBiome; + blockState = newBlockState; + mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); + blockLight = newBlockLight; + skyLight = newSkyLight; + lastY = y; + } } } longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); @@ -257,6 +269,22 @@ public class LodDataBuilder private static boolean blockInDirectionVisible(IChunkWrapper chunkWrapper, EDhDirection direction, DhBlockPos originalBlockPos, DhBlockPos testBlockPos) { originalBlockPos.mutateOffset(direction, testBlockPos); + + // if the block is next to the border of a chunk, assume it's visible + if (testBlockPos.x < 0 || testBlockPos.x >= LodUtil.CHUNK_WIDTH) + { + return true; + } + if (testBlockPos.z < 0 || testBlockPos.z >= LodUtil.CHUNK_WIDTH) + { + return true; + } + if (testBlockPos.y < chunkWrapper.getMinBuildHeight() || testBlockPos.y > chunkWrapper.getMaxBuildHeight()) + { + return true; + } + + // this block isn't on a chunk boundary, check if it is next to a transparent/air block IBlockStateWrapper blockState = chunkWrapper.getBlockState(testBlockPos); return blockState.isAir() || blockState.getOpacity() != IBlockStateWrapper.FULLY_OPAQUE; } diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 81caf9e39..4ecf33813 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -348,6 +348,10 @@ "Data Compression", "distanthorizons.config.client.advanced.lodBuilding.dataCompression.@tooltip": "What algorithm should be used to compress new LOD data? \nThis setting will only affect new or updated LOD data, \nany data already generated when this setting is changed will be \nunaffected until it needs to be re-written to the database. \n\nFastest: LZ4 \nHighest Compression: LZMA2", + "distanthorizons.config.client.advanced.lodBuilding.worldCompression": + "Lossy World Compression", + "distanthorizons.config.client.advanced.lodBuilding.worldCompression.@tooltip": + "TODO", "distanthorizons.config.client.advanced.multiplayer": @@ -774,6 +778,11 @@ "distanthorizons.config.enum.EDhApiDataCompressionMode.LZMA2": "LZMA2", + "distanthorizons.config.enum.EDhApiWorldCompressionMode.UNCOMPRESSED": + "Uncompressed", + "distanthorizons.config.enum.EDhApiWorldCompressionMode.VISUALLY_EQUAL": + "Visually Equal", + "distanthorizons.config.enum.ELightGenerationMode.DISTANT_HORIZONS": "Distant Horizons", "distanthorizons.config.enum.ELightGenerationMode.MINECRAFT": From d41af88494eace5bc6403d16e54458f0ac1ce70f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 28 Mar 2024 07:36:02 -0500 Subject: [PATCH 102/183] Add ColumnWorldCompressionMode to the database --- .../config/EDhApiWorldCompressionMode.java | 9 ++-- .../fullData/sources/FullDataSourceV2.java | 49 +++++++++++++++++-- .../core/sql/dto/FullDataSourceV2DTO.java | 36 ++++++++++++-- .../core/sql/repo/FullDataSourceV2Repo.java | 10 ++-- .../assets/distanthorizons/lang/en_us.json | 14 +++--- ...20-sqlite-createFullDataSourceV2Tables.sql | 1 + 6 files changed, 99 insertions(+), 20 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java index 2d9534787..d8abc9eb0 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiWorldCompressionMode.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.api.enums.config; /** - * UNCOMPRESSED
+ * MERGE_SAME_BLOCKS
* VISUALLY_EQUAL

* * @version 2024-3-27 @@ -32,8 +32,11 @@ public enum EDhApiWorldCompressionMode // when adding items up the API minor version // when removing items up the API major version - /** Every block/biome change is recorded in the database. */ - UNCOMPRESSED(0), + /** + * Every block/biome change is recorded in the database.
+ * This is what DH 2.0 and 2.0.1 all used by default and will store a lot of data. + */ + MERGE_SAME_BLOCKS(0), /** * Only visible block/biome changes are recorded in the database.
diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 0d4a35e35..75ab0ec71 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -19,6 +19,7 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources; +import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; @@ -93,6 +94,11 @@ public class FullDataSourceV2 implements IDataSource * @see EDhApiWorldGenerationStep */ public byte[] columnGenerationSteps; + /** + * stores what world compression was used for each column. + * @see EDhApiWorldCompressionMode + */ + public byte[] columnWorldCompressionMode; /** * stored x/z, y
@@ -122,10 +128,11 @@ public class FullDataSourceV2 implements IDataSource // doesn't need to be populated since nothing has been generated yet // the default value of all 0's is adequate this.columnGenerationSteps = new byte[WIDTH * WIDTH]; + this.columnWorldCompressionMode = new byte[WIDTH * WIDTH]; } - public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep); } - private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps) + public static FullDataSourceV2 createWithData(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationStep, byte[] columnWorldCompressionMode) { return new FullDataSourceV2(pos, mapping, data, columnGenerationStep, columnWorldCompressionMode); } + private FullDataSourceV2(DhSectionPos pos, FullDataPointIdMap mapping, LongArrayList[] data, byte[] columnGenerationSteps, byte[] columnWorldCompressionMode) { LodUtil.assertTrue(data.length == WIDTH * WIDTH); @@ -135,6 +142,7 @@ public class FullDataSourceV2 implements IDataSource this.isEmpty = false; this.columnGenerationSteps = columnGenerationSteps; + this.columnWorldCompressionMode = columnWorldCompressionMode; } public static FullDataSourceV2 createFromChunk(IChunkWrapper chunkWrapper) { return LodDataBuilder.createGeneratedDataSource(chunkWrapper); } @@ -152,6 +160,7 @@ public class FullDataSourceV2 implements IDataSource // Note: this logic only works if the data point data is the same between both versions byte[] columnGenerationSteps = new byte[WIDTH * WIDTH]; + byte[] columnWorldCompressionMode = new byte[WIDTH * WIDTH]; LongArrayList[] dataPoints = new LongArrayList[WIDTH * WIDTH]; for (int x = 0; x < WIDTH; x++) { @@ -197,11 +206,12 @@ public class FullDataSourceV2 implements IDataSource // the old data sources didn't have a generation step written down // if the column has any data points, assume it's fully generated, otherwise assume it's empty columnGenerationSteps[index] = (columnHasNonAirBlock ? EDhApiWorldGenerationStep.LIGHT.value : EDhApiWorldGenerationStep.EMPTY.value); + columnWorldCompressionMode[index] = EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS.value; } } } - FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps); + FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps, columnWorldCompressionMode); // should only be used if debugging, this is a very expensive operation @@ -361,7 +371,10 @@ public class FullDataSourceV2 implements IDataSource } } + this.columnGenerationSteps[index] = inputGenState; + // always overwrite the compression mode since we're replacing this column + this.columnWorldCompressionMode[index] = inputDataSource.columnWorldCompressionMode[index]; this.isEmpty = false; } } @@ -407,6 +420,11 @@ public class FullDataSourceV2 implements IDataSource this.columnGenerationSteps[recipientIndex] = inputGenStep; + // world compression // + byte worldCompressionMode = determineHighestWorldCompressionForTwoByTwoColumn(inputDataSource.columnWorldCompressionMode, x, z); + this.columnWorldCompressionMode[recipientIndex] = worldCompressionMode; + + // data points // LongArrayList mergedInputDataArray = mergeInputTwoByTwoDataColumn(inputDataSource, x, z); @@ -465,6 +483,7 @@ public class FullDataSourceV2 implements IDataSource */ private static byte determineMinWorldGenStepForTwoByTwoColumn(byte[] columnGenerationSteps, int relX, int relZ) { + // TODO merge similar logic with determineHighestWorldCompressionForTwoByTwoColumn byte minWorldGenStepValue = Byte.MAX_VALUE; for (int x = 0; x < 2; x++) { @@ -477,6 +496,25 @@ public class FullDataSourceV2 implements IDataSource } return minWorldGenStepValue; } + /** + * The minimum value is used because we don't want to accidentally record that + * something was generated when it wasn't. + */ + private static byte determineHighestWorldCompressionForTwoByTwoColumn(byte[] columnCompressionMode, int relX, int relZ) + { + // TODO merge similar logic with determineMinWorldGenStepForTwoByTwoColumn + byte minWorldGenStepValue = Byte.MIN_VALUE; + for (int x = 0; x < 2; x++) + { + for (int z = 0; z < 2; z++) + { + int index = relativePosToIndex(x + relX, z + relZ); + byte worldGenStepValue = columnCompressionMode[index]; + minWorldGenStepValue = (byte) Math.max(minWorldGenStepValue, worldGenStepValue); + } + } + return minWorldGenStepValue; + } private static LongArrayList mergeInputTwoByTwoDataColumn(FullDataSourceV2 inputDataSource, int x, int z) { LongArrayList newColumnList = new LongArrayList(); @@ -817,11 +855,12 @@ public class FullDataSourceV2 implements IDataSource return EDhApiWorldGenerationStep.fromValue(this.columnGenerationSteps[index]); } - public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep) + public void setSingleColumn(LongArrayList longArray, int relX, int relZ, EDhApiWorldGenerationStep worldGenStep, EDhApiWorldCompressionMode worldCompressionMode) { int index = relativePosToIndex(relX, relZ); this.dataPoints[index] = longArray; this.columnGenerationSteps[index] = worldGenStep.value; + this.columnWorldCompressionMode[index] = worldCompressionMode.value; if (RUN_UPDATE_DEV_VALIDATION) @@ -863,6 +902,7 @@ public class FullDataSourceV2 implements IDataSource int result = this.pos.hashCode(); result = 31 * result + Arrays.deepHashCode(this.dataPoints); result = 17 * result + Arrays.hashCode(this.columnGenerationSteps); + result = 43 * result + Arrays.hashCode(this.columnWorldCompressionMode); this.cachedHashCode = result; } @@ -938,6 +978,7 @@ public class FullDataSourceV2 implements IDataSource } Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0); } return dataSource; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index ddae6c4d1..35ec4a892 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -20,6 +20,7 @@ package com.seibel.distanthorizons.core.sql.dto; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; @@ -50,6 +51,8 @@ public class FullDataSourceV2DTO implements IBaseDTO /** @see EDhApiWorldGenerationStep */ public byte[] compressedColumnGenStepByteArray; + /** @see EDhApiWorldCompressionMode */ + public byte[] compressedWorldCompressionModeByteArray; public byte[] compressedMappingByteArray; @@ -71,11 +74,12 @@ public class FullDataSourceV2DTO implements IBaseDTO { CheckedByteArray checkedDataPointArray = writeDataSourceDataArrayToBlob(dataSource.dataPoints, compressionModeEnum); byte[] compressedWorldGenStepByteArray = writeGenerationStepsToBlob(dataSource.columnGenerationSteps, compressionModeEnum); + byte[] compressedWorldCompressionModeByteArray = writeWorldCompressionModeToBlob(dataSource.columnWorldCompressionMode, compressionModeEnum); byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum); return new FullDataSourceV2DTO( dataSource.getSectionPos(), - checkedDataPointArray.checksum, compressedWorldGenStepByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, + checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, dataSource.levelMinY @@ -84,7 +88,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public FullDataSourceV2DTO( DhSectionPos pos, - int dataChecksum, byte[] compressedColumnGenStepByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, + int dataChecksum, byte[] compressedColumnGenStepByteArray, byte[] compressedWorldCompressionModeByteArray, byte dataFormatVersion, EDhApiDataCompressionMode compressionModeEnum, byte[] compressedDataByteArray, long lastModifiedUnixDateTime, long createdUnixDateTime, byte[] compressedMappingByteArray, boolean applyToParent, int levelMinY) @@ -92,6 +96,7 @@ public class FullDataSourceV2DTO implements IBaseDTO this.pos = pos; this.dataChecksum = dataChecksum; this.compressedColumnGenStepByteArray = compressedColumnGenStepByteArray; + this.compressedWorldCompressionModeByteArray = compressedWorldCompressionModeByteArray; this.dataFormatVersion = dataFormatVersion; this.compressionModeEnum = compressionModeEnum; @@ -133,10 +138,11 @@ public class FullDataSourceV2DTO implements IBaseDTO { if (FullDataSourceV2.DATA_FORMAT_VERSION != this.dataFormatVersion) { - throw new IllegalStateException("There should only be one data format right now anyway."); + throw new IllegalStateException("There should only be one data format ["+FullDataSourceV2.DATA_FORMAT_VERSION+"]."); } dataSource.columnGenerationSteps = readBlobToGenerationSteps(this.compressedColumnGenStepByteArray, this.compressionModeEnum); + dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum); dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); dataSource.mapping.clear(dataSource.getSectionPos()); @@ -260,6 +266,30 @@ public class FullDataSourceV2DTO implements IBaseDTO } + private static byte[] writeWorldCompressionModeToBlob(byte[] worldCompressionModeByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException + { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DhDataOutputStream compressedOut = new DhDataOutputStream(byteArrayOutputStream, compressionModeEnum); + + compressedOut.write(worldCompressionModeByteArray); + + compressedOut.flush(); + byteArrayOutputStream.close(); + + return byteArrayOutputStream.toByteArray(); + } + private static byte[] readBlobToWorldCompressionMode(byte[] dataByteArray, EDhApiDataCompressionMode compressionModeEnum) throws IOException, InterruptedException + { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(dataByteArray); + DhDataInputStream compressedIn = new DhDataInputStream(byteArrayInputStream, compressionModeEnum); + + byte[] worldCompressionModeByteArray = new byte[FullDataSourceV2.WIDTH * FullDataSourceV2.WIDTH]; + compressedIn.readFully(worldCompressionModeByteArray); + + return worldCompressionModeByteArray; + } + + private static byte[] writeDataMappingToBlob(FullDataPointIdMap mapping, EDhApiDataCompressionMode compressionModeEnum) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java index b325aacd7..619581a14 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV2Repo.java @@ -86,6 +86,7 @@ public class FullDataSourceV2Repo extends AbstractDhRepo Date: Thu, 28 Mar 2024 07:42:53 -0500 Subject: [PATCH 103/183] Add compression mode config documentation and missing LodDataBuilder lines --- .../distanthorizons/core/config/Config.java | 24 +++++++++++-------- .../transformers/LodDataBuilder.java | 9 ++++--- .../assets/distanthorizons/lang/en_us.json | 2 +- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 046a99677..979b55c96 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -825,16 +825,20 @@ public class Config public static ConfigEntry worldCompression = new ConfigEntry.Builder() .set(EDhApiWorldCompressionMode.VISUALLY_EQUAL) .comment("" - //+ "What algorithm should be used to compress new LOD data? \n" - //+ "This setting will only affect new or updated LOD data, \n" - //+ "any data already generated when this setting is changed will be\n" - //+ "unaffected until it needs to be re-written to the database.\n" - //+ "\n" - //+ EDhApiDataCompressionMode.UNCOMPRESSED + " \n" - //+ "Should only be used for testing, is worse in every way vs ["+EDhApiDataCompressionMode.LZ4+"].\n" - //+ "Expected Compression Ratio: 1.0\n" - //+ "Estimated average DTO read speed: 1.64 milliseconds\n" - //+ "Estimated average DTO write speed: 12.44 milliseconds\n" + + "How should block data be compressed when creating LOD data? \n" + + "This setting will only affect new or updated LOD data, \n" + + "any data already generated when this setting is changed will be\n" + + "unaffected until it is modified or re-loaded.\n" + + "\n" + + EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS + " \n" + + "Every block/biome change is recorded in the database. \n" + + "This is what DH 2.0 and 2.0.1 all used by default and will store a lot of data. \n" + + "Expected Compression Ratio: 1.0\n" + + "\n" + + EDhApiWorldCompressionMode.VISUALLY_EQUAL + " \n" + + "Only visible block/biome changes are recorded in the database. \n" + + "Hidden blocks (IE ores) are ignored. \n" + + "Expected Compression Ratio: 0.7\n" + "") .build(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 430b85495..56d7b9363 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -131,7 +131,7 @@ public class LodDataBuilder //==========================// EDhApiWorldCompressionMode worldCompressionMode = Config.Client.Advanced.LodBuilding.worldCompression.get(); - boolean ignoreHiddenBlocks = (worldCompressionMode != EDhApiWorldCompressionMode.UNCOMPRESSED); + boolean ignoreHiddenBlocks = (worldCompressionMode != EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS); int minBuildHeight = chunkWrapper.getMinNonEmptyHeight(); for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++) @@ -221,7 +221,8 @@ public class LodDataBuilder dataSource.setSingleColumn(longs, relBlockX + chunkOffsetX, relBlockZ + chunkOffsetZ, - EDhApiWorldGenerationStep.LIGHT); + EDhApiWorldGenerationStep.LIGHT, + worldCompressionMode); } } @@ -326,7 +327,9 @@ public class LodDataBuilder )); } - accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT); + // TODO add the ability for API users to define a different compression mode + // or add a "unkown" compression mode + accessor.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT, EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS); } } diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 997f1bc64..e8590d543 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -351,7 +351,7 @@ "distanthorizons.config.client.advanced.lodBuilding.worldCompression": "Lossy World Compression", "distanthorizons.config.client.advanced.lodBuilding.worldCompression.@tooltip": - "TODO", + "How should block data be compressed when creating LOD data? \nThis setting will only affect new or updated LOD data, \nany data already generated when this setting is changed will be \nunaffected until it is modified or re-loaded. \n\nMost Accurate: Merge Same Blocks \nHighest Compression: Visually Equal", "distanthorizons.config.client.advanced.multiplayer": From e7eb2ff9cc35e58dfb5112ee394f2703d61b3cbd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 28 Mar 2024 07:47:07 -0500 Subject: [PATCH 104/183] Remove the DhRenderData DB table --- .../resources/sqlScripts/0040-sqlite-removeRenderCache.sql | 4 ++++ core/src/main/resources/sqlScripts/scriptList.txt | 1 + 2 files changed, 5 insertions(+) create mode 100644 core/src/main/resources/sqlScripts/0040-sqlite-removeRenderCache.sql diff --git a/core/src/main/resources/sqlScripts/0040-sqlite-removeRenderCache.sql b/core/src/main/resources/sqlScripts/0040-sqlite-removeRenderCache.sql new file mode 100644 index 000000000..8e7aef49a --- /dev/null +++ b/core/src/main/resources/sqlScripts/0040-sqlite-removeRenderCache.sql @@ -0,0 +1,4 @@ + +-- The render cache was discovered to not speed up LOD loading, +-- so to reduce DB file size it was removed. +drop table DhRenderData; diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index ee42f56b0..9899c9e0a 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -2,3 +2,4 @@ 0010-sqlite-createInitialDataTables.sql 0020-sqlite-createFullDataSourceV2Tables.sql 0030-sqlite-changeTableJournaling.sql +0040-sqlite-removeRenderCache.sql From 57f3d4432669808fa003a58f7b2c12a858ed74bc Mon Sep 17 00:00:00 2001 From: cola98765 Date: Thu, 28 Mar 2024 19:01:42 +0000 Subject: [PATCH 105/183] Update file FullDataToRenderDataTransformer.java --- .../transformers/FullDataToRenderDataTransformer.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 28d7dd301..8c562cf4f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -30,6 +30,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; @@ -169,6 +170,8 @@ public class FullDataToRenderDataTransformer boolean isVoid = true; int colorToApplyToNextBlock = -1; + byte skylightToApplyToNextBlock = -1; + byte blocklightToApplyToNextBlock = -1; int columnOffset = 0; // goes from the top down @@ -234,7 +237,9 @@ public class FullDataToRenderDataTransformer { if (colorBelowWithAvoidedBlocks) { - colorToApplyToNextBlock = level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block); + colorToApplyToNextBlock = ColorUtil.setAlpha(level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block),255); + skylightToApplyToNextBlock = skyLight; + blocklightToApplyToNextBlock = blockLight; } // don't add this block @@ -246,13 +251,15 @@ public class FullDataToRenderDataTransformer if (colorToApplyToNextBlock == -1) { // use this block's color - color = level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block); + colorToApplyToNextBlock = level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block); } else { // use the previous block's color color = colorToApplyToNextBlock; colorToApplyToNextBlock = -1; + skyLight = skylightToApplyToNextBlock; + blockLight = blocklightToApplyToNextBlock; } From ea0a62b6a354ed2a0a76fab6a0429db7f0702db9 Mon Sep 17 00:00:00 2001 From: cola98765 Date: Thu, 28 Mar 2024 20:14:19 +0000 Subject: [PATCH 106/183] panic fix --- .../transformers/FullDataToRenderDataTransformer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 8c562cf4f..0e9c03e50 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -251,7 +251,7 @@ public class FullDataToRenderDataTransformer if (colorToApplyToNextBlock == -1) { // use this block's color - colorToApplyToNextBlock = level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block); + color = level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block); } else { From cf97c08aaf49afe1039b574d98e2f4d0ecded59a Mon Sep 17 00:00:00 2001 From: cola98765 Date: Thu, 28 Mar 2024 21:27:01 +0000 Subject: [PATCH 107/183] @James check if it doesn't explote; This should help with rendering, and fix the issue with stacking water with 2block resolution --- .../FullDataToRenderDataTransformer.java | 30 ++++++++++++++----- .../core/util/RenderDataPointUtil.java | 1 + 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 0e9c03e50..ff590a50f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -170,6 +170,8 @@ public class FullDataToRenderDataTransformer boolean isVoid = true; int colorToApplyToNextBlock = -1; + int lastColor = 0; + int lastBottom = -10000; byte skylightToApplyToNextBlock = -1; byte blocklightToApplyToNextBlock = -1; int columnOffset = 0; @@ -237,6 +239,7 @@ public class FullDataToRenderDataTransformer { if (colorBelowWithAvoidedBlocks) { + //mare sure to not trnasfer alpha if for some reason grass is transparent colorToApplyToNextBlock = ColorUtil.setAlpha(level.computeBaseColor(new DhBlockPos(blockX, bottomY + level.getMinY(), blockZ), biome, block),255); skylightToApplyToNextBlock = skyLight; blocklightToApplyToNextBlock = blockLight; @@ -261,13 +264,26 @@ public class FullDataToRenderDataTransformer skyLight = skylightToApplyToNextBlock; blockLight = blocklightToApplyToNextBlock; } - - - // add the block - isVoid = false; - long columnData = RenderDataPointUtil.createDataPoint(bottomY + blockHeight, bottomY, color, skyLight, blockLight, block.getIrisBlockMaterialId()); - renderColumnData.set(columnOffset, columnData); - columnOffset++; + + //check if they share a top-bottom face and if they have same collor + if (color == lastColor && bottomY + blockHeight == lastBottom && columnOffset > 0) + { + //replace the previus block with new bottom + long columnData = renderColumnData.get(columnOffset - 1); + columnData = RenderDataPointUtil.setYMin(columnData, bottom); + renderColumnData.set(columnOffset - 1, columnData); + } + else + { + // add the block + isVoid = false; + long columnData = RenderDataPointUtil.createDataPoint(bottomY + blockHeight, bottomY, color, skyLight, blockLight, block.getIrisBlockMaterialId()); + renderColumnData.set(columnOffset, columnData); + columnOffset++; + } + lastBottom = bottomY; + lastColor = color; + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 85fd5ae1a..7b3397099 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -201,6 +201,7 @@ public class RenderDataPointUtil public static short getYMax(long dataPoint) { return (short) ((dataPoint >>> HEIGHT_SHIFT) & HEIGHT_MASK); } /** AKA the starting/bottom/lowest Y value above {@link AbstractDhLevel#getMinY()} */ public static short getYMin(long dataPoint) { return (short) ((dataPoint >>> DEPTH_SHIFT) & DEPTH_MASK); } + public static long setYMin(long dataPoint, int depth) { return (long) ((dataPoint & ~(DEPTH_MASK << DEPTH_SHIFT)) | (depth & DEPTH_MASK) << DEPTH_SHIFT)} public static short getAlpha(long dataPoint) { return (short) ((((dataPoint >>> ALPHA_SHIFT) & ALPHA_MASK) << ALPHA_DOWNSIZE_SHIFT) | 0b1111); } public static short getRed(long dataPoint) { return (short) ((dataPoint >>> RED_SHIFT) & RED_MASK); } From 543c3ffc540c75fd60585dba70f376d0bb7c6365 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 28 Mar 2024 17:14:08 -0500 Subject: [PATCH 108/183] Fix compiler bugs --- .../transformers/FullDataToRenderDataTransformer.java | 6 +++--- .../distanthorizons/core/util/RenderDataPointUtil.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index ff590a50f..74fea11c1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -172,8 +172,8 @@ public class FullDataToRenderDataTransformer int colorToApplyToNextBlock = -1; int lastColor = 0; int lastBottom = -10000; - byte skylightToApplyToNextBlock = -1; - byte blocklightToApplyToNextBlock = -1; + int skylightToApplyToNextBlock = -1; + int blocklightToApplyToNextBlock = -1; int columnOffset = 0; // goes from the top down @@ -270,7 +270,7 @@ public class FullDataToRenderDataTransformer { //replace the previus block with new bottom long columnData = renderColumnData.get(columnOffset - 1); - columnData = RenderDataPointUtil.setYMin(columnData, bottom); + columnData = RenderDataPointUtil.setYMin(columnData, bottomY); renderColumnData.set(columnOffset - 1, columnData); } else diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 7b3397099..19eb93460 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -201,7 +201,7 @@ public class RenderDataPointUtil public static short getYMax(long dataPoint) { return (short) ((dataPoint >>> HEIGHT_SHIFT) & HEIGHT_MASK); } /** AKA the starting/bottom/lowest Y value above {@link AbstractDhLevel#getMinY()} */ public static short getYMin(long dataPoint) { return (short) ((dataPoint >>> DEPTH_SHIFT) & DEPTH_MASK); } - public static long setYMin(long dataPoint, int depth) { return (long) ((dataPoint & ~(DEPTH_MASK << DEPTH_SHIFT)) | (depth & DEPTH_MASK) << DEPTH_SHIFT)} + public static long setYMin(long dataPoint, int depth) { return (long) ((dataPoint & ~(DEPTH_MASK << DEPTH_SHIFT)) | (depth & DEPTH_MASK) << DEPTH_SHIFT); } public static short getAlpha(long dataPoint) { return (short) ((((dataPoint >>> ALPHA_SHIFT) & ALPHA_MASK) << ALPHA_DOWNSIZE_SHIFT) | 0b1111); } public static short getRed(long dataPoint) { return (short) ((dataPoint >>> RED_SHIFT) & RED_MASK); } From ddd475d537b419c2d7101a9f0dccbcf9f8c29750 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 30 Mar 2024 08:07:53 -0500 Subject: [PATCH 109/183] Remove AbstractRenderBuffer --- .../bufferBuilding/ColumnRenderBuffer.java | 20 ++++-- .../render/bufferBuilding/LodQuadBuilder.java | 33 +++++----- .../core/render/AbstractRenderBuffer.java | 61 ------------------- .../core/render/RenderBufferHandler.java | 7 ++- .../core/render/renderer/LodRenderer.java | 3 +- 5 files changed, 36 insertions(+), 88 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/render/AbstractRenderBuffer.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java index e041efb21..1432c7fc7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java @@ -26,7 +26,6 @@ import com.seibel.distanthorizons.core.enums.EGLProxyContext; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.render.AbstractRenderBuffer; import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; @@ -46,13 +45,19 @@ import java.util.concurrent.*; * * @see ColumnRenderBufferBuilder */ -public class ColumnRenderBuffer extends AbstractRenderBuffer +public class ColumnRenderBuffer implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper minecraftClient = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); private static final long MAX_BUFFER_UPLOAD_TIMEOUT_NANOSECONDS = 1_000_000; + public static final int QUADS_BYTE_SIZE = LodUtil.LOD_VERTEX_FORMAT.getByteSize() * 4; // TODO what does the 4 represent + public static final int MAX_QUADS_PER_BUFFER = (1024 * 1024 * 1) / QUADS_BYTE_SIZE; // TODO what do these multiples represent? + public static final int FULL_SIZED_BUFFER = MAX_QUADS_PER_BUFFER * QUADS_BYTE_SIZE; + + + public final DhBlockPos pos; @@ -275,7 +280,7 @@ public class ColumnRenderBuffer extends AbstractRenderBuffer // render // //========// - @Override + /** @return true if something was rendered, false otherwise */ public boolean renderOpaque(LodRenderer renderContext, DhApiRenderParam renderEventParam) { boolean hasRendered = false; @@ -299,7 +304,7 @@ public class ColumnRenderBuffer extends AbstractRenderBuffer return hasRendered; } - @Override + /** @return true if something was rendered, false otherwise */ public boolean renderTransparent(LodRenderer renderContext, DhApiRenderParam renderEventParam) { boolean hasRendered = false; @@ -354,7 +359,6 @@ public class ColumnRenderBuffer extends AbstractRenderBuffer return false; } - @Override public void debugDumpStats(StatsMap statsMap) { statsMap.incStat("RenderBuffers"); @@ -378,6 +382,12 @@ public class ColumnRenderBuffer extends AbstractRenderBuffer } } + /** + * This method is called when object is no longer in use. + * Called either after uploadBuffers() returned false (On buffer Upload + * thread), or by others when the object is not being used. (not in build, + * upload, or render state). + */ @Override public void close() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index 1e4932de1..29acdd418 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -25,7 +25,6 @@ import java.util.*; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.render.AbstractRenderBuffer; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; @@ -386,7 +385,7 @@ public class LodQuadBuilder { return new Iterator() { - final ByteBuffer bb = ByteBuffer.allocateDirect(AbstractRenderBuffer.FULL_SIZED_BUFFER) + final ByteBuffer bb = ByteBuffer.allocateDirect(ColumnRenderBuffer.FULL_SIZED_BUFFER) .order(ByteOrder.nativeOrder()); int dir = skipEmpty(0); int quad = 0; @@ -414,7 +413,7 @@ public class LodQuadBuilder return null; } bb.clear(); - bb.limit(AbstractRenderBuffer.FULL_SIZED_BUFFER); + bb.limit(ColumnRenderBuffer.FULL_SIZED_BUFFER); while (bb.hasRemaining() && dir < 6) { writeData(); @@ -454,7 +453,7 @@ public class LodQuadBuilder { return new Iterator() { - final ByteBuffer bb = ByteBuffer.allocateDirect(AbstractRenderBuffer.FULL_SIZED_BUFFER) + final ByteBuffer bb = ByteBuffer.allocateDirect(ColumnRenderBuffer.FULL_SIZED_BUFFER) .order(ByteOrder.nativeOrder()); int directionIndex = this.skipEmptyDirectionIndices(0); int quad = 0; @@ -483,7 +482,7 @@ public class LodQuadBuilder } this.bb.clear(); - this.bb.limit(AbstractRenderBuffer.FULL_SIZED_BUFFER); + this.bb.limit(ColumnRenderBuffer.FULL_SIZED_BUFFER); while (this.bb.hasRemaining() && this.directionIndex < 6) { this.writeData(); @@ -541,19 +540,19 @@ public class LodQuadBuilder } int numOfQuads = _countRemainingQuads(); - if (numOfQuads > AbstractRenderBuffer.MAX_QUADS_PER_BUFFER) - numOfQuads = AbstractRenderBuffer.MAX_QUADS_PER_BUFFER; + if (numOfQuads > ColumnRenderBuffer.MAX_QUADS_PER_BUFFER) + numOfQuads = ColumnRenderBuffer.MAX_QUADS_PER_BUFFER; if (numOfQuads == 0) { vbo.setVertexCount(0); return false; } - ByteBuffer bb = vbo.mapBuffer(numOfQuads * AbstractRenderBuffer.QUADS_BYTE_SIZE, method, - AbstractRenderBuffer.FULL_SIZED_BUFFER); + ByteBuffer bb = vbo.mapBuffer(numOfQuads * ColumnRenderBuffer.QUADS_BYTE_SIZE, method, + ColumnRenderBuffer.FULL_SIZED_BUFFER); if (bb == null) throw new NullPointerException("mapBuffer returned null"); bb.clear(); - bb.limit(numOfQuads * AbstractRenderBuffer.QUADS_BYTE_SIZE); + bb.limit(numOfQuads * ColumnRenderBuffer.QUADS_BYTE_SIZE); while (bb.hasRemaining() && dir < 6) { writeData(bb); @@ -621,19 +620,19 @@ public class LodQuadBuilder } int numOfQuads = _countRemainingQuads(); - if (numOfQuads > AbstractRenderBuffer.MAX_QUADS_PER_BUFFER) - numOfQuads = AbstractRenderBuffer.MAX_QUADS_PER_BUFFER; + if (numOfQuads > ColumnRenderBuffer.MAX_QUADS_PER_BUFFER) + numOfQuads = ColumnRenderBuffer.MAX_QUADS_PER_BUFFER; if (numOfQuads == 0) { vbo.setVertexCount(0); return false; } - ByteBuffer bb = vbo.mapBuffer(numOfQuads * AbstractRenderBuffer.QUADS_BYTE_SIZE, method, - AbstractRenderBuffer.FULL_SIZED_BUFFER); + ByteBuffer bb = vbo.mapBuffer(numOfQuads * ColumnRenderBuffer.QUADS_BYTE_SIZE, method, + ColumnRenderBuffer.FULL_SIZED_BUFFER); if (bb == null) throw new NullPointerException("mapBuffer returned null"); bb.clear(); - bb.limit(numOfQuads * AbstractRenderBuffer.QUADS_BYTE_SIZE); + bb.limit(numOfQuads * ColumnRenderBuffer.QUADS_BYTE_SIZE); while (bb.hasRemaining() && dir < 6) { writeData(bb); @@ -718,7 +717,7 @@ public class LodQuadBuilder } /** Returns how many GpuBuffers will be needed to render opaque quads in this builder. */ - public int getCurrentNeededOpaqueVertexBufferCount() { return MathUtil.ceilDiv(this.getCurrentOpaqueQuadsCount(), AbstractRenderBuffer.MAX_QUADS_PER_BUFFER); } + public int getCurrentNeededOpaqueVertexBufferCount() { return MathUtil.ceilDiv(this.getCurrentOpaqueQuadsCount(), ColumnRenderBuffer.MAX_QUADS_PER_BUFFER); } /** Returns how many GpuBuffers will be needed to render transparent quads in this builder. */ public int getCurrentNeededTransparentVertexBufferCount() { @@ -727,7 +726,7 @@ public class LodQuadBuilder return 0; } - return MathUtil.ceilDiv(this.getCurrentTransparentQuadsCount(), AbstractRenderBuffer.MAX_QUADS_PER_BUFFER); + return MathUtil.ceilDiv(this.getCurrentTransparentQuadsCount(), ColumnRenderBuffer.MAX_QUADS_PER_BUFFER); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/AbstractRenderBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/AbstractRenderBuffer.java deleted file mode 100644 index 29e303fbb..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/AbstractRenderBuffer.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.render; - -import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.StatsMap; -import com.seibel.distanthorizons.core.render.renderer.LodRenderer; - -public abstract class AbstractRenderBuffer implements AutoCloseable -{ - // ====================================================================== - // ====================== Methods for implementations =================== - // ====================================================================== - - // ========== Called by render thread ========== - /** @return true if something was rendered, false otherwise */ - public abstract boolean renderOpaque(LodRenderer renderContext, DhApiRenderParam renderEventParam); - /** @return true if something was rendered, false otherwise */ - public abstract boolean renderTransparent(LodRenderer renderContext, DhApiRenderParam renderEventParam); - - // ========== Called by any thread. (thread safe) ========== - - /* Called by anyone. This method is allowed to throw exceptions, but - * are never allowed to modify any values. This should behave the same - * to other methods as if the method have never been called. - * Note: This method is PURELY for debug or stats logging ONLY! */ - public abstract void debugDumpStats(StatsMap statsMap); - - // ========= Called only when 1 thread is using it ======= - /* This method is called when object is no longer in use. - * Called either after uploadBuffers() returned false (On buffer Upload - * thread), or by others when the object is not being used. (not in build, - * upload, or render state). */ - public abstract void close(); - - - - public static final int QUADS_BYTE_SIZE = LodUtil.LOD_VERTEX_FORMAT.getByteSize() * 4; // TODO what does the 4 represent - public static final int MAX_QUADS_PER_BUFFER = (1024 * 1024 * 1) / QUADS_BYTE_SIZE; // TODO what do these multiples represent? - public static final int FULL_SIZED_BUFFER = MAX_QUADS_PER_BUFFER * QUADS_BYTE_SIZE; - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java index a66d3c028..9e7e7931f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java @@ -24,6 +24,7 @@ import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiCullin import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiShadowCullingFrustum; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.config.Config; +import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer; import com.seibel.distanthorizons.core.dependencyInjection.ModAccessorInjector; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; @@ -349,7 +350,7 @@ public class RenderBufferHandler implements AutoCloseable continue; } - AbstractRenderBuffer buffer = renderSection.activeRenderBufferRef.get(); + ColumnRenderBuffer buffer = renderSection.activeRenderBufferRef.get(); if (buffer == null) { continue; @@ -435,10 +436,10 @@ public class RenderBufferHandler implements AutoCloseable private static class LoadedRenderBuffer { - public final AbstractRenderBuffer buffer; + public final ColumnRenderBuffer buffer; public final DhSectionPos pos; - LoadedRenderBuffer(AbstractRenderBuffer buffer, DhSectionPos pos) + LoadedRenderBuffer(ColumnRenderBuffer buffer, DhSectionPos pos) { this.buffer = buffer; this.pos = pos; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index 40986bd19..6b8d983e8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -31,7 +31,6 @@ import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.logging.ConfigBasedSpamLogger; import com.seibel.distanthorizons.core.pos.DhBlockPos; -import com.seibel.distanthorizons.core.render.AbstractRenderBuffer; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; import com.seibel.distanthorizons.core.render.RenderBufferHandler; import com.seibel.distanthorizons.core.render.glObject.GLProxy; @@ -700,7 +699,7 @@ public class LodRenderer if (ENABLE_IBO) { this.quadIBO = new QuadElementBuffer(); - this.quadIBO.reserve(AbstractRenderBuffer.MAX_QUADS_PER_BUFFER); + this.quadIBO.reserve(ColumnRenderBuffer.MAX_QUADS_PER_BUFFER); } From 0ddf1dd640749ad4da27e497e56da72832c4a8a4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 30 Mar 2024 22:07:08 -0500 Subject: [PATCH 110/183] Remove ColumnRenderSourceLoader --- .../render/ColumnRenderSource.java | 23 --- .../render/ColumnRenderSourceLoader.java | 171 ------------------ .../core/render/LodRenderSection.java | 4 +- 3 files changed, 2 insertions(+), 196 deletions(-) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 0d40f9e1a..22278386a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -108,29 +108,6 @@ public class ColumnRenderSource implements IDataSource this.worldGenStep = EDhApiWorldGenerationStep.EMPTY; } - /** - * Creates a new ColumnRenderSource from the parsedColumnData. - * - * @throws IOException if the DataInputStream's detail level isn't what was expected - */ - public ColumnRenderSource(DhSectionPos sectionPos, ColumnRenderSourceLoader.ParsedColumnData parsedColumnData, IDhLevel level) throws IOException - { - if (sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET != parsedColumnData.detailLevel) - { - throw new IOException("Invalid data: detail level does not match"); - } - - this.sectionPos = sectionPos; - this.yOffset = level.getMinY(); - this.verticalDataCount = parsedColumnData.verticalSize; - this.renderDataContainer = parsedColumnData.dataContainer; - this.worldGenStep = parsedColumnData.worldGenStep; - this.isEmpty = parsedColumnData.isEmpty; - - this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; - this.fillDebugFlag(0, 0, SECTION_SIZE, SECTION_SIZE, DebugSourceFlag.FILE); - } - //========================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java deleted file mode 100644 index 1e816ce0f..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSourceLoader.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.dataObjects.render; - -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; -import com.seibel.distanthorizons.core.level.IDhLevel; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -/** - * Handles loading and parsing {@link FullDataSourceV1DTO}s to create {@link ColumnRenderSource}s.

- * - * Please see the {@link ColumnRenderSourceLoader#loadRenderSource} method to see what - * file versions this class can handle. - */ -public class ColumnRenderSourceLoader -{ - public static ColumnRenderSourceLoader INSTANCE = new ColumnRenderSourceLoader(); - - private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - - - - private ColumnRenderSourceLoader() { } - - - - public ColumnRenderSource loadRenderSource(FullDataSourceV1DTO dto, DhDataInputStream inputStream, IDhLevel level) throws IOException - { - int dataFileVersion = dto.binaryDataFormatVersion; - - switch (dataFileVersion) - { - case 1: - ParsedColumnData parsedColumnData = readDataV1(inputStream, level.getMinY()); - return new ColumnRenderSource(dto.pos, parsedColumnData, level); - default: - throw new IOException("Invalid Data: The data version [" + dataFileVersion + "] is not supported"); - } - } - - - - //========================// - // versioned file parsing // - //========================// - - /** - * @param inputStream Expected format: 1st byte: detail level, 2nd byte: vertical size, 3rd byte on: column data - * @throws IOException if there was an issue reading the stream - */ - private static ParsedColumnData readDataV1(DhDataInputStream inputStream, int expectedYOffset) throws IOException - { - // TODO move into ColumnRenderSource - - byte detailLevel = inputStream.readByte(); - - int verticalDataCount = inputStream.readInt(); - if (verticalDataCount <= 0) - { - throw new IOException("Invalid data: vertical size must be 0 or greater"); - } - - int maxNumberOfDataPoints = ColumnRenderSource.SECTION_SIZE * ColumnRenderSource.SECTION_SIZE * verticalDataCount; - - - byte dataPresentFlag = inputStream.readByte(); - if (dataPresentFlag != ColumnRenderSource.NO_DATA_FLAG_BYTE && dataPresentFlag != ColumnRenderSource.DATA_GUARD_BYTE) - { - throw new IOException("Incorrect render file format. Expected either: NO_DATA_FLAG_BYTE [" + ColumnRenderSource.NO_DATA_FLAG_BYTE + "] or DATA_GUARD_BYTE [" + ColumnRenderSource.DATA_GUARD_BYTE + "], Found: [" + dataPresentFlag + "]"); - } - else if (dataPresentFlag == ColumnRenderSource.NO_DATA_FLAG_BYTE) - { - // no data is present - return new ParsedColumnData(detailLevel, verticalDataCount, EDhApiWorldGenerationStep.EMPTY, new long[maxNumberOfDataPoints], true); - } - else - { - // data is present - - int fileYOffset = inputStream.readInt(); - if (fileYOffset != expectedYOffset) - { - throw new IOException("Invalid data: yOffset is incorrect. Expected: [" + expectedYOffset + "], found: [" + fileYOffset + "]."); - } - - - // read the column data - byte[] rawByteData = new byte[maxNumberOfDataPoints * Long.BYTES]; - ByteBuffer columnDataByteBuffer = ByteBuffer.wrap(rawByteData).order(ByteOrder.LITTLE_ENDIAN); - inputStream.readFully(rawByteData); - - - // parse the column data - long[] dataPoints = new long[maxNumberOfDataPoints]; - columnDataByteBuffer.asLongBuffer().get(dataPoints); - - boolean isEmpty = true; - for (long dataPoint : dataPoints) - { - if (dataPoint != 0) - { - isEmpty = false; - break; - } - } - - - - byte guardByteFlag = inputStream.readByte(); - if (guardByteFlag != ColumnRenderSource.DATA_GUARD_BYTE) - { - throw new IOException("invalid world gen step end guard"); - } - EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromValue(inputStream.readByte()); - if (worldGenStep == null) - { - LOGGER.warn("Missing WorldGenStep, defaulting to: " + EDhApiWorldGenerationStep.SURFACE.name()); - worldGenStep = EDhApiWorldGenerationStep.SURFACE; - } - - - - return new ParsedColumnData(detailLevel, verticalDataCount, worldGenStep, dataPoints, isEmpty); - } - } - - public static class ParsedColumnData - { - public final byte detailLevel; - public final int verticalSize; - public final EDhApiWorldGenerationStep worldGenStep; - public final long[] dataContainer; - public final boolean isEmpty; - - public ParsedColumnData(byte detailLevel, int verticalSize, EDhApiWorldGenerationStep worldGenStep, long[] dataContainer, boolean isEmpty) - { - this.detailLevel = detailLevel; - this.verticalSize = verticalSize; - this.worldGenStep = worldGenStep; - this.dataContainer = dataContainer; - this.isEmpty = isEmpty; - } - - } - - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index e70f20e94..3a747bc87 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -58,8 +58,8 @@ public class LodRenderSection implements IDebugRenderable private boolean isRenderingEnabled = false; /** - * If this is true, then {@ link LodRenderSection#reload(IRenderSourceProvider)} was called while - * a {@ link IRenderSourceProvider} was already being loaded. + * If this is true, then {@link LodRenderSection#reload(FullDataSourceProviderV2)} was called while + * a {@link ColumnRenderSource} was already being loaded. */ private boolean reloadRenderSourceOnceLoaded = false; From 88d6092153e2237ac87d054234eb5987286b6b18 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 31 Mar 2024 18:42:46 -0500 Subject: [PATCH 111/183] Clean up LodRenderSection and fix massive memory use at long render distances --- .../bufferBuilding/ColumnRenderBuffer.java | 4 +- .../ColumnRenderBufferBuilder.java | 48 +- .../core/render/LodQuadTree.java | 50 +- .../core/render/LodRenderSection.java | 461 +++++------------- .../core/render/RenderBufferHandler.java | 16 +- 5 files changed, 150 insertions(+), 429 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java index 1432c7fc7..796aebd4c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java @@ -66,17 +66,15 @@ public class ColumnRenderBuffer implements AutoCloseable private GLVertexBuffer[] vbos; private GLVertexBuffer[] vbosTransparent; - private final DhSectionPos debugPos; //==============// // constructors // //==============// - public ColumnRenderBuffer(DhBlockPos pos, DhSectionPos debugPos) + public ColumnRenderBuffer(DhBlockPos pos) { this.pos = pos; - this.debugPos = debugPos; this.vbos = new GLVertexBuffer[0]; this.vbosTransparent = new GLVertexBuffer[0]; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 0761b66bb..39501c962 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -31,7 +31,6 @@ import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; -import com.seibel.distanthorizons.core.util.objects.Reference; import com.seibel.distanthorizons.core.util.objects.UncheckedInterruptedException; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; @@ -63,8 +62,8 @@ public class ColumnRenderBufferBuilder // vbo building // //==============// - public static CompletableFuture buildBuffersAsync( - IDhClientLevel clientLevel, Reference renderBufferRef, + public static CompletableFuture buildAndUploadBuffersAsync( + IDhClientLevel clientLevel, ColumnRenderSource renderSource, ColumnRenderSource[] adjData) { ThreadPoolExecutor bufferBuilderExecutor = ThreadPoolUtil.getBufferBuilderExecutor(); @@ -126,20 +125,11 @@ public class ColumnRenderBufferBuilder { try { - //EVENT_LOGGER.trace("RenderRegion start Upload @ " + renderSource.sectionPos); - - ColumnRenderBuffer buffer = renderBufferRef.swap(null); - if (buffer == null) - { - buffer = new ColumnRenderBuffer(new DhBlockPos(renderSource.sectionPos.getMinCornerLodPos().getCornerBlockPos(), clientLevel.getMinY()), renderSource.sectionPos); - } - + ColumnRenderBuffer buffer = new ColumnRenderBuffer(new DhBlockPos(renderSource.sectionPos.getMinCornerLodPos().getCornerBlockPos(), clientLevel.getMinY())); try { buffer.uploadBuffer(quadBuilder, GLProxy.getInstance().getGpuUploadMethod()); LodUtil.assertTrue(buffer.buffersUploaded); - - //EVENT_LOGGER.trace("RenderRegion end Upload @ " + renderSource.sectionPos); return buffer; } catch (Exception e) @@ -154,35 +144,10 @@ public class ColumnRenderBufferBuilder } catch (Throwable e3) { - LOGGER.error("\"LodNodeBufferBuilder\" was unable to upload buffer: ", e3); + LOGGER.error("LodNodeBufferBuilder was unable to upload buffer: "+e3.getMessage(), e3); throw e3; } - }, bufferUploaderExecutor) - .handle((columnRenderBuffer, ex) -> - { - //LOGGER.info("RenderRegion endBuild @ {}", renderSource.sectionPos); - if (ex != null) - { - LOGGER.warn("Buffer building failed: " + ex.getMessage(), ex); - - if (!renderBufferRef.isEmpty()) - { - ColumnRenderBuffer buffer = renderBufferRef.swap(null); - buffer.close(); - } - - return null; - } - else - { - if (columnRenderBuffer != null) - { - LodUtil.assertTrue(columnRenderBuffer.buffersUploaded); - } - - return columnRenderBuffer; - } - }); + }, bufferUploaderExecutor); } private static void makeLodRenderData(LodQuadBuilder quadBuilder, ColumnRenderSource renderSource, ColumnRenderSource[] adjRegions) { @@ -321,8 +286,7 @@ public class ColumnRenderBufferBuilder } catch (RuntimeException e) { - EVENT_LOGGER.warn("Failed to get adj data for [" + detailLevel + ":" + x + "," + z + "] at [" + lodDirection + "]"); - EVENT_LOGGER.warn("Detail exception: ", e); + EVENT_LOGGER.warn("Failed to get adj data for [" + detailLevel + ":" + x + "," + z + "] at [" + lodDirection + "], Error: "+e.getMessage(), e); } } // for adjacent directions diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index ac69e5126..f10e6c8fe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -23,6 +23,7 @@ import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; +import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -131,7 +132,7 @@ public class LodQuadTree extends QuadTree implements AutoClose try { // recenter if necessary, removing out of bounds sections - this.setCenterBlockPos(playerPos, LodRenderSection::dispose); + this.setCenterBlockPos(playerPos, LodRenderSection::close); this.updateAllRenderSections(playerPos); } @@ -158,9 +159,9 @@ public class LodQuadTree extends QuadTree implements AutoClose try { LodRenderSection renderSection = this.getValue(pos); - if (renderSection != null) + if (renderSection != null && renderSection.renderingEnabled) { - renderSection.reload(this.fullDataSourceProvider); + renderSection.loadRenderSourceAsync(); } } catch (IndexOutOfBoundsException e) @@ -180,7 +181,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos rootPos = rootPosIterator.next(); if (this.getNode(rootPos) == null) { - this.setValue(rootPos, new LodRenderSection(this, rootPos)); + this.setValue(rootPos, new LodRenderSection(rootPos, this.level, this.fullDataSourceProvider)); } QuadNode rootNode = this.getNode(rootPos); @@ -209,7 +210,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // make sure the node is created if (quadNode == null && this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance { - rootNode.setValue(sectionPos, new LodRenderSection(this, sectionPos)); + rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this.level, this.fullDataSourceProvider)); quadNode = rootNode.getNode(sectionPos); } if (quadNode == null) @@ -223,7 +224,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // create a new render section if missing if (renderSection == null) { - LodRenderSection newRenderSection = new LodRenderSection(this, sectionPos); + LodRenderSection newRenderSection = new LodRenderSection(sectionPos, this.level, this.fullDataSourceProvider); rootNode.setValue(sectionPos, newRenderSection); renderSection = newRenderSection; // TODO this never seemed to be called, is it necessary? @@ -231,7 +232,6 @@ public class LodQuadTree extends QuadTree implements AutoClose - //===============================// // handle enabling, loading, // // and disabling render sections // @@ -246,7 +246,7 @@ public class LodQuadTree extends QuadTree implements AutoClose if (sectionPos.getDetailLevel() > expectedDetailLevel) { // section detail level too high // - boolean canThisPosRender = renderSection.isRenderingEnabled(); + boolean canThisPosRender = renderSection.canRender(); boolean allChildrenSectionsAreLoaded = true; // recursively update all child render sections @@ -268,8 +268,7 @@ public class LodQuadTree extends QuadTree implements AutoClose else { // all child positions are loaded, disable this section and enable its children. - renderSection.disableRendering(); - renderSection.disposeRenderData(); + renderSection.renderingEnabled = false; // walk back down the tree and enable the child sections //TODO there are probably more efficient ways of doing this, but this will work for now childPosIterator = quadNode.getChildPosIterator(); @@ -309,23 +308,26 @@ public class LodQuadTree extends QuadTree implements AutoClose { // prepare this section for rendering // TODO this should fire for the lowest detail level first to improve loading speed - renderSection.loadRenderSource(this.fullDataSourceProvider, this.level); + if (!renderSection.renderSourceLoading && renderSection.renderBuffer == null) + { + renderSection.loadRenderSourceAsync(); + } // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections - if (!parentRenderSectionIsEnabled && renderSection.canRenderNow()) + if (!parentRenderSectionIsEnabled && renderSection.canRender()) { // if rendering is already enabled we don't have to re-enable it - if (!renderSection.isRenderingEnabled()) + if (!renderSection.renderingEnabled) { - renderSection.enableRendering(); + renderSection.renderingEnabled = true; // delete/disable children, all of them will be a lower detail level than requested quadNode.deleteAllChildren((childRenderSection) -> { if (childRenderSection != null) { - childRenderSection.disableRendering(); - childRenderSection.disposeRenderData(); + childRenderSection.renderingEnabled = false; + childRenderSection.close(); } }); } @@ -341,7 +343,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // renderSection.disableRendering(); //} - return renderSection.canRenderNow(); + return renderSection.canRender(); } else { @@ -440,7 +442,7 @@ public class LodQuadTree extends QuadTree implements AutoClose QuadNode quadNode = nodeIterator.next(); if (quadNode.value != null) { - quadNode.value.disposeRenderData(); + quadNode.value.close(); quadNode.value = null; } } @@ -471,8 +473,16 @@ public class LodQuadTree extends QuadTree implements AutoClose return; } - //LOGGER.info("LodQuadTree reloadPos ["+pos+"]."); + this.sectionsToReload.add(pos); + + // the adjacent locations also need to be updated to make sure lighting + // and water updates correctly, otherwise oceans may have walls + // and lights may not show up over LOD borders + for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) + { + this.sectionsToReload.add(pos.getAdjacentPos(direction)); + } } @@ -561,7 +571,7 @@ public class LodQuadTree extends QuadTree implements AutoClose QuadNode quadNode = nodeIterator.next(); if (quadNode.value != null) { - quadNode.value.dispose(); + quadNode.value.close(); quadNode.value = null; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 3a747bc87..ea1a2bb4e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -30,62 +30,40 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; -import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.objects.Reference; -import com.seibel.distanthorizons.core.util.objects.quadTree.QuadTree; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import java.awt.*; import java.util.ArrayList; import java.util.Objects; -import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.ThreadPoolExecutor; /** * A render section represents an area that could be rendered. * For more information see {@link LodQuadTree}. */ -public class LodRenderSection implements IDebugRenderable +public class LodRenderSection implements IDebugRenderable, AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); public final DhSectionPos pos; - private boolean isRenderingEnabled = false; - /** - * If this is true, then {@link LodRenderSection#reload(FullDataSourceProviderV2)} was called while - * a {@link ColumnRenderSource} was already being loaded. - */ - private boolean reloadRenderSourceOnceLoaded = false; + private final IDhClientLevel level; + private final FullDataSourceProviderV2 fullDataSourceProvider; - private FullDataSourceProviderV2 fullDataSourceProvider = null; - private CompletableFuture fullDataSourceLoadFuture; - private ColumnRenderSource renderSource; - private IDhClientLevel level = null; + public boolean renderingEnabled = false; - //FIXME: Temp Hack to prevent swapping buffers too quickly - private long lastNs = -1; - private long lastSwapLocalVersion = -1; - private boolean neighborUpdated = false; - /** 2 sec */ - private static final long SWAP_TIMEOUT_IN_NS = 2_000000000L; - /** 1 sec */ - private static final long SWAP_BUSY_COLLISION_TIMEOUT_IN_NS = 1_000000000L; + /** this reference is necessary so we can determine what VBO to render */ + public ColumnRenderBuffer renderBuffer; - private CompletableFuture buildRenderBufferFuture = null; - private final Reference inactiveRenderBufferRef = new Reference<>(); - /** a reference is used so the render buffer can be swapped to and from the buffer builder */ - public final AtomicReference activeRenderBufferRef = new AtomicReference<>(); - private volatile boolean disposeActiveBuffer = false; - - private final QuadTree parentQuadTree; + public boolean renderSourceLoading = false; + private boolean canRender = false; private boolean missingPositionsCalculated = false; /** should be an empty array if no positions need to be generated */ @@ -97,112 +75,117 @@ public class LodRenderSection implements IDebugRenderable // constructor // //=============// - public LodRenderSection(QuadTree parentQuadTree, DhSectionPos pos) + public LodRenderSection(DhSectionPos pos, IDhClientLevel level, FullDataSourceProviderV2 fullDataSourceProvider) { this.pos = pos; - this.parentQuadTree = parentQuadTree; + this.level = level; + this.fullDataSourceProvider = fullDataSourceProvider; DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus); } - //===========// - // rendering // - //===========// - - public void enableRendering() { this.isRenderingEnabled = true; } - public void disableRendering() { this.isRenderingEnabled = false; } - - - //=============// // render data // //=============// - /** does nothing if a render source is already loaded or in the process of loading */ - public void loadRenderSource(FullDataSourceProviderV2 fullDataSourceProvider, IDhClientLevel level) + public void loadRenderSourceAsync() { - this.fullDataSourceProvider = fullDataSourceProvider; - this.level = level; - if (this.fullDataSourceProvider == null) + if (this.renderSourceLoading) { - LOGGER.warn("LodRenderSection [" + this.pos + "] called loadRenderSource with a empty source provider"); return; } - // don't re-load or double load the render source - if (this.renderSource != null || this.fullDataSourceLoadFuture != null) + this.renderSourceLoading = true; + + + + // run on the file handler pool since a number of operations + // require a number of database hits + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) { - // since the render source has been loaded, make sure the render buffers are populated - // FIXME this is a duck tape solution, since the renderBufferRef should be populated elsewhere, but this does fix empty LODs when moving around the world - if (this.activeRenderBufferRef.get() == null) - { - this.markBufferDirty(); // empty LOD fix #3, all solutions revolve around markBufferDirty() - } + return; + } + + executor.execute(() -> + { + FullDataSourceV2 fullDataSource = null; + ColumnRenderSource[] adjacentRenderSections = null; - return; - } - - this.startLoadRenderSourceAsync(); - } - - public void reload(FullDataSourceProviderV2 fullDataSourceProvider) - { - // debug rendering - boolean showRenderSectionStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get(); - if (showRenderSectionStatus && this.pos.getDetailLevel() == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL) - { - DebugRenderer.makeParticle( - new DebugRenderer.BoxParticle( - new DebugRenderer.Box(this.pos, 0, 256f, 0.03f, Color.cyan), - 0.5, 512f - ) - ); - } - - - this.fullDataSourceProvider = fullDataSourceProvider; - if (this.fullDataSourceProvider == null) - { - LOGGER.warn("LodRenderSection [" + this.pos + "] called reload with a empty source provider"); - return; - } - - // don't accidentally enable rendering for a disabled section - if (!this.isRenderingEnabled) - { - return; - } - // wait for the current load future to finish before re-loading - if (this.fullDataSourceLoadFuture != null) - { - this.reloadRenderSourceOnceLoaded = true; - return; - } - - this.startLoadRenderSourceAsync(); - } - - private void startLoadRenderSourceAsync() - { - this.fullDataSourceLoadFuture = this.fullDataSourceProvider.getAsync(this.pos); - this.fullDataSourceLoadFuture.whenComplete((fullDataSource, ex) -> - { - // this runs on the a file handler thread, so transforming the data - // here shouldn't cause any stutters - // (Although it might be good to have it on a separate thread anyway) - this.renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - this.lastNs = -1; - this.markBufferDirty(); - if (this.reloadRenderSourceOnceLoaded) + try { - this.reloadRenderSourceOnceLoaded = false; - this.reload(this.fullDataSourceProvider); + // get this positions data source + fullDataSource = this.fullDataSourceProvider.get(this.pos); + ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); + if (renderSource.isEmpty()) + { + // nothing needs to be rendered + this.canRender = false; + return; + } + + + adjacentRenderSections = this.getAndCreateNeighborRenderSources(); + + CompletableFuture uploadFuture = ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections); + this.renderBuffer = uploadFuture.join(); + + this.canRender = true; + } + catch (Exception e) + { + LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); + } + finally + { + // clean up pooled data sources + try + { + if (fullDataSource != null) + { + fullDataSource.close(); + } + + if (adjacentRenderSections != null) + { + for (int i = 0; i < adjacentRenderSections.length; i++) + { + ColumnRenderSource renderSource = adjacentRenderSections[i]; + if (renderSource != null) + { + renderSource.close(); + } + } + } + } + catch (Exception ignore){ } + + this.renderSourceLoading = false; } - - this.fullDataSourceLoadFuture = null; }); } + /** Should be called on the {@link ThreadPoolUtil#getFileHandlerExecutor()} */ + private ColumnRenderSource[] getAndCreateNeighborRenderSources() + { + ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; + for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) + { + DhSectionPos adjPos = this.pos.getAdjacentPos(direction); + try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(adjPos)) + { + // TODO some temporary caching could be done here + ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); + adjacentRenderSections[direction.ordinal() - 2] = renderSource; + } + catch (Exception e) + { + LOGGER.warn("Unable to get neighbor render source "+this.pos+" - "+adjPos+", error: "+e.getMessage(), e); + } + } + + return adjacentRenderSections; + } @@ -210,211 +193,7 @@ public class LodRenderSection implements IDebugRenderable // getters and properties // //========================// - /** This can return true before the render data is loaded */ - public boolean isRenderingEnabled() { return this.isRenderingEnabled; } - - public ColumnRenderSource getRenderSource() { return this.renderSource; } - - public boolean canRenderNow() - { - if (this.fullDataSourceLoadFuture != null || this.buildRenderBufferFuture != null) - { - // wait for loading to finish - return false; - } - - return this.renderSource != null - && - ( - ( - // if true; either this section represents empty chunks or un-generated chunks. - // Either way, there isn't any data to render, but this should be considered "loaded" - this.renderSource.isEmpty() - ) - || - ( - // check if the buffers have been loaded - this.activeRenderBufferRef.get() != null // in the case of missing sections, this is probably null - && this.lastSwapLocalVersion != -1 - ) - ); - } - - public void markBufferDirty() { this.lastSwapLocalVersion = -1; } - - - - //=================// - // buffer building // - //=================// - - private LodRenderSection[] getNeighbors() - { - LodRenderSection[] adjacentRenderSections = new LodRenderSection[EDhDirection.ADJ_DIRECTIONS.length]; - for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) - { - try - { - DhSectionPos adjPos = this.pos.getAdjacentPos(direction); - LodRenderSection adjRenderSection = this.parentQuadTree.getValue(adjPos); - // adjacent render sources might be null - adjacentRenderSections[direction.ordinal() - 2] = adjRenderSection; - } - catch (IndexOutOfBoundsException e) - { - // adjacent positions can be out of bounds, in that case a null render source will be used - } - } - - return adjacentRenderSections; - } - - private void tellNeighborsUpdated() - { - LodRenderSection[] adjacentRenderSections = this.getNeighbors(); - for (LodRenderSection adj : adjacentRenderSections) - { - if (adj != null) - { - adj.neighborUpdated = true; - } - } - } - - /** @return true if this section is loaded and set to render */ - public boolean canBuildBuffer() { return this.fullDataSourceLoadFuture == null && this.renderSource != null && this.buildRenderBufferFuture == null && !this.renderSource.isEmpty() && this.isBufferOutdated(); } - private boolean isBufferOutdated() { return this.neighborUpdated || this.renderSource.localVersion.get() != this.lastSwapLocalVersion; } - - /** @return true if this section is loaded and set to render */ - public boolean canSwapBuffer() { return this.buildRenderBufferFuture != null && this.buildRenderBufferFuture.isDone(); } - - - public synchronized void disposeRenderData() // synchronized is a band-aid solution to prevent a rare bug where the future isn't canceled in the right order - { - if (this.buildRenderBufferFuture != null) - { - //LOGGER.info("Cancelling build of render buffer for {}", sectionPos); - this.buildRenderBufferFuture.cancel(true); - this.buildRenderBufferFuture = null; - } - this.disposeActiveBuffer = true; - - this.renderSource = null; - if (this.fullDataSourceLoadFuture != null) - { - this.fullDataSourceLoadFuture.cancel(true); - this.fullDataSourceLoadFuture = null; - } - } - - - /** - * Try and swap in new render buffer for this section. Note that before this call, there should be no other - * places storing or referencing the render buffer. - * - * @return True if the swap was successful. False if swap is not needed or if it is in progress. - */ - public boolean tryBuildAndSwapBuffer() - { - // delete the existing buffer if it should be disposed - if (this.disposeActiveBuffer && this.activeRenderBufferRef.get() != null) - { - this.disposeActiveBuffer = false; - this.activeRenderBufferRef.getAndSet(null).close(); - return false; - } - - - // attempt to build the buffer - boolean didSwapped = false; - if (this.canBuildBuffer()) - { - // debug - boolean showRenderSectionStatus = Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get(); - if (showRenderSectionStatus && this.pos.getDetailLevel() == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL) - { - DebugRenderer.makeParticle( - new DebugRenderer.BoxParticle( - new DebugRenderer.Box(this.pos, 32f, 64f, 0.2f, Color.yellow), - 0.5, 16f - ) - ); - } - - - this.neighborUpdated = false; - long newVersion = this.renderSource.localVersion.get(); - if (this.lastSwapLocalVersion != newVersion) - { - this.lastSwapLocalVersion = newVersion; - this.tellNeighborsUpdated(); - } - - - LodRenderSection[] adjacentRenderSections = this.getNeighbors(); - ColumnRenderSource[] adjacentSources = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; - for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) - { - LodRenderSection adj = adjacentRenderSections[i]; - if (adj != null) - { - adjacentSources[i] = adj.getRenderSource(); - } - } - - this.buildRenderBufferFuture = ColumnRenderBufferBuilder.buildBuffersAsync(this.level, this.inactiveRenderBufferRef, this.renderSource, adjacentSources); - } - - - // attempt to swap in the buffer - if (this.canSwapBuffer()) - { - this.lastNs = System.nanoTime(); - ColumnRenderBuffer newBuffer; - try - { - newBuffer = this.buildRenderBufferFuture.getNow(null); - if (newBuffer == null) - { - // failed. - this.markBufferDirty(); - return false; - } - - if (!newBuffer.buffersUploaded) - { - LodUtil.assertNotReach("The buffer future for " + this.pos + " returned an un-built buffer."); - } - - ColumnRenderBuffer oldBuffer = this.activeRenderBufferRef.getAndSet(newBuffer); - if (oldBuffer != null) - { - // the old buffer is now considered unloaded, it will need to be freshly re-loaded - oldBuffer.buffersUploaded = false; - oldBuffer.close(); - } - ColumnRenderBuffer swapped = this.inactiveRenderBufferRef.swap(oldBuffer); - didSwapped = true; - LodUtil.assertTrue(swapped == null); - } - catch (CancellationException e1) - { - // ignore. - this.buildRenderBufferFuture = null; - } - catch (CompletionException e) - { - LOGGER.error("Unable to get render buffer for " + pos + ".", e); - this.buildRenderBufferFuture = null; - } - finally - { - this.buildRenderBufferFuture = null; - } - } - - return didSwapped; - } + public boolean canRender() { return this.canRender; } @@ -467,32 +246,26 @@ public class LodRenderSection implements IDebugRenderable - //==============// // base methods // //==============// + @Override public String toString() { return "LodRenderSection{" + "pos=" + this.pos + - ", lodRenderSource=" + this.renderSource + - ", loadFuture=" + this.fullDataSourceLoadFuture + - ", isRenderEnabled=" + this.isRenderingEnabled + '}'; } - public void dispose() + @Override + public void close() { - this.disposeRenderData(); DebugRenderer.unregister(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus); - if (this.activeRenderBufferRef.get() != null) + + if (this.renderBuffer != null) { - this.activeRenderBufferRef.get().close(); - } - if (this.inactiveRenderBufferRef.value != null) - { - this.inactiveRenderBufferRef.value.close(); + this.renderBuffer.close(); } } @@ -500,29 +273,17 @@ public class LodRenderSection implements IDebugRenderable public void debugRender(DebugRenderer debugRenderer) { Color color = Color.red; - if (this.fullDataSourceProvider == null) + if (this.renderingEnabled) { - color = Color.black; + color = Color.green; } - else if (this.fullDataSourceLoadFuture != null) + else if (this.renderSourceLoading) { color = Color.yellow; } - else if (this.renderSource != null) + else if (this.canRender) { - color = Color.blue; - if (this.buildRenderBufferFuture != null) - { - color = Color.magenta; - } - else if (this.canRenderNow()) - { - color = Color.cyan; - } - else if (this.canRenderNow() && this.isRenderingEnabled) - { - color = Color.green; - } + color = Color.cyan; } debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 400, 8f, Objects.hashCode(this), 0.1f, color)); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java index 9e7e7931f..cb2af1437 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java @@ -339,18 +339,7 @@ public class RenderBufferHandler implements AutoCloseable } } - if (rebuildAllBuffers) - { - renderSection.markBufferDirty(); - } - - renderSection.tryBuildAndSwapBuffer(); - if (!renderSection.isRenderingEnabled()) - { - continue; - } - - ColumnRenderBuffer buffer = renderSection.activeRenderBufferRef.get(); + ColumnRenderBuffer buffer = renderSection.renderBuffer; if (buffer == null) { continue; @@ -362,7 +351,6 @@ public class RenderBufferHandler implements AutoCloseable catch (Exception e) { LOGGER.error("Error updating QuadTree render source at " + renderSection.pos + ".", e); - renderSection.markBufferDirty(); } } @@ -421,7 +409,7 @@ public class RenderBufferHandler implements AutoCloseable LodRenderSection renderSection = nodeIterator.next().value; if (renderSection != null) { - renderSection.dispose(); + renderSection.close(); } } From 43392ca0e479d304fde7d122d08caf7ae1807426 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 31 Mar 2024 18:57:23 -0500 Subject: [PATCH 112/183] Have LodRenderSection cancel loading on render distance change --- .../core/render/LodQuadTree.java | 2 +- .../core/render/LodRenderSection.java | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index f10e6c8fe..bc9761931 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -308,7 +308,7 @@ public class LodQuadTree extends QuadTree implements AutoClose { // prepare this section for rendering // TODO this should fire for the lowest detail level first to improve loading speed - if (!renderSection.renderSourceLoading && renderSection.renderBuffer == null) + if (!renderSection.loadingRenderSource() && renderSection.renderBuffer == null) { renderSection.loadRenderSourceAsync(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index ea1a2bb4e..289e2c332 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -62,7 +62,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public ColumnRenderBuffer renderBuffer; - public boolean renderSourceLoading = false; + private CompletableFuture renderSourceLoadingFuture = null; private boolean canRender = false; private boolean missingPositionsCalculated = false; @@ -92,11 +92,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public void loadRenderSourceAsync() { - if (this.renderSourceLoading) + if (this.renderSourceLoadingFuture != null) { return; } - this.renderSourceLoading = true; @@ -108,7 +107,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable return; } - executor.execute(() -> + + this.renderSourceLoadingFuture = CompletableFuture.runAsync(() -> { FullDataSourceV2 fullDataSource = null; ColumnRenderSource[] adjacentRenderSections = null; @@ -161,9 +161,9 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable } catch (Exception ignore){ } - this.renderSourceLoading = false; + this.renderSourceLoadingFuture = null; } - }); + }, executor); } /** Should be called on the {@link ThreadPoolUtil#getFileHandlerExecutor()} */ private ColumnRenderSource[] getAndCreateNeighborRenderSources() @@ -195,6 +195,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public boolean canRender() { return this.canRender; } + public boolean loadingRenderSource() { return this.renderSourceLoadingFuture != null; } + //=================================// @@ -267,6 +269,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { this.renderBuffer.close(); } + + if (this.renderSourceLoadingFuture != null) + { + this.renderSourceLoadingFuture.cancel(true); + } } @Override @@ -277,7 +284,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { color = Color.green; } - else if (this.renderSourceLoading) + else if (this.renderSourceLoadingFuture != null) { color = Color.yellow; } From dc5968b0b554cf1d0a497ebc84e8c8bb2c704831 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 1 Apr 2024 20:16:07 -0500 Subject: [PATCH 113/183] pool columnRenderSources --- .../fullData/sources/FullDataSourceV2.java | 123 +++++----------- .../render/ColumnRenderSource.java | 132 +++++++----------- .../render/columnViews/ColumnArrayView.java | 28 ++-- .../render/columnViews/ColumnQuadView.java | 19 ++- .../FullDataToRenderDataTransformer.java | 4 +- .../core/file/DataSourcePool.java | 122 ++++++++++++++++ .../FullDataSourceProviderV2.java | 4 +- .../core/render/LodRenderSection.java | 14 +- .../core/sql/dto/FullDataSourceV2DTO.java | 2 +- 9 files changed, 246 insertions(+), 202 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 75ab0ec71..12fc16459 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -24,8 +24,8 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratio import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; +import com.seibel.distanthorizons.core.file.DataSourcePool; import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -33,7 +33,6 @@ import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; @@ -41,16 +40,11 @@ import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.Arrays; -import java.util.concurrent.locks.ReentrantLock; /** * This data source contains every datapoint over its given {@link DhSectionPos}.

* - * TODO create a child object that extends AutoClosable - * that can be pooled to reduce GC overhead - * * @see FullDataPointUtilV2 * @see FullDataSourceV1 */ @@ -71,7 +65,9 @@ public class FullDataSourceV2 implements IDataSource public static final int WIDTH = 64; public static final byte DATA_FORMAT_VERSION = 1; - + + public static final DataSourcePool DATA_SOURCE_POOL = new DataSourcePool<>(FullDataSourceV2::createEmpty, FullDataSourceV2::prepPooledDataSource); + private int cachedHashCode = 0; @@ -836,6 +832,33 @@ public class FullDataSourceV2 implements IDataSource + //=========// + // pooling // + //=========// + + private static void prepPooledDataSource(DhSectionPos pos, boolean clearData, FullDataSourceV2 dataSource) + { + dataSource.pos = pos; + + if (clearData) + { + dataSource.mapping.clear(pos); + + for (int i = 0; i < dataSource.dataPoints.length; i++) + { + if (dataSource.dataPoints[i] != null) + { + dataSource.dataPoints[i].clear(); + } + } + + Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); + Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0); + } + } + + + //=====================// // setters and getters // //=====================// @@ -929,92 +952,10 @@ public class FullDataSourceV2 implements IDataSource } } - - - //=========// - // pooling // - //=========// - @Override public void close() throws Exception { - returnPooledDataSource(this); + DATA_SOURCE_POOL.returnPooledDataSource(this); } - - /** used when pooling data sources */ - private static final ArrayList CACHED_SOURCES = new ArrayList<>(); - private static final ReentrantLock CACHE_LOCK = new ReentrantLock(); - - - /** @return an empty data source if non are cached */ - public static FullDataSourceV2 getPooledSource(DhSectionPos pos, boolean clearData) - { - try - { - CACHE_LOCK.lock(); - - int index = CACHED_SOURCES.size() - 1; - if (index == -1) - { - // no pooled sources exist - return createEmpty(pos); - } - else - { - FullDataSourceV2 dataSource = CACHED_SOURCES.remove(index); - dataSource.pos = pos; - - if (clearData) - { - dataSource.mapping.clear(pos); - - for (int i = 0; i < dataSource.dataPoints.length; i++) - { - if (dataSource.dataPoints[i] != null) - { - dataSource.dataPoints[i].clear(); - } - } - - Arrays.fill(dataSource.columnGenerationSteps, (byte) 0); - Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0); - } - - return dataSource; - } - } - finally - { - CACHE_LOCK.unlock(); - } - } - - /** - * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. - * It just means a new source must be constructed next time {@link FullDataSourceV2#getPooledSource} is called. - */ - public static void returnPooledDataSource(FullDataSourceV2 dataSource) - { - if (dataSource == null) - { - return; - } - else if (CACHED_SOURCES.size() > 25) - { - return; - } - - try - { - CACHE_LOCK.lock(); - CACHED_SOURCES.add(dataSource); - } - finally - { - CACHE_LOCK.unlock(); - } - } - - } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 22278386a..6a2b44a7f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -22,25 +22,22 @@ package com.seibel.distanthorizons.core.dataObjects.render; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dataObjects.transformers.FullDataToRenderDataTransformer; +import com.seibel.distanthorizons.core.file.DataSourcePool; import com.seibel.distanthorizons.core.file.IDataSource; -import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; import com.seibel.distanthorizons.coreapi.ModInfo; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnQuadView; -import com.seibel.distanthorizons.core.dataObjects.render.columnViews.IColumnDataView; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; -import com.seibel.distanthorizons.core.util.LodUtil; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; -import java.io.*; +import java.util.Arrays; import java.util.concurrent.atomic.AtomicLong; /** @@ -56,27 +53,21 @@ public class ColumnRenderSource implements IDataSource public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); + @Deprecated public static final byte DATA_FORMAT_VERSION = 1; @Override public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - public static final String DATA_NAME = "ColumnRenderSource"; + public static final DataSourcePool DATA_SOURCE_POOL = new DataSourcePool<>(ColumnRenderSource::createEmptyRenderSource, null /* data source prep/cleanup needs to be done outside the pool since it requires additional inputs */); - /** - * This is the byte put between different sections in the binary save file. - * The presence and absence of this byte indicates if the file is correctly formatted. - */ - public static final int DATA_GUARD_BYTE = 0xFFFFFFFF; - /** indicates the binary save file represents an empty data source */ - public static final int NO_DATA_FLAG_BYTE = 0x00000001; /** will be zero if an empty data source was created */ public int verticalDataCount; - public final DhSectionPos sectionPos; - public final int yOffset; + public DhSectionPos sectionPos; + public int yOffset; - public long[] renderDataContainer; + public LongArrayList renderDataContainer; public final DebugSourceFlag[] debugSourceFlags; @@ -91,19 +82,52 @@ public class ColumnRenderSource implements IDataSource // constructors // //==============// - public static ColumnRenderSource createEmptyRenderSource(DhSectionPos sectionPos) { return new ColumnRenderSource(sectionPos, 0, 0); } + /** + * This is separate from {@link DataSourcePool#getPooledSource(DhSectionPos, boolean)} + * because we need to pass in a couple extra values, + * specifically maxVerticalSize and yOffset. + */ + public static ColumnRenderSource getPooledRenderSource(DhSectionPos pos, int maxVerticalSize, int yOffset, boolean clearData) + { + ColumnRenderSource renderSource = DATA_SOURCE_POOL.getPooledSource(pos); + + // set necessary properties + renderSource.sectionPos = pos; + renderSource.verticalDataCount = maxVerticalSize; + renderSource.yOffset = yOffset; + + + // resize the array if necessary + int dataArraySize = SECTION_SIZE * SECTION_SIZE * maxVerticalSize; + renderSource.renderDataContainer.ensureCapacity(dataArraySize); + while (renderSource.renderDataContainer.size() < dataArraySize) + { + renderSource.renderDataContainer.add(0); + } + + if (clearData) + { + Arrays.fill(renderSource.renderDataContainer.elements(), 0); + Arrays.fill(renderSource.debugSourceFlags, null); + } + + return renderSource; + } + + + private static ColumnRenderSource createEmptyRenderSource(DhSectionPos sectionPos) { return new ColumnRenderSource(sectionPos, 0, 0); } /** * Creates an empty ColumnRenderSource. * - * @param sectionPos the relative position of the container + * @param pos the relative position of the container * @param maxVerticalSize the maximum vertical size of the container */ - public ColumnRenderSource(DhSectionPos sectionPos, int maxVerticalSize, int yOffset) + private ColumnRenderSource(DhSectionPos pos, int maxVerticalSize, int yOffset) { this.verticalDataCount = maxVerticalSize; - this.renderDataContainer = new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]; + this.renderDataContainer = new LongArrayList(new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]); this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; - this.sectionPos = sectionPos; + this.sectionPos = pos; this.yOffset = yOffset; this.worldGenStep = EDhApiWorldGenerationStep.EMPTY; } @@ -114,65 +138,7 @@ public class ColumnRenderSource implements IDataSource // datapoint manipulation // //========================// - public void clearDataPoint(int posX, int posZ) - { - for (int verticalIndex = 0; verticalIndex < this.verticalDataCount; verticalIndex++) - { - this.renderDataContainer[posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex] = RenderDataPointUtil.EMPTY_DATA; - } - } - - public boolean setDataPoint(long data, int posX, int posZ, int verticalIndex) - { - this.renderDataContainer[posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex] = data; - return true; - } - - public boolean copyVerticalData(IColumnDataView newData, int posX, int posZ, boolean overwriteDataWithSameGenerationMode) - { - if (DO_SAFETY_CHECKS) - { - if (newData.size() != this.verticalDataCount) - throw new IllegalArgumentException("newData size not the same as this column's vertical size"); - if (posX < 0 || posX >= SECTION_SIZE) - throw new IllegalArgumentException("X position is out of bounds"); - if (posZ < 0 || posZ >= SECTION_SIZE) - throw new IllegalArgumentException("Z position is out of bounds"); - } - - int dataOffset = posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount; - int compare = RenderDataPointUtil.compareDatapointPriority(newData.get(0), this.renderDataContainer[dataOffset]); - if (overwriteDataWithSameGenerationMode) - { - if (compare < 0) - { - return false; - } - } - else - { - if (compare <= 0) - { - return false; - } - } - - // copy the newData into this column's data - newData.copyTo(this.renderDataContainer, dataOffset, newData.size()); - return true; - } - - - public long getFirstDataPoint(int posX, int posZ) { return getDataPoint(posX, posZ, 0); } - public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer[posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex]; } - - public long[] getVerticalDataPointArray(int posX, int posZ) - { - long[] result = new long[this.verticalDataCount]; - int index = posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount; - System.arraycopy(this.renderDataContainer, index, result, 0, this.verticalDataCount); - return result; - } + public long getDataPoint(int posX, int posZ, int verticalIndex) { return this.renderDataContainer.getLong(posX * SECTION_SIZE * this.verticalDataCount + posZ * this.verticalDataCount + verticalIndex); } public ColumnArrayView getVerticalDataPointView(int posX, int posZ) { @@ -184,8 +150,6 @@ public class ColumnRenderSource implements IDataSource public ColumnQuadView getFullQuadView() { return this.getQuadViewOverRange(0, 0, SECTION_SIZE, SECTION_SIZE); } public ColumnQuadView getQuadViewOverRange(int quadX, int quadZ, int quadXSize, int quadZSize) { return new ColumnQuadView(this.renderDataContainer, SECTION_SIZE, this.verticalDataCount, quadX, quadZ, quadXSize, quadZSize); } - public int getVerticalSize() { return this.verticalDataCount; } - //=============// @@ -357,7 +321,9 @@ public class ColumnRenderSource implements IDataSource @Override public void close() throws Exception - { /* not currently needed */ } + { + DATA_SOURCE_POOL.returnPooledDataSource(this); + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java index 17b1e9be2..88bd337b3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnArrayView.java @@ -21,12 +21,13 @@ package com.seibel.distanthorizons.core.dataObjects.render.columnViews; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; +import it.unimi.dsi.fastutil.longs.LongArrayList; import java.util.Arrays; public final class ColumnArrayView implements IColumnDataView { - public final long[] data; + public final LongArrayList data; public final int size; public final int offset; // offset in longs /** can be 0 if this column was created for an empty data source */ @@ -34,7 +35,7 @@ public final class ColumnArrayView implements IColumnDataView - public ColumnArrayView(long[] data, int size, int offset, int vertSize) + public ColumnArrayView(LongArrayList data, int size, int offset, int vertSize) { this.data = data; this.size = size; @@ -45,9 +46,9 @@ public final class ColumnArrayView implements IColumnDataView @Override - public long get(int index) { return data[index + offset]; } + public long get(int index) { return data.getLong(index + offset); } - public void set(int index, long value) { data[index + offset] = value; } + public void set(int index, long value) { data.set(index + offset, value); } @Override public int size() { return size; } @@ -64,7 +65,7 @@ public final class ColumnArrayView implements IColumnDataView return new ColumnArrayView(data, dataCount * vertSize, offset + dataIndexStart * vertSize, vertSize); } - public void fill(long value) { Arrays.fill(data, offset, offset + size, value); } + public void fill(long value) { Arrays.fill(data.elements(), offset, offset + size, value); } public void copyFrom(IColumnDataView source) { copyFrom(source, 0); } public void copyFrom(IColumnDataView source, int outputDataIndexOffset) @@ -82,19 +83,19 @@ public final class ColumnArrayView implements IColumnDataView for (int i = 0; i < source.dataCount(); i++) { int outputOffset = offset + outputDataIndexOffset * vertSize + i * vertSize; - source.subView(i, 1).copyTo(data, outputOffset, source.verticalSize()); - Arrays.fill(data, outputOffset + source.verticalSize(), + source.subView(i, 1).copyTo(data.elements(), outputOffset, source.verticalSize()); + Arrays.fill(data.elements(), outputOffset + source.verticalSize(), outputOffset + vertSize, 0); } } else { - source.copyTo(data, offset + outputDataIndexOffset * vertSize, source.size()); + source.copyTo(data.elements(), offset + outputDataIndexOffset * vertSize, source.size()); } } @Override - public void copyTo(long[] target, int offset, int size) { System.arraycopy(data, this.offset, target, offset, size); } + public void copyTo(long[] target, int offset, int size) { System.arraycopy(data.elements(), this.offset, target, offset, size); } public boolean mergeWith(ColumnArrayView source, boolean override) { @@ -170,7 +171,7 @@ public final class ColumnArrayView implements IColumnDataView sb.append(" ["); for (int i = 0; i < size; i++) { - sb.append(RenderDataPointUtil.toString(data[offset + i])); + sb.append(RenderDataPointUtil.toString(data.getLong(offset + i))); if (i < size - 1) { sb.append(",\n"); @@ -186,15 +187,18 @@ public final class ColumnArrayView implements IColumnDataView return arrayHash(data, offset, size); } - private static int arrayHash(long[] a, int offset, int length) + private static int arrayHash(LongArrayList a, int offset, int length) { if (a == null) + { return 0; + } + int result = 1; int end = offset + length; for (int i = offset; i < end; i++) { - long element = a[i]; + long element = a.getLong(i); int elementHash = (int) (element ^ (element >>> 32)); result = 31 * result + elementHash; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnQuadView.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnQuadView.java index 7c798dcb0..68d380735 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnQuadView.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/columnViews/ColumnQuadView.java @@ -19,19 +19,24 @@ package com.seibel.distanthorizons.core.dataObjects.render.columnViews; +import it.unimi.dsi.fastutil.longs.LongArrayList; + public class ColumnQuadView implements IColumnDataView { - private final long[] data; + private final LongArrayList data; private final int perColumnOffset; // per column (of columns of data) offset in longs private final int xSize; // x size in datapoints private final int zSize; // x size in datapoints private final int offset; // offset in longs private final int vertSize; // vertical size in longs - public ColumnQuadView(long[] data, int dataZWidth, int dataVertSize, int viewXOffset, int viewZOffset, int xSize, int zSize) + public ColumnQuadView(LongArrayList data, int dataZWidth, int dataVertSize, int viewXOffset, int viewZOffset, int xSize, int zSize) { - if (viewXOffset + xSize > (data.length / (dataZWidth * dataVertSize)) || viewZOffset + zSize > dataZWidth) + if (viewXOffset + xSize > (data.size() / (dataZWidth * dataVertSize)) || viewZOffset + zSize > dataZWidth) + { throw new IllegalArgumentException("View is out of bounds"); + } + this.data = data; this.xSize = xSize; this.zSize = zSize; @@ -39,7 +44,7 @@ public class ColumnQuadView implements IColumnDataView this.perColumnOffset = dataZWidth * dataVertSize; this.offset = viewXOffset * perColumnOffset + viewZOffset * dataVertSize; } - private ColumnQuadView(long[] data, int perColumnOffset, int offset, int vertSize, int xSize, int zSize) + private ColumnQuadView(LongArrayList data, int perColumnOffset, int offset, int vertSize, int xSize, int zSize) { this.data = data; this.perColumnOffset = perColumnOffset; @@ -60,12 +65,12 @@ public class ColumnQuadView implements IColumnDataView public long get(int x, int z, int v) { - return data[offset + x * perColumnOffset + z * vertSize + v]; + return data.getLong(offset + x * perColumnOffset + z * vertSize + v); } public long set(int x, int z, int v, long value) { - return data[offset + x * perColumnOffset + z * vertSize + v] = value; + return data.set(offset + x * perColumnOffset + z * vertSize + v, value); } public ColumnArrayView get(int x, int z) @@ -82,7 +87,7 @@ public class ColumnQuadView implements IColumnDataView { if (singleColumn.verticalSize() != vertSize) throw new IllegalArgumentException("Vertical size of singleColumn must be equal to vertSize"); if (singleColumn.dataCount() != 1) throw new IllegalArgumentException("SingleColumn must contain exactly one data point"); - singleColumn.copyTo(data, offset + x * perColumnOffset + z * vertSize, singleColumn.size()); + singleColumn.copyTo(data.elements(), offset + x * perColumnOffset + z * vertSize, singleColumn.size()); } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 74fea11c1..65d1b873b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -99,7 +99,7 @@ public class FullDataToRenderDataTransformer final DhSectionPos pos = fullDataSource.getSectionPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); final int vertSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(fullDataSource.getDataDetailLevel()); - final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY()); + final ColumnRenderSource columnSource = ColumnRenderSource.getPooledRenderSource(pos, vertSize, level.getMinY(), true); if (fullDataSource.isEmpty) { return columnSource; @@ -304,7 +304,7 @@ public class FullDataToRenderDataTransformer int dataTotalLength = fullDataColumn.size(); if (dataTotalLength > columnArrayView.verticalSize()) { - ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength); + ColumnArrayView totalColumnData = new ColumnArrayView(new LongArrayList(new long[dataTotalLength]), dataTotalLength, 0, dataTotalLength); iterateAndConvert(level, fullDataMapping, blockX, blockZ, totalColumnData, fullDataColumn); columnArrayView.changeVerticalSizeFrom(totalColumnData); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java new file mode 100644 index 000000000..82ffb1fd9 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java @@ -0,0 +1,122 @@ +package com.seibel.distanthorizons.core.file; + +import com.seibel.distanthorizons.core.level.IDhLevel; +import com.seibel.distanthorizons.core.pos.DhSectionPos; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; + +/** + * Data sources are often very large objects and aren't used for very long. + * This means their frequent construction and garbage collection can result in quite a bit of GC pressure. + * By pooling said data sources and reusing them we can drastically reduce this GC pressure and improve + * performance significantly. + */ +public class DataSourcePool, TDhLevel extends IDhLevel> +{ + private final ArrayList pooledDataSources = new ArrayList<>(); + private final ReentrantLock poolLock = new ReentrantLock(); + + private final Function createEmptyDatasourceFunc; + @Nullable + private final IPrepPooledDataSourceFunc prepDatasourceFunc; + + + + //=============// + // constructor // + //=============// + + public DataSourcePool(Function createEmptyDatasourceFunc, @Nullable IPrepPooledDataSourceFunc prepDatasourceFunc) + { + this.createEmptyDatasourceFunc = createEmptyDatasourceFunc; + this.prepDatasourceFunc = prepDatasourceFunc; + } + + + + //===============// + // pool handlers // + //===============// + + /** + * Returns a cleared data source. + * @see DataSourcePool#getPooledSource(DhSectionPos, boolean) + */ + public TDataSource getPooledSource(DhSectionPos pos) { return this.getPooledSource(pos, true);} + + /** @return an empty data source if non are cached */ + public TDataSource getPooledSource(DhSectionPos pos, boolean clearData) + { + try + { + this.poolLock.lock(); + + int index = this.pooledDataSources.size() - 1; + if (index == -1) + { + // no pooled sources exist + return this.createEmptyDatasourceFunc.apply(pos); + } + else + { + TDataSource dataSource = this.pooledDataSources.remove(index); + + // some data sources may want to handle prep themselves + // (due to needing additional inputs than what this pool keeps track of) + if (this.prepDatasourceFunc != null) + { + this.prepDatasourceFunc.prepDataSource(pos, clearData, dataSource); + } + + return dataSource; + } + } + finally + { + this.poolLock.unlock(); + } + } + + /** + * Doesn't have to be called, if a data source isn't returned, nothing will be leaked. + * It just means a new source must be constructed next time {@link DataSourcePool#getPooledSource} is called. + */ + public void returnPooledDataSource(TDataSource dataSource) + { + if (dataSource == null) + { + return; + } + else if (this.pooledDataSources.size() > 25) + { + return; + } + + try + { + this.poolLock.lock(); + this.pooledDataSources.add(dataSource); + } + finally + { + this.poolLock.unlock(); + } + } + + + + //================// + // helper classes // + //================// + + @FunctionalInterface + public interface IPrepPooledDataSourceFunc, TDhLevel extends IDhLevel> + { + /** @param clearData will be false if the data will be immediately overwritten anyway */ + void prepDataSource(DhSectionPos pos, boolean clearData, TDataSource dataSource); + } + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 94ce359b7..f479d5a16 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -162,11 +162,11 @@ public class FullDataSourceProviderV2 { // TODO maybe just set children update flags to true? // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway - return FullDataSourceV2.getPooledSource(pos, true); + return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); } @Override - protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.getPooledSource(pos, true); } + protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 289e2c332..7b856c78d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -112,12 +112,13 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { FullDataSourceV2 fullDataSource = null; ColumnRenderSource[] adjacentRenderSections = null; + ColumnRenderSource renderSource = null; try { // get this positions data source fullDataSource = this.fullDataSourceProvider.get(this.pos); - ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); + renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); if (renderSource.isEmpty()) { // nothing needs to be rendered @@ -147,14 +148,19 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable fullDataSource.close(); } + if (renderSource != null) + { + renderSource.close(); + } + if (adjacentRenderSections != null) { for (int i = 0; i < adjacentRenderSections.length; i++) { - ColumnRenderSource renderSource = adjacentRenderSections[i]; - if (renderSource != null) + ColumnRenderSource adjacentRenderSource = adjacentRenderSections[i]; + if (adjacentRenderSource != null) { - renderSource.close(); + adjacentRenderSource.close(); } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 35ec4a892..5a87a0cb2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -120,7 +120,7 @@ public class FullDataSourceV2DTO implements IBaseDTO public FullDataSourceV2 createPooledDataSource(@NotNull ILevelWrapper levelWrapper) throws IOException, InterruptedException { - FullDataSourceV2 dataSource = FullDataSourceV2.getPooledSource(this.pos, false); + FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(this.pos, false); return this.populateDataSource(dataSource, levelWrapper); } From 81f921623e29603fb5eca4880711463709b19e37 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 1 Apr 2024 20:18:44 -0500 Subject: [PATCH 114/183] rename IDataSource getSectionPos -> getPos --- .../fullData/sources/FullDataSourceV1.java | 2 +- .../fullData/sources/FullDataSourceV2.java | 4 ++-- .../render/ColumnRenderSource.java | 22 +++++++++---------- .../ColumnRenderBufferBuilder.java | 10 ++++----- .../FullDataToRenderDataTransformer.java | 2 +- .../file/AbstractNewDataSourceHandler.java | 8 +++---- .../core/file/IDataSource.java | 6 +---- .../DelayedFullDataSourceSaveCache.java | 2 +- .../FullDataSourceProviderV2.java | 4 ++-- .../SubDimensionLevelMatcher.java | 4 ++-- .../core/level/AbstractDhLevel.java | 4 ++-- .../core/level/ClientLevelModule.java | 2 +- .../core/sql/dto/FullDataSourceV2DTO.java | 6 ++--- 13 files changed, 36 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 33e89daaa..c33b0ac13 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -113,7 +113,7 @@ public class FullDataSourceV1 implements IDataSource public DhSectionPos getKey() { return this.sectionPos; } @Override - public DhSectionPos getSectionPos() { return this.sectionPos; } + public DhSectionPos getPos() { return this.sectionPos; } public void resizeDataStructuresForRepopulation(DhSectionPos pos) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 12fc16459..1dc4f7fa5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -207,7 +207,7 @@ public class FullDataSourceV2 implements IDataSource } } - FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getSectionPos(), legacyData.mapping, dataPoints, columnGenerationSteps, columnWorldCompressionMode); + FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getPos(), legacyData.mapping, dataPoints, columnGenerationSteps, columnWorldCompressionMode); // should only be used if debugging, this is a very expensive operation @@ -864,7 +864,7 @@ public class FullDataSourceV2 implements IDataSource //=====================// @Override - public DhSectionPos getSectionPos() { return this.pos; } + public DhSectionPos getPos() { return this.pos; } @Override public byte getDataDetailLevel() { return (byte) (this.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 6a2b44a7f..9c068f68f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -64,7 +64,7 @@ public class ColumnRenderSource implements IDataSource /** will be zero if an empty data source was created */ public int verticalDataCount; - public DhSectionPos sectionPos; + public DhSectionPos pos; public int yOffset; public LongArrayList renderDataContainer; @@ -92,7 +92,7 @@ public class ColumnRenderSource implements IDataSource ColumnRenderSource renderSource = DATA_SOURCE_POOL.getPooledSource(pos); // set necessary properties - renderSource.sectionPos = pos; + renderSource.pos = pos; renderSource.verticalDataCount = maxVerticalSize; renderSource.yOffset = yOffset; @@ -127,7 +127,7 @@ public class ColumnRenderSource implements IDataSource this.verticalDataCount = maxVerticalSize; this.renderDataContainer = new LongArrayList(new long[SECTION_SIZE * SECTION_SIZE * this.verticalDataCount]); this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; - this.sectionPos = pos; + this.pos = pos; this.yOffset = yOffset; this.worldGenStep = EDhApiWorldGenerationStep.EMPTY; } @@ -159,10 +159,10 @@ public class ColumnRenderSource implements IDataSource @Override public boolean update(FullDataSourceV2 inputFullDataSource, IDhClientLevel level) { - final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.sectionPos + "] and pos: [" + inputFullDataSource.getSectionPos() + "]. Error:"; + final String errorMessagePrefix = "Unable to complete update for RenderSource pos: [" + this.pos + "] and pos: [" + inputFullDataSource.getPos() + "]. Error:"; boolean dataChanged = false; - if (inputFullDataSource.getSectionPos().getDetailLevel() == this.sectionPos.getDetailLevel()) + if (inputFullDataSource.getPos().getDetailLevel() == this.pos.getDetailLevel()) { try { @@ -174,8 +174,8 @@ public class ColumnRenderSource implements IDataSource - DhBlockPos2D centerBlockPos = inputFullDataSource.getSectionPos().getCenterBlockPos(); - int halfBlockWidth = inputFullDataSource.getSectionPos().getBlockWidth() / 2; + DhBlockPos2D centerBlockPos = inputFullDataSource.getPos().getCenterBlockPos(); + int halfBlockWidth = inputFullDataSource.getPos().getBlockWidth() / 2; DhBlockPos2D minBlockPos = new DhBlockPos2D(centerBlockPos.x - halfBlockWidth, centerBlockPos.z - halfBlockWidth); for (int x = 0; x < FullDataSourceV2.WIDTH; x++) @@ -222,11 +222,11 @@ public class ColumnRenderSource implements IDataSource // data helper methods // //=====================// - public DhSectionPos getSectionPos() { return this.sectionPos; } + public DhSectionPos getPos() { return this.pos; } @Override - public DhSectionPos getKey() { return this.sectionPos; } + public DhSectionPos getKey() { return this.pos; } - public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } + public byte getDataDetailLevel() { return (byte) (this.pos.getDetailLevel() - SECTION_SIZE_OFFSET); } public boolean isEmpty() { return this.isEmpty; } public void markNotEmpty() { this.isEmpty = false; } @@ -293,7 +293,7 @@ public class ColumnRenderSource implements IDataSource String SUBDATA_DELIMITER = ","; StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append(this.sectionPos); + stringBuilder.append(this.pos); stringBuilder.append(LINE_DELIMITER); int size = 1; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 39501c962..6a7ab9380 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -107,7 +107,7 @@ public class ColumnRenderBufferBuilder long builderEndTime = System.currentTimeMillis(); long buildMs = builderEndTime - builderStartTime; - LOGGER.debug("RenderRegion end QuadBuild @ " + renderSource.sectionPos + " took: " + buildMs); + LOGGER.debug("RenderRegion end QuadBuild @ " + renderSource.pos + " took: " + buildMs); return builder; } @@ -125,7 +125,7 @@ public class ColumnRenderBufferBuilder { try { - ColumnRenderBuffer buffer = new ColumnRenderBuffer(new DhBlockPos(renderSource.sectionPos.getMinCornerLodPos().getCornerBlockPos(), clientLevel.getMinY())); + ColumnRenderBuffer buffer = new ColumnRenderBuffer(new DhBlockPos(renderSource.pos.getMinCornerLodPos().getCornerBlockPos(), clientLevel.getMinY())); try { buffer.uploadBuffer(quadBuilder, GLProxy.getInstance().getGpuUploadMethod()); @@ -159,9 +159,9 @@ public class ColumnRenderBufferBuilder boolean enableColumnBufferLimit = Config.Client.Advanced.Debugging.columnBuilderDebugEnable.get(); if (enableColumnBufferLimit) { - if (renderSource.sectionPos.getDetailLevel() == Config.Client.Advanced.Debugging.columnBuilderDebugDetailLevel.get() - && renderSource.sectionPos.getX() == Config.Client.Advanced.Debugging.columnBuilderDebugXPos.get() - && renderSource.sectionPos.getZ() == Config.Client.Advanced.Debugging.columnBuilderDebugZPos.get()) + if (renderSource.pos.getDetailLevel() == Config.Client.Advanced.Debugging.columnBuilderDebugDetailLevel.get() + && renderSource.pos.getX() == Config.Client.Advanced.Debugging.columnBuilderDebugXPos.get() + && renderSource.pos.getZ() == Config.Client.Advanced.Debugging.columnBuilderDebugZPos.get()) { int test = 0; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 65d1b873b..3d7b373df 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -96,7 +96,7 @@ public class FullDataToRenderDataTransformer */ private static ColumnRenderSource transformCompleteFullDataToColumnData(IDhClientLevel level, FullDataSourceV2 fullDataSource) throws InterruptedException { - final DhSectionPos pos = fullDataSource.getSectionPos(); + final DhSectionPos pos = fullDataSource.getPos(); final byte dataDetail = fullDataSource.getDataDetailLevel(); final int vertSize = Config.Client.Advanced.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(fullDataSource.getDataDetailLevel()); final ColumnRenderSource columnSource = ColumnRenderSource.getPooledRenderSource(pos, vertSize, level.getMinY(), true); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 542723e80..bf0ace1b9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -173,12 +173,12 @@ public abstract class AbstractNewDataSourceHandler try { // run file handling on a separate thread - this.markUpdateStart(inputDataSource.getSectionPos()); + this.markUpdateStart(inputDataSource.getPos()); return CompletableFuture.runAsync(() -> { try { - this.updateDataSourceAtPos(inputDataSource.getSectionPos(), inputDataSource, true); + this.updateDataSourceAtPos(inputDataSource.getPos(), inputDataSource, true); } catch (Exception e) { @@ -186,14 +186,14 @@ public abstract class AbstractNewDataSourceHandler } finally { - this.markUpdateEnd(inputDataSource.getSectionPos()); + this.markUpdateEnd(inputDataSource.getPos()); } }, executor); } catch (RejectedExecutionException ignore) { // can happen if the executor was shutdown while this task was queued - this.markUpdateEnd(inputDataSource.getSectionPos()); + this.markUpdateEnd(inputDataSource.getPos()); return CompletableFuture.completedFuture(null); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index edcb69407..44c708068 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -1,14 +1,10 @@ package com.seibel.distanthorizons.core.file; import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel; -import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.IBaseDTO; -import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; - -import java.io.IOException; /** * Base for all data sources.

@@ -19,7 +15,7 @@ import java.io.IOException; */ public interface IDataSource extends IBaseDTO, AutoCloseable { - DhSectionPos getSectionPos(); + DhSectionPos getPos(); /** @return true if the data was changed */ boolean update(FullDataSourceV2 chunkData, TDhLevel level); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java index f92cfa7dc..aa87662f8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/DelayedFullDataSourceSaveCache.java @@ -48,7 +48,7 @@ public class DelayedFullDataSourceSaveCache public void queueDataSourceForUpdateAndSave(FullDataSourceV2 inputDataSource) { - DhSectionPos dataSourcePos = inputDataSource.getSectionPos(); + DhSectionPos dataSourcePos = inputDataSource.getPos(); this.dataSourceByPosition.compute(dataSourcePos, (inputPos, temporaryDataSource) -> { if (temporaryDataSource == null) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index f479d5a16..876cd3745 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -340,12 +340,12 @@ public class FullDataSourceProviderV2 future.thenRun(() -> { // after the update finishes the legacy data source can be safely deleted - this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getSectionPos()); + this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos()); }); } catch (Exception e) { - DhSectionPos migrationPos = legacyDataSource.getSectionPos(); + DhSectionPos migrationPos = legacyDataSource.getPos(); LOGGER.error("Unexpected issue migrating data source at pos "+migrationPos+". Error: "+e.getMessage(), e); this.legacyFileHandler.markMigrationFailed(migrationPos); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index d4ef7bd55..3d701ff6f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -221,8 +221,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable // confirm both data sources have the same section pos - DhSectionPos newSectionChunkPos = newDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_CHUNK_DETAIL_LEVEL); - DhSectionPos testSectionChunkPos = testFullDataSource.getSectionPos().convertNewToDetailLevel(DhSectionPos.SECTION_CHUNK_DETAIL_LEVEL); + DhSectionPos newSectionChunkPos = newDataSource.getPos().convertNewToDetailLevel(DhSectionPos.SECTION_CHUNK_DETAIL_LEVEL); + DhSectionPos testSectionChunkPos = testFullDataSource.getPos().convertNewToDetailLevel(DhSectionPos.SECTION_CHUNK_DETAIL_LEVEL); LodUtil.assertTrue(newSectionChunkPos.equals(testSectionChunkPos), "data source positions don't match"); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java index 46c7ad97d..ccaba1934 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/AbstractDhLevel.java @@ -67,7 +67,7 @@ public abstract class AbstractDhLevel implements IDhLevel } - this.updatedChunkPosSetBySectionPos.compute(dataSource.getSectionPos(), (dataSourcePos, chunkPosSet) -> + this.updatedChunkPosSetBySectionPos.compute(dataSource.getPos(), (dataSourcePos, chunkPosSet) -> { if (chunkPosSet == null) { @@ -85,7 +85,7 @@ public abstract class AbstractDhLevel implements IDhLevel { this.updateDataSourcesAsync(fullDataSource).thenRun(() -> { - HashSet updatedChunkPosSet = this.updatedChunkPosSetBySectionPos.remove(fullDataSource.getSectionPos()); + HashSet updatedChunkPosSet = this.updatedChunkPosSetBySectionPos.remove(fullDataSource.getPos()); if (updatedChunkPosSet != null) { for (DhChunkPos chunkPos : updatedChunkPosSet) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 139e2d73f..6b424652e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -219,7 +219,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle ClientRenderState ClientRenderState = this.ClientRenderStateRef.get(); if (ClientRenderState != null) { - ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getSectionPos()); + ClientRenderState.quadtree.reloadPos(updatedFullDataSource.getPos()); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java index 5a87a0cb2..e59007129 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/dto/FullDataSourceV2DTO.java @@ -78,7 +78,7 @@ public class FullDataSourceV2DTO implements IBaseDTO byte[] mappingByteArray = writeDataMappingToBlob(dataSource.mapping, compressionModeEnum); return new FullDataSourceV2DTO( - dataSource.getSectionPos(), + dataSource.getPos(), checkedDataPointArray.checksum, compressedWorldGenStepByteArray, compressedWorldCompressionModeByteArray, FullDataSourceV2.DATA_FORMAT_VERSION, compressionModeEnum, checkedDataPointArray.byteArray, dataSource.lastModifiedUnixDateTime, dataSource.createdUnixDateTime, mappingByteArray, dataSource.applyToParent, @@ -145,7 +145,7 @@ public class FullDataSourceV2DTO implements IBaseDTO dataSource.columnWorldCompressionMode = readBlobToGenerationSteps(this.compressedWorldCompressionModeByteArray, this.compressionModeEnum); dataSource.dataPoints = readBlobToDataSourceDataArray(this.compressedDataByteArray, this.compressionModeEnum); - dataSource.mapping.clear(dataSource.getSectionPos()); + dataSource.mapping.clear(dataSource.getPos()); // should only be null when used in a unit test if (!unitTest) { @@ -154,7 +154,7 @@ public class FullDataSourceV2DTO implements IBaseDTO throw new NullPointerException("No level wrapper present, unable to deserialize data map. This should only be used for unit tests."); } - dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getSectionPos(), levelWrapper, this.compressionModeEnum)); + dataSource.mapping.mergeAndReturnRemappedEntityIds(readBlobToDataMapping(this.compressedMappingByteArray, dataSource.getPos(), levelWrapper, this.compressionModeEnum)); } dataSource.lastModifiedUnixDateTime = this.lastModifiedUnixDateTime; From 9835af0845e764224d473134ba23d8d7644da381 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 1 Apr 2024 20:19:31 -0500 Subject: [PATCH 115/183] remove unused ColumnRenderSource.worldGenStep --- .../core/dataObjects/render/ColumnRenderSource.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index 9c068f68f..e211d4087 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -72,7 +72,6 @@ public class ColumnRenderSource implements IDataSource public final DebugSourceFlag[] debugSourceFlags; private boolean isEmpty = true; - public EDhApiWorldGenerationStep worldGenStep; public AtomicLong localVersion = new AtomicLong(0); // used to track changes to the data source, so that buffers can be updated when necessary @@ -129,7 +128,6 @@ public class ColumnRenderSource implements IDataSource this.debugSourceFlags = new DebugSourceFlag[SECTION_SIZE * SECTION_SIZE]; this.pos = pos; this.yOffset = yOffset; - this.worldGenStep = EDhApiWorldGenerationStep.EMPTY; } From 964f5feb4bee7140c265b8c271706336424209f3 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 1 Apr 2024 20:21:56 -0500 Subject: [PATCH 116/183] Remove unused data source properties --- .../core/dataObjects/fullData/sources/FullDataSourceV1.java | 5 ----- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 3 --- .../core/dataObjects/render/ColumnRenderSource.java | 5 ----- .../com/seibel/distanthorizons/core/file/IDataSource.java | 3 --- 4 files changed, 16 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index c33b0ac13..1505e4566 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -124,11 +124,6 @@ public class FullDataSourceV1 implements IDataSource @Override public byte getDataDetailLevel() { return (byte) (this.sectionPos.getDetailLevel() - SECTION_SIZE_OFFSET); } - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - - public EDhApiWorldGenerationStep getWorldGenStep() { return this.worldGenStep; } - public boolean isEmpty() { return this.isEmpty; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 1dc4f7fa5..058bf13db 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -869,9 +869,6 @@ public class FullDataSourceV2 implements IDataSource @Override public byte getDataDetailLevel() { return (byte) (this.pos.getDetailLevel() - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); } - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - public EDhApiWorldGenerationStep getWorldGenStepAtRelativePos(int relX, int relZ) { int index = relativePosToIndex(relX, relZ); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java index e211d4087..453f5b490 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/ColumnRenderSource.java @@ -53,11 +53,6 @@ public class ColumnRenderSource implements IDataSource public static final byte SECTION_SIZE_OFFSET = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; public static final int SECTION_SIZE = BitShiftUtil.powerOfTwo(SECTION_SIZE_OFFSET); - @Deprecated - public static final byte DATA_FORMAT_VERSION = 1; - @Override - public byte getDataFormatVersion() { return DATA_FORMAT_VERSION; } - public static final DataSourcePool DATA_SOURCE_POOL = new DataSourcePool<>(ColumnRenderSource::createEmptyRenderSource, null /* data source prep/cleanup needs to be done outside the pool since it requires additional inputs */); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java index 44c708068..09d4d4ee8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/IDataSource.java @@ -34,7 +34,4 @@ public interface IDataSource extends IBaseDTO Date: Mon, 1 Apr 2024 20:26:13 -0500 Subject: [PATCH 117/183] Add non-magic max pooled data source number --- .../distanthorizons/core/file/DataSourcePool.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java index 82ffb1fd9..7af2fc17e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java @@ -2,6 +2,7 @@ package com.seibel.distanthorizons.core.file; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.util.ThreadUtil; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; @@ -16,6 +17,13 @@ import java.util.function.Function; */ public class DataSourcePool, TDhLevel extends IDhLevel> { + /** + * James tested with a static 25 on a 8 core 16 processor machine and didn't have any issues. + * In most cases the number of pooled sources won't probably even get close to the number of processors, + * but just in case the user has a overkill CPU (or config) this should hopefully prevent thrashing. + */ + private static final int MAX_POOLED_SOURCES = Runtime.getRuntime().availableProcessors() * 2; + private final ArrayList pooledDataSources = new ArrayList<>(); private final ReentrantLock poolLock = new ReentrantLock(); @@ -90,7 +98,7 @@ public class DataSourcePool, TDhLevel { return; } - else if (this.pooledDataSources.size() > 25) + else if (this.pooledDataSources.size() > MAX_POOLED_SOURCES) { return; } From b59d505725ebaa43288d9778567e6083c44bcb13 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 2 Apr 2024 07:17:31 -0500 Subject: [PATCH 118/183] Add IP only server folder mode --- .../api/enums/config/EServerFolderNameMode.java | 6 +++++- .../java/com/seibel/distanthorizons/core/config/Config.java | 1 + .../core/file/structure/ClientOnlySaveStructure.java | 3 +++ .../main/resources/assets/distanthorizons/lang/en_us.json | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java index 944e61457..de7e9dcec 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java @@ -22,6 +22,7 @@ package com.seibel.distanthorizons.api.enums.config; /** * NAME_ONLY,
+ * IP_ONLY,
* NAME_IP,
* NAME_IP_PORT,
* NAME_IP_PORT_MC_VERSION,

@@ -29,7 +30,7 @@ package com.seibel.distanthorizons.api.enums.config; * Determines how the multiplayer folders should be named. * * @author James Seibel - * @version 2022-7-1 + * @version 2024-4-2 * @since API 1.0.0 */ public enum EServerFolderNameMode @@ -42,6 +43,9 @@ public enum EServerFolderNameMode /** Only use the server name */ NAME_ONLY, + /** Only use the server IP */ + IP_ONLY, + /** * {SERVER_NAME} IP {IP}
* Example: Minecraft Server IP 192.168.1.40 diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 979b55c96..62ea54776 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -852,6 +852,7 @@ public class Config + "How should multiplayer save folders should be named? \n" + "\n" + EServerFolderNameMode.NAME_ONLY + ": Example: \"Minecraft Server\" \n" + + EServerFolderNameMode.IP_ONLY + ": Example: \"192.168.1.40\" \n" + EServerFolderNameMode.NAME_IP + ": Example: \"Minecraft Server IP 192.168.1.40\" \n" + EServerFolderNameMode.NAME_IP_PORT + ": Example: \"Minecraft Server IP 192.168.1.40:25565\"" + EServerFolderNameMode.NAME_IP_PORT_MC_VERSION + ": Example: \"Minecraft Server IP 192.168.1.40:25565 GameVersion 1.16.5\"") diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java b/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java index 1060076fd..a0b900f8d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java @@ -277,6 +277,9 @@ public class ClientOnlySaveStructure extends AbstractSaveStructure case NAME_ONLY: folderName = serverName; break; + case IP_ONLY: + folderName = serverIpCleaned; + break; case NAME_IP: folderName = serverName + ", IP " + serverIpCleaned; diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index e8590d543..e72910bb7 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -813,6 +813,8 @@ "distanthorizons.config.enum.EServerFolderNameMode.NAME_ONLY": "Name Only", + "distanthorizons.config.enum.EServerFolderNameMode.IP_ONLY": + "IP Only", "distanthorizons.config.enum.EServerFolderNameMode.NAME_IP": "Name and IP", "distanthorizons.config.enum.EServerFolderNameMode.NAME_IP_PORT": From 750d8b7bc3509103e5f6ea91d83d50c114367136 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 3 Apr 2024 07:19:34 -0500 Subject: [PATCH 119/183] Fix crash in 1.16 due to buffer upload happening before rendering is setup --- .../distanthorizons/core/render/LodRenderSection.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 7b856c78d..1b87956f0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -29,6 +29,7 @@ import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; @@ -92,6 +93,13 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public void loadRenderSourceAsync() { + if (!GLProxy.hasInstance()) + { + // it's possible to try uploading buffers before the GLProxy has been initialized + // which would cause the system to crash + return; + } + if (this.renderSourceLoadingFuture != null) { return; From 1491487328252858355f5cab818a8e563053581c Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 3 Apr 2024 07:28:59 -0500 Subject: [PATCH 120/183] Remove unneeded V1 -> V2 data migration validation --- .../fullData/sources/FullDataSourceV2.java | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 058bf13db..4e3a57031 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -53,7 +53,6 @@ public class FullDataSourceV2 implements IDataSource private static final Logger LOGGER = DhLoggerBuilder.getLogger(); /** useful for debugging, but can slow down update operations quite a bit due to being called so often. */ private static final boolean RUN_UPDATE_DEV_VALIDATION = false; - private static final boolean RUN_V1_MIGRATION_CONSTRUCTOR_VALIDATION = ModInfo.IS_DEV_BUILD; /** * If the data column order isn't correct * block lighting may appear broken @@ -208,43 +207,6 @@ public class FullDataSourceV2 implements IDataSource } FullDataSourceV2 fullDataSource = FullDataSourceV2.createWithData(legacyData.getPos(), legacyData.mapping, dataPoints, columnGenerationSteps, columnWorldCompressionMode); - - - // should only be used if debugging, this is a very expensive operation - if (RUN_V1_MIGRATION_CONSTRUCTOR_VALIDATION) - { - for (int x = 0; x < WIDTH; x++) - { - for (int z = 0; z < WIDTH; z++) - { - long[] legacyDataColumn = legacyData.get(x, z); - if (legacyDataColumn != null && legacyDataColumn.length != 0) - { - LongArrayList newDataColumn = fullDataSource.get(x, z); - if (newDataColumn == null) - { - LodUtil.assertNotReach("Accessor column mismatch"); - } - else if (legacyDataColumn.length != newDataColumn.size()) - { - LodUtil.assertNotReach("Accessor column length mismatch"); - } - else - { - for (int i = 0; i < legacyDataColumn.length; i++) - { - if (legacyDataColumn[i] != newDataColumn.getLong(i)) - { - LodUtil.assertNotReach("Data mismatch"); - } - } - } - - } - } - } - } - return fullDataSource; } From 4bc7cf1e86ce321ec1ce9d0395382fbf9239885f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 3 Apr 2024 21:26:05 -0500 Subject: [PATCH 121/183] Fix LodRenderSections not disabling rendering --- .../distanthorizons/core/render/LodRenderSection.java | 7 +++++++ .../distanthorizons/core/render/RenderBufferHandler.java | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 1b87956f0..a150293e3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -137,9 +137,16 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable adjacentRenderSections = this.getAndCreateNeighborRenderSources(); + ColumnRenderBuffer previousBuffer = this.renderBuffer; + CompletableFuture uploadFuture = ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections); this.renderBuffer = uploadFuture.join(); + if (previousBuffer != null) + { + previousBuffer.close(); + } + this.canRender = true; } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java index cb2af1437..6c8efc3ed 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java @@ -340,7 +340,7 @@ public class RenderBufferHandler implements AutoCloseable } ColumnRenderBuffer buffer = renderSection.renderBuffer; - if (buffer == null) + if (buffer == null || !renderSection.renderingEnabled) { continue; } From 4e1155f8a7b0d877938a39fda68b83f3a4f8c90b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 3 Apr 2024 22:07:25 -0500 Subject: [PATCH 122/183] Improve initial LOD loading speed --- .../core/file/DataSourcePool.java | 9 ++ .../core/render/LodRenderSection.java | 94 ++++++++++++++----- 2 files changed, 77 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java index 7af2fc17e..fa6b80fdf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/DataSourcePool.java @@ -116,6 +116,15 @@ public class DataSourcePool, TDhLevel + //===============// + // debug methods // + //===============// + + /** Returns how many data sources are in the pool */ + public int size() { return this.pooledDataSources.size(); } + + + //================// // helper classes // //================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index a150293e3..082772eb8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -33,13 +33,14 @@ import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; +import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; import java.awt.*; -import java.util.ArrayList; -import java.util.Objects; +import java.util.*; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; /** @@ -50,6 +51,16 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + /** + * Only the adjacent render sources should be cached to prevent accidentally using cached data when the LOD data was changed. + * This cache should really only be used when initially loading LODs or generating new terrain. + */ + private static final ConcurrentHashMap ADJACENT_RENDER_SOURCE_BY_POS = new ConcurrentHashMap<>(); + + private static final ConcurrentHashMap RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS = new ConcurrentHashMap<>(); + private static final Timer RENDER_SOURCE_CACHE_REMOVAL_TIMER = TimerUtil.CreateTimer("LodRenderSection Render Source Cache Removal Timer"); + + public final DhSectionPos pos; @@ -119,7 +130,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable this.renderSourceLoadingFuture = CompletableFuture.runAsync(() -> { FullDataSourceV2 fullDataSource = null; - ColumnRenderSource[] adjacentRenderSections = null; ColumnRenderSource renderSource = null; try @@ -135,7 +145,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable } - adjacentRenderSections = this.getAndCreateNeighborRenderSources(); + ColumnRenderSource[] adjacentRenderSections = this.getAndCreateNeighborRenderSources(); ColumnRenderBuffer previousBuffer = this.renderBuffer; @@ -167,18 +177,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { renderSource.close(); } - - if (adjacentRenderSections != null) - { - for (int i = 0; i < adjacentRenderSections.length; i++) - { - ColumnRenderSource adjacentRenderSource = adjacentRenderSections[i]; - if (adjacentRenderSource != null) - { - adjacentRenderSource.close(); - } - } - } } catch (Exception ignore){ } @@ -193,20 +191,64 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) { DhSectionPos adjPos = this.pos.getAdjacentPos(direction); - try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(adjPos)) - { - // TODO some temporary caching could be done here - ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - adjacentRenderSections[direction.ordinal() - 2] = renderSource; - } - catch (Exception e) - { - LOGGER.warn("Unable to get neighbor render source "+this.pos+" - "+adjPos+", error: "+e.getMessage(), e); - } + + ColumnRenderSource renderSource = ADJACENT_RENDER_SOURCE_BY_POS.compute(adjPos, this::computeCachedRenderSource); + adjacentRenderSections[direction.ordinal() - 2] = renderSource; } return adjacentRenderSections; } + private ColumnRenderSource computeCachedRenderSource(DhSectionPos pos, ColumnRenderSource oldRenderSource) + { + try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos)) + { + // use the old render source if it isn't null + if (oldRenderSource == null) + { + oldRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); + } + + // create a new timer task to reset the cache timeout + TimerTask timerTask = RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS.compute(pos, (timerPos, oldTimerTask) -> + { + if (oldTimerTask != null) + { + oldTimerTask.cancel(); + } + + return new TimerTask() + { + @Override + public void run() + { + // remove the finished task + RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS.remove(pos); + + // return the pooled data source if present + ColumnRenderSource expiredRenderSource = ADJACENT_RENDER_SOURCE_BY_POS.remove(pos); + if (expiredRenderSource != null) + { + try { expiredRenderSource.close(); } catch (Exception ignored) { } + } + + //LOGGER.info("cache size " +cachedNeighborSections.size()+" pool size:"+ColumnRenderSource.DATA_SOURCE_POOL.size()); + } + }; + }); + try + { + RENDER_SOURCE_CACHE_REMOVAL_TIMER.schedule(timerTask, 1000L); + } + catch (IllegalStateException ignore) { /* can rarely happen due to some minor concurrency bug with how Timer works. It isn't an issue and can be ignored. */ } + + return oldRenderSource; + } + catch (Exception e) + { + LOGGER.warn("Unable to get neighbor render source " + this.pos + " - " + pos + ", error: " + e.getMessage(), e); + return null; + } + } From 08f0f8ee17f9862d0541be142ae06bfacefdd27c Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 4 Apr 2024 07:08:51 -0500 Subject: [PATCH 123/183] Add a constant for LodRenderSection cache expiration --- .../seibel/distanthorizons/core/render/LodRenderSection.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 082772eb8..23253fb17 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -60,6 +60,8 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable private static final ConcurrentHashMap RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS = new ConcurrentHashMap<>(); private static final Timer RENDER_SOURCE_CACHE_REMOVAL_TIMER = TimerUtil.CreateTimer("LodRenderSection Render Source Cache Removal Timer"); + public static final long RENDER_CACHE_EXPIRATION_TIME_IN_MS = 4000L; + public final DhSectionPos pos; @@ -237,7 +239,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable }); try { - RENDER_SOURCE_CACHE_REMOVAL_TIMER.schedule(timerTask, 1000L); + RENDER_SOURCE_CACHE_REMOVAL_TIMER.schedule(timerTask, RENDER_CACHE_EXPIRATION_TIME_IN_MS); } catch (IllegalStateException ignore) { /* can rarely happen due to some minor concurrency bug with how Timer works. It isn't an issue and can be ignored. */ } From 42675abef13e15766606265cabfb94fdda984200 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 4 Apr 2024 07:09:17 -0500 Subject: [PATCH 124/183] Comment out LodRenderSection neighbor collection --- .../core/render/LodRenderSection.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 23253fb17..0952559b2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -190,13 +190,13 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable private ColumnRenderSource[] getAndCreateNeighborRenderSources() { ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; - for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) - { - DhSectionPos adjPos = this.pos.getAdjacentPos(direction); - - ColumnRenderSource renderSource = ADJACENT_RENDER_SOURCE_BY_POS.compute(adjPos, this::computeCachedRenderSource); - adjacentRenderSections[direction.ordinal() - 2] = renderSource; - } + //for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) + //{ + // DhSectionPos adjPos = this.pos.getAdjacentPos(direction); + // + // ColumnRenderSource renderSource = ADJACENT_RENDER_SOURCE_BY_POS.compute(adjPos, this::computeCachedRenderSource); + // adjacentRenderSections[direction.ordinal() - 2] = renderSource; + //} return adjacentRenderSections; } From a1a45f50bf7e978545fec2820f86290e7028b415 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 4 Apr 2024 07:50:02 -0500 Subject: [PATCH 125/183] Change Sqlite Journaling to WAL to potentially improve concurrent performance --- .../sqlScripts/0031-sqlite-useSqliteWalJournaling.sql | 8 ++++++++ core/src/main/resources/sqlScripts/scriptList.txt | 1 + 2 files changed, 9 insertions(+) create mode 100644 core/src/main/resources/sqlScripts/0031-sqlite-useSqliteWalJournaling.sql diff --git a/core/src/main/resources/sqlScripts/0031-sqlite-useSqliteWalJournaling.sql b/core/src/main/resources/sqlScripts/0031-sqlite-useSqliteWalJournaling.sql new file mode 100644 index 000000000..1c48df1c7 --- /dev/null +++ b/core/src/main/resources/sqlScripts/0031-sqlite-useSqliteWalJournaling.sql @@ -0,0 +1,8 @@ + +-- these PRAGMA's will automatically commit, so we have to disable +-- DH's automatic transactions, otherwise the connection will throw an error + +--No Transactions-- + +pragma journal_mode = WAL; +pragma synchronous = NORMAL; diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index 9899c9e0a..62901d865 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -2,4 +2,5 @@ 0010-sqlite-createInitialDataTables.sql 0020-sqlite-createFullDataSourceV2Tables.sql 0030-sqlite-changeTableJournaling.sql +0031-sqlite-useSqliteWalJournaling.sql 0040-sqlite-removeRenderCache.sql From 5ef6cb2e0d3db63f60a1b21d118014e9e28671e6 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 07:53:52 -0500 Subject: [PATCH 126/183] Fix wrong data source Version ref in SubDim Matcher --- .../core/file/subDimMatching/SubDimensionLevelMatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 3d701ff6f..5eed7a278 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -230,9 +230,9 @@ public class SubDimensionLevelMatcher implements AutoCloseable // compare the data sources int equalDataPoints = 0; int totalDataPointCount = 0; - for (int x = 0; x < FullDataSourceV1.WIDTH; x++) + for (int x = 0; x < FullDataSourceV2.WIDTH; x++) { - for (int z = 0; z < FullDataSourceV1.WIDTH; z++) + for (int z = 0; z < FullDataSourceV2.WIDTH; z++) { LongArrayList newColumn = newDataSource.get(x, z); LongArrayList testColumn = testFullDataSource.get(x, z); From ae829cbe3e915bb834cc5b933f480e252d76b0fb Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 07:54:47 -0500 Subject: [PATCH 127/183] Fix legacy migration conversion column order --- .../fullData/sources/FullDataSourceV2.java | 65 +++++++++++-------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 4e3a57031..65b7ca659 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -161,18 +161,18 @@ public class FullDataSourceV2 implements IDataSource { for (int z = 0; z < WIDTH; z++) { - long[] dataColumn = legacyData.get(x, z); - if (dataColumn != null && dataColumn.length != 0) + long[] legacyDataColumn = legacyData.get(x, z); + if (legacyDataColumn != null && legacyDataColumn.length != 0) { int index = relativePosToIndex(x, z); - dataPoints[index] = new LongArrayList(dataColumn); + LongArrayList newDataColumn = new LongArrayList(legacyDataColumn); // convert the data point format boolean columnHasNonAirBlock = false; - for (int i = 0; i < dataColumn.length; i++) + for (int i = 0; i < legacyDataColumn.length; i++) { - long dataPoint = dataColumn[i]; + long dataPoint = legacyDataColumn[i]; int id = FullDataPointUtilV1.getId(dataPoint); int height = FullDataPointUtilV1.getHeight(dataPoint); @@ -188,7 +188,7 @@ public class FullDataSourceV2 implements IDataSource } long newDataPoint = FullDataPointUtilV2.encode(id, height, bottomY, blockLight, skyLight); - dataColumn[i] = newDataPoint; + newDataColumn.set(i, newDataPoint); // check if this datapoint is air @@ -198,6 +198,11 @@ public class FullDataSourceV2 implements IDataSource } } + + // save the converted data point + ensureDataColumnOrder(newDataColumn); + dataPoints[index] = newDataColumn; + // the old data sources didn't have a generation step written down // if the column has any data points, assume it's fully generated, otherwise assume it's empty columnGenerationSteps[index] = (columnHasNonAirBlock ? EDhApiWorldGenerationStep.LIGHT.value : EDhApiWorldGenerationStep.EMPTY.value); @@ -627,30 +632,10 @@ public class FullDataSourceV2 implements IDataSource } - // flip the array if necessary // TODO why is this sometimes necessary? What did I (James) screw up that causes the mergedInputDataArray // to sometimes be in a different order? Is it potentially related to what detail level is coming in? - { - long firstDataPoint = newColumnList.getLong(0); - int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); - - long lastDataPoint = newColumnList.getLong(newColumnList.size() - 1); - int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); - - if (firstBottomY < lastBottomY) - { - // reverse the array so index 0 is the highest, - // this is necessary for later logic - // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java - for(int i = 0; i < newColumnList.size() / 2; i++) - { - long temp = newColumnList.getLong(i); - newColumnList.set(i, newColumnList.getLong(newColumnList.size() - i - 1)); - newColumnList.set(newColumnList.size() - i - 1, temp); - } - } - } + ensureDataColumnOrder(newColumnList); return newColumnList; } @@ -792,6 +777,32 @@ public class FullDataSourceV2 implements IDataSource } } + /** + * Ensures the given data column is in the correct Y order, specifically + * top-to-bottom. + */ + private static void ensureDataColumnOrder(LongArrayList dataColumn) + { + long firstDataPoint = dataColumn.getLong(0); + int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); + + long lastDataPoint = dataColumn.getLong(dataColumn.size() - 1); + int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); + + if (firstBottomY < lastBottomY) + { + // reverse the array so index 0 is the highest, + // this is necessary for later logic + // source: https://stackoverflow.com/questions/2137755/how-do-i-reverse-an-int-array-in-java + for(int i = 0; i < dataColumn.size() / 2; i++) + { + long temp = dataColumn.getLong(i); + dataColumn.set(i, dataColumn.getLong(dataColumn.size() - i - 1)); + dataColumn.set(dataColumn.size() - i - 1, temp); + } + } + } + //=========// From ddbad36d8a7f38924fa6107530e10b199ad486f1 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 08:17:09 -0500 Subject: [PATCH 128/183] Remove FullDataPointUtilV2 Not sure what happened to make me think we needed to change the data point format, but luckily we don't have to deal with it any more and can just stick with the single file. --- .../methods/data/DhApiTerrainDataRepo.java | 16 +-- .../fullData/sources/FullDataSourceV1.java | 4 +- .../fullData/sources/FullDataSourceV2.java | 41 +++--- .../FullDataToRenderDataTransformer.java | 12 +- .../transformers/LodDataBuilder.java | 8 +- .../SubDimensionLevelMatcher.java | 15 +- ...ointUtilV2.java => FullDataPointUtil.java} | 50 +++---- .../core/util/FullDataPointUtilV1.java | 133 ------------------ 8 files changed, 68 insertions(+), 211 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/util/{FullDataPointUtilV2.java => FullDataPointUtil.java} (82%) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java index d922064f4..e3f4f5210 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/data/DhApiTerrainDataRepo.java @@ -32,7 +32,7 @@ import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RayCastUtil; import com.seibel.distanthorizons.core.world.AbstractDhWorld; @@ -249,8 +249,8 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo if (dataPoint != 0) { int requestedY = nullableBlockYPos; - int bottomY = FullDataPointUtilV2.getBottomY(dataPoint) + levelMinimumHeight; - int height = FullDataPointUtilV2.getHeight(dataPoint); + int bottomY = FullDataPointUtil.getBottomY(dataPoint) + levelMinimumHeight; + int height = FullDataPointUtil.getHeight(dataPoint); int topY = bottomY + height; // does this datapoint contain the requested Y position? @@ -282,15 +282,15 @@ public class DhApiTerrainDataRepo implements IDhApiTerrainDataRepo private static DhApiTerrainDataPoint generateApiDatapoint(IDhApiLevelWrapper levelWrapper, FullDataPointIdMap mapping, byte detailLevel, long dataPoint) { - IBlockStateWrapper blockState = mapping.getBlockStateWrapper(FullDataPointUtilV2.getId(dataPoint)); - IBiomeWrapper biomeWrapper = mapping.getBiomeWrapper(FullDataPointUtilV2.getId(dataPoint)); + IBlockStateWrapper blockState = mapping.getBlockStateWrapper(FullDataPointUtil.getId(dataPoint)); + IBiomeWrapper biomeWrapper = mapping.getBiomeWrapper(FullDataPointUtil.getId(dataPoint)); - int bottomY = FullDataPointUtilV2.getBottomY(dataPoint) + levelWrapper.getMinHeight(); - int height = FullDataPointUtilV2.getHeight(dataPoint); + int bottomY = FullDataPointUtil.getBottomY(dataPoint) + levelWrapper.getMinHeight(); + int height = FullDataPointUtil.getHeight(dataPoint); int topY = bottomY + height; return new DhApiTerrainDataPoint(detailLevel, - FullDataPointUtilV2.getBlockLight(dataPoint), FullDataPointUtilV2.getSkyLight(dataPoint), + FullDataPointUtil.getBlockLight(dataPoint), FullDataPointUtil.getSkyLight(dataPoint), topY, bottomY, blockState, biomeWrapper); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java index 1505e4566..37cdf0452 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV1.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataInputStream; import com.seibel.distanthorizons.core.util.objects.dataStreams.DhDataOutputStream; @@ -43,7 +43,7 @@ import java.util.Arrays; * * Replaced by {@link FullDataSourceV2}. * - * @see FullDataPointUtilV1 + * @see FullDataPointUtil * @see FullDataSourceV2 */ public class FullDataSourceV1 implements IDataSource diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 65b7ca659..9ebb4d892 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -29,8 +29,7 @@ import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV1; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -174,11 +173,11 @@ public class FullDataSourceV2 implements IDataSource { long dataPoint = legacyDataColumn[i]; - int id = FullDataPointUtilV1.getId(dataPoint); - int height = FullDataPointUtilV1.getHeight(dataPoint); - int bottomY = FullDataPointUtilV1.getBottomY(dataPoint); - byte blockLight = (byte) FullDataPointUtilV1.getBlockLight(dataPoint); - byte skyLight = (byte) FullDataPointUtilV1.getSkyLight(dataPoint); + int id = FullDataPointUtil.getId(dataPoint); + int height = FullDataPointUtil.getHeight(dataPoint); + int bottomY = FullDataPointUtil.getBottomY(dataPoint); + byte blockLight = (byte) FullDataPointUtil.getBlockLight(dataPoint); + byte skyLight = (byte) FullDataPointUtil.getSkyLight(dataPoint); IBlockStateWrapper blockState = legacyData.mapping.getBlockStateWrapper(id); if (blockState.isAir()) @@ -187,7 +186,7 @@ public class FullDataSourceV2 implements IDataSource blockLight = 0; } - long newDataPoint = FullDataPointUtilV2.encode(id, height, bottomY, blockLight, skyLight); + long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, blockLight, skyLight); newDataColumn.set(i, newDataPoint); @@ -551,8 +550,8 @@ public class FullDataSourceV2 implements IDataSource } long datapoint = inputDataArray.getLong(dataPointIndex); - int datapointMinY = FullDataPointUtilV2.getBottomY(datapoint); - int numbOfBlocksTall = FullDataPointUtilV2.getHeight(datapoint); + int datapointMinY = FullDataPointUtil.getBottomY(datapoint); + int numbOfBlocksTall = FullDataPointUtil.getHeight(datapoint); int datapointMaxY = (datapointMinY + numbOfBlocksTall); @@ -595,9 +594,9 @@ public class FullDataSourceV2 implements IDataSource Arrays.fill(mergeSkyLights, 0); for (int i = 0; i < 4; i++) { - mergeIds[i] = FullDataPointUtilV2.getId(datapointsForYSlice[i]); - mergeBlockLights[i] = FullDataPointUtilV2.getBlockLight(datapointsForYSlice[i]); - mergeSkyLights[i] = FullDataPointUtilV2.getSkyLight(datapointsForYSlice[i]); + mergeIds[i] = FullDataPointUtil.getId(datapointsForYSlice[i]); + mergeBlockLights[i] = FullDataPointUtil.getBlockLight(datapointsForYSlice[i]); + mergeSkyLights[i] = FullDataPointUtil.getSkyLight(datapointsForYSlice[i]); } @@ -614,7 +613,7 @@ public class FullDataSourceV2 implements IDataSource { if (height != 0) { - newColumnList.add(FullDataPointUtilV2.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); } lastId = id; @@ -628,7 +627,7 @@ public class FullDataSourceV2 implements IDataSource // add the last slice if present if (height != 0) { - newColumnList.add(FullDataPointUtilV2.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); + newColumnList.add(FullDataPointUtil.encode(lastId, height, minY, lastBlockLight, lastSkyLight)); } @@ -649,7 +648,7 @@ public class FullDataSourceV2 implements IDataSource LongArrayList dataColumn = this.dataPoints[dataPointIndex]; for (int i = 0; i < dataColumn.size(); i++) { - dataColumn.set(i, FullDataPointUtilV2.remap(remappedIds, dataColumn.getLong(i))); + dataColumn.set(i, FullDataPointUtil.remap(remappedIds, dataColumn.getLong(i))); } } private static boolean areDataColumnsDifferent(long[] oldDataArray, long[] newDataArray) @@ -766,10 +765,10 @@ public class FullDataSourceV2 implements IDataSource public static void throwIfDataColumnInWrongOrder(DhSectionPos pos, LongArrayList dataArray) throws IllegalStateException { long firstDataPoint = dataArray.getLong(0); - int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); + int firstBottomY = FullDataPointUtil.getBottomY(firstDataPoint); long lastDataPoint = dataArray.getLong(dataArray.size() - 1); - int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); + int lastBottomY = FullDataPointUtil.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) { @@ -784,10 +783,10 @@ public class FullDataSourceV2 implements IDataSource private static void ensureDataColumnOrder(LongArrayList dataColumn) { long firstDataPoint = dataColumn.getLong(0); - int firstBottomY = FullDataPointUtilV2.getBottomY(firstDataPoint); + int firstBottomY = FullDataPointUtil.getBottomY(firstDataPoint); long lastDataPoint = dataColumn.getLong(dataColumn.size() - 1); - int lastBottomY = FullDataPointUtilV2.getBottomY(lastDataPoint); + int lastBottomY = FullDataPointUtil.getBottomY(lastDataPoint); if (firstBottomY < lastBottomY) { @@ -863,7 +862,7 @@ public class FullDataSourceV2 implements IDataSource for (int i = 0; i < longArray.size(); i++) { long dataPoint = longArray.getLong(i); - int id = FullDataPointUtilV2.getId(dataPoint); + int id = FullDataPointUtil.getId(dataPoint); if (id > maxValidId) { LodUtil.assertNotReach("Column set with higher than possible ID. ID [" + id + "], max valid ID [" + maxValidId + "]."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 3d7b373df..65c3636e0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -31,7 +31,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.ColorUtil; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -180,11 +180,11 @@ public class FullDataToRenderDataTransformer for (int i = 0; i < fullColumnData.size(); i++) { long fullData = fullColumnData.getLong(i); - int bottomY = FullDataPointUtilV2.getBottomY(fullData); - int blockHeight = FullDataPointUtilV2.getHeight(fullData); - int id = FullDataPointUtilV2.getId(fullData); - int blockLight = FullDataPointUtilV2.getBlockLight(fullData); - int skyLight = FullDataPointUtilV2.getSkyLight(fullData); + int bottomY = FullDataPointUtil.getBottomY(fullData); + int blockHeight = FullDataPointUtil.getHeight(fullData); + int id = FullDataPointUtil.getId(fullData); + int blockLight = FullDataPointUtil.getBlockLight(fullData); + int skyLight = FullDataPointUtil.getSkyLight(fullData); // TODO how should corrupted data be handled? // TODO why is the full data corrupted in the first place? FullDataPointUtil hasn't been changed in a long time, could one of the full data point objects be corrupted? diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java index 56d7b9363..9437ec217 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/LodDataBuilder.java @@ -33,7 +33,7 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; @@ -206,7 +206,7 @@ public class LodDataBuilder // check if this block is visible from any direction || blockVisible(chunkWrapper, relBlockX, y, relBlockZ)) { - longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); biome = newBiome; blockState = newBlockState; mappedId = dataSource.mapping.addIfNotPresentAndGetId(biome, blockState); @@ -216,7 +216,7 @@ public class LodDataBuilder } } } - longs.add(FullDataPointUtilV2.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); + longs.add(FullDataPointUtil.encode(mappedId, lastY - y, y + 1 - chunkWrapper.getMinBuildHeight(), blockLight, skyLight)); dataSource.setSingleColumn(longs, relBlockX + chunkOffsetX, @@ -318,7 +318,7 @@ public class LodDataBuilder (IBlockStateWrapper) (dataPoint.blockStateWrapper) ); - packedDataPoints.set(index, FullDataPointUtilV2.encode( + packedDataPoints.set(index, FullDataPointUtil.encode( id, dataPoint.topYBlockPos - dataPoint.bottomYBlockPos, dataPoint.bottomYBlockPos - dataPoints.topYBlockPos, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java index 5eed7a278..a07c3176a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/subDimMatching/SubDimensionLevelMatcher.java @@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.file.subDimMatching; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.structure.ClientOnlySaveStructure; @@ -32,7 +31,7 @@ import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; -import com.seibel.distanthorizons.core.util.FullDataPointUtilV2; +import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; @@ -251,13 +250,13 @@ public class SubDimensionLevelMatcher implements AutoCloseable long newDataPoint = newColumn.getLong(i); long testDataPoint = testColumn.getLong(i); - int newId = FullDataPointUtilV2.getId(newDataPoint); - int testId = FullDataPointUtilV2.getId(testDataPoint); + int newId = FullDataPointUtil.getId(newDataPoint); + int testId = FullDataPointUtil.getId(testDataPoint); // bottom Y - int newBottom = FullDataPointUtilV2.getBottomY(newDataPoint); - int testBottom = FullDataPointUtilV2.getBottomY(testDataPoint); + int newBottom = FullDataPointUtil.getBottomY(newDataPoint); + int testBottom = FullDataPointUtil.getBottomY(testDataPoint); if (newBottom == testBottom) { equalDataPoints++; @@ -265,8 +264,8 @@ public class SubDimensionLevelMatcher implements AutoCloseable totalDataPointCount++; // height - int newHeight = FullDataPointUtilV2.getHeight(newDataPoint); - int testHeight = FullDataPointUtilV2.getHeight(testDataPoint); + int newHeight = FullDataPointUtil.getHeight(newDataPoint); + int testHeight = FullDataPointUtil.getHeight(testDataPoint); if (newHeight == testHeight) { equalDataPoints++; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java similarity index 82% rename from core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java rename to core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index 491d0c256..a1eb81042 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -1,24 +1,7 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - package com.seibel.distanthorizons.core.util; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; +import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.coreapi.ModInfo; import org.jetbrains.annotations.Contract; @@ -28,6 +11,12 @@ import org.jetbrains.annotations.Contract; * A full data point contains the most information and is the * source of truth used when creating render data.

* + * Specifically used by the data sources:
+ * - {@link FullDataSourceV2}
+ * - {@link FullDataSourceV1} aka CompleteFullDataSource
+ * - (Deleted) HighDetailIncompleteFullDataSource aka SparseDataSource
+ * - (Deleted) LowDetailIncompleteFullDataSource aka SpottyDataSource

+ * * DataPoint Format:
* * ID: blockState id
@@ -37,7 +26,7 @@ import org.jetbrains.annotations.Contract; * SL: Sky light

* * =======Bit layout=======
- * BL BL BL BL SL SL SL SL <-- Top bits
+ * SL SL SL SL BL BL BL BL <-- Top bits
* MY MY MY MY MY MY MY MY
* MY MY MY MY HI HI HI HI
* HI HI HI HI HI HI HI HI
@@ -46,11 +35,11 @@ import org.jetbrains.annotations.Contract; * ID ID ID ID ID ID ID ID
* ID ID ID ID ID ID ID ID <-- Bottom bits
* - * - * @see RenderDataPointUtil - * @see FullDataPointUtilV1 + * + * @see FullDataSourceV1 + * @see FullDataSourceV2 */ -public class FullDataPointUtilV2 +public class FullDataPointUtil { public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD; @@ -60,23 +49,23 @@ public class FullDataPointUtilV2 public static final int ID_WIDTH = 32; public static final int HEIGHT_WIDTH = 12; public static final int MIN_Y_WIDTH = 12; - public static final int BLOCK_LIGHT_WIDTH = 4; public static final int SKY_LIGHT_WIDTH = 4; + public static final int BLOCK_LIGHT_WIDTH = 4; public static final int ID_OFFSET = 0; public static final int HEIGHT_OFFSET = ID_OFFSET + ID_WIDTH; /** indicates the Y position where the LOD starts relative to the level's minimum height */ public static final int MIN_Y_OFFSET = HEIGHT_OFFSET + HEIGHT_WIDTH; - public static final int BLOCK_LIGHT_OFFSET = MIN_Y_OFFSET + MIN_Y_WIDTH; - public static final int SKY_LIGHT_OFFSET = BLOCK_LIGHT_OFFSET + BLOCK_LIGHT_WIDTH; + public static final int SKY_LIGHT_OFFSET = MIN_Y_OFFSET + MIN_Y_WIDTH; + public static final int BLOCK_LIGHT_OFFSET = SKY_LIGHT_OFFSET + SKY_LIGHT_WIDTH; public static final long ID_MASK = Integer.MAX_VALUE; public static final long INVERSE_ID_MASK = ~ID_MASK; public static final int HEIGHT_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1; public static final int MIN_Y_MASK = (int) Math.pow(2, MIN_Y_WIDTH) - 1; - public static final int BLOCK_LIGHT_MASK = (int) Math.pow(2, BLOCK_LIGHT_WIDTH) - 1; public static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1; + public static final int BLOCK_LIGHT_MASK = (int) Math.pow(2, BLOCK_LIGHT_WIDTH) - 1; /** @@ -114,7 +103,7 @@ public class FullDataPointUtilV2 if (RUN_VALIDATION) { if (getId(data) != id || getHeight(data) != height || getBottomY(data) != relMinY - || getBlockLight(data) != Byte.toUnsignedInt(blockLight) || getSkyLight(data) != Byte.toUnsignedInt(skyLight)) + || getBlockLight(data) != Byte.toUnsignedInt(blockLight) || getSkyLight(data) != Byte.toUnsignedInt(skyLight)) { LodUtil.assertNotReach( "Trying to create datapoint with " + @@ -127,6 +116,7 @@ public class FullDataPointUtilV2 return data; } + /** Returns the BlockState/Biome pair ID used to identify this LOD's color */ public static int getId(long data) { return (int) (data & ID_MASK); } /** Returns how many blocks tall this LOD is. */ @@ -142,6 +132,8 @@ public class FullDataPointUtilV2 public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",BlockLight:" + getBlockLight(data) + ",SkyLight:" + getSkyLight(data) + "]"; } + + /** Remaps the biome/blockState ID of the given datapoint */ @Contract(pure = true) public static long remap(int[] newIdMapping, long data) throws IndexOutOfBoundsException diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java deleted file mode 100644 index 065d1b890..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtilV1.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.seibel.distanthorizons.core.util; - -import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; -import com.seibel.distanthorizons.coreapi.ModInfo; - -/** - * Only for Legacy support
- * Used by DH versions 2.0.0 and 2.0.1.

- * - * Specifically used by the data sources:
- * - {@link FullDataSourceV1} aka CompleteFullDataSource
- * - (Deleted) HighDetailIncompleteFullDataSource
- * - (Deleted) LowDetailIncompleteFullDataSource

- * - * DataPoint Format:
- * - * ID: blockState id
- * MY: Min Y Height (unsigned, relative to the minimum level height)
- * HI: Height (how tall this data point is in blocks)
- * BL: Block light
- * SL: Sky light

- * - * =======Bit layout=======
- * SL SL SL SL BL BL BL BL <-- Top bits
- * MY MY MY MY MY MY MY MY
- * MY MY MY MY HI HI HI HI
- * HI HI HI HI HI HI HI HI
- * ID ID ID ID ID ID ID ID
- * ID ID ID ID ID ID ID ID
- * ID ID ID ID ID ID ID ID
- * ID ID ID ID ID ID ID ID <-- Bottom bits
- * - * - * @see FullDataSourceV1 - * @see FullDataPointUtilV2 - */ -public class FullDataPointUtilV1 -{ - public static final boolean RUN_VALIDATION = ModInfo.IS_DEV_BUILD; - - /** Represents the data held by an empty data point */ - public static final int EMPTY_DATA_POINT = 0; - - public static final int ID_WIDTH = 32; - public static final int HEIGHT_WIDTH = 12; - public static final int MIN_Y_WIDTH = 12; - public static final int SKY_LIGHT_WIDTH = 4; - public static final int BLOCK_LIGHT_WIDTH = 4; - - public static final int ID_OFFSET = 0; - public static final int HEIGHT_OFFSET = ID_OFFSET + ID_WIDTH; - /** indicates the Y position where the LOD starts relative to the level's minimum height */ - public static final int MIN_Y_OFFSET = HEIGHT_OFFSET + HEIGHT_WIDTH; - public static final int SKY_LIGHT_OFFSET = MIN_Y_OFFSET + MIN_Y_WIDTH; - public static final int BLOCK_LIGHT_OFFSET = SKY_LIGHT_OFFSET + SKY_LIGHT_WIDTH; - - - public static final long ID_MASK = Integer.MAX_VALUE; - public static final long INVERSE_ID_MASK = ~ID_MASK; - public static final int HEIGHT_MASK = (int) Math.pow(2, HEIGHT_WIDTH) - 1; - public static final int MIN_Y_MASK = (int) Math.pow(2, MIN_Y_WIDTH) - 1; - public static final int SKY_LIGHT_MASK = (int) Math.pow(2, SKY_LIGHT_WIDTH) - 1; - public static final int BLOCK_LIGHT_MASK = (int) Math.pow(2, BLOCK_LIGHT_WIDTH) - 1; - - - /** - * creates a new datapoint with the given values - * @param relMinY relative to the minimum level Y value - * - * @deprecated Should not be used anymore, just here as a reference for how the data points were constructed. - */ - @Deprecated - public static long encode(int id, int height, int relMinY, byte blockLight, byte skyLight) - { - if (RUN_VALIDATION) - { - // assertions are inside if-blocks to prevent unnecessary string concatenations - if (relMinY < 0 || relMinY >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with y[" + relMinY + "] out of range!"); - } - if (height <= 0 || height >= RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with height[" + height + "] out of range!"); - } - if (relMinY + height > RenderDataPointUtil.MAX_WORLD_Y_SIZE) - { - LodUtil.assertNotReach("Trying to create datapoint with y+depth[" + (relMinY + height) + "] out of range!"); - } - } - - - long data = 0; - data |= id & ID_MASK; - data |= (long) (height & HEIGHT_MASK) << HEIGHT_OFFSET; - data |= (long) (relMinY & MIN_Y_MASK) << MIN_Y_OFFSET; - data |= (long) blockLight << BLOCK_LIGHT_OFFSET; - data |= (long) skyLight << SKY_LIGHT_OFFSET; - - - if (RUN_VALIDATION) - { - if (getId(data) != id || getHeight(data) != height || getBottomY(data) != relMinY - || getBlockLight(data) != Byte.toUnsignedInt(blockLight) || getSkyLight(data) != Byte.toUnsignedInt(skyLight)) - { - LodUtil.assertNotReach( - "Trying to create datapoint with " + - "id[" + id + "], height[" + height + "], minY[" + relMinY + "], blockLight[" + blockLight + "], skyLight[" + skyLight + "] " + - "but got " + - "id[" + getId(data) + "], height[" + getHeight(data) + "], minY[" + getBottomY(data) + "], blockLight[" + getBlockLight(data) + "], skyLight[" + getSkyLight(data) + "]!"); - } - } - - return data; - } - - - /** Returns the BlockState/Biome pair ID used to identify this LOD's color */ - public static int getId(long data) { return (int) (data & ID_MASK); } - /** Returns how many blocks tall this LOD is. */ - public static int getHeight(long data) { return (int) ((data >> HEIGHT_OFFSET) & HEIGHT_MASK); } - /** - * Returns the unsigned block position of the bottom vertices for this LOD relative to the level's minimum height. - * Should be between 0 and {@link RenderDataPointUtil#MAX_WORLD_Y_SIZE} - */ - public static int getBottomY(long data) { return (int) ((data >> MIN_Y_OFFSET) & MIN_Y_MASK); } - public static int getBlockLight(long data) { return (int) ((data >> BLOCK_LIGHT_OFFSET) & BLOCK_LIGHT_MASK); } - public static int getSkyLight(long data) { return (int) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } - - - public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",BlockLight:" + getBlockLight(data) + ",SkyLight:" + getSkyLight(data) + "]"; } - -} From f09818e564da8d17cb858172469949af9a4356e9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 08:42:29 -0500 Subject: [PATCH 129/183] Update ELodShading names and descriptions --- .../api/enums/config/ELodShading.java | 16 +++++++++++----- .../distanthorizons/core/config/Config.java | 6 +++--- .../assets/distanthorizons/lang/en_us.json | 10 +++++----- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java index b098bbab0..125a26d6f 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java @@ -25,6 +25,7 @@ package com.seibel.distanthorizons.api.enums.config; * NONE
* * @since API 1.0.0 + * @version 2024-4-6 */ public enum ELodShading { @@ -32,15 +33,20 @@ public enum ELodShading // when adding items up the API minor version // when removing items up the API major version - /** Uses Minecraft's shading for LODs */ - MINECRAFT, /** - * Simulates Minecraft's shading. + * Uses Minecraft's shading for LODs.
+ * This means if Minecraft's shading is disabled DH's shading will be as well. + */ + AUTO, + + /** + * Simulates Minecraft's shading.
* This is most useful for shaders that disable Minecraft's shading * but still require shading on LODs. */ - OLD_LIGHTING, + ENABLED, + /** LODs will have no shading */ - NONE; + DISABLED; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 62ea54776..1594afe79 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -629,12 +629,12 @@ public class Config .build(); public static ConfigEntry lodShading = new ConfigEntry.Builder() - .set(ELodShading.MINECRAFT) + .set(ELodShading.AUTO) .comment("" + "How should LODs be shaded? \n" + "\n" - + ELodShading.MINECRAFT + ": Uses the same side shading as vanilla Minecraft blocks. \n" - + ELodShading.OLD_LIGHTING + ": Simulates Minecraft's block shading for LODs. \n" + + ELodShading.AUTO + ": Uses the same side shading as vanilla Minecraft blocks. \n" + + ELodShading.FORCED + ": Simulates Minecraft's block shading for LODs. \n" + " Can be used to force LOD shading when using some shaders. \n" + ELodShading.NONE + ": All LOD sides will be rendered with the same brightness. \n" + "") diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index 89e9da5e0..f8d189706 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -903,12 +903,12 @@ "distanthorizons.config.enum.EBufferRebuildTimes.RARE": "Rare", - "distanthorizons.config.enum.ELodShading.MINECRAFT": + "distanthorizons.config.enum.ELodShading.AUTO": "Auto", - "distanthorizons.config.enum.ELodShading.OLD_LIGHTING": - "Force", - "distanthorizons.config.enum.ELodShading.NONE": - "None", + "distanthorizons.config.enum.ELodShading.ENABLED": + "Enabled", + "distanthorizons.config.enum.ELodShading.DISABLED": + "Disabled", "distanthorizons.config.enum.EUpdateBranch.STABLE": "Stable", From 8fd37d72c731986c2943ba49041d6caa17132f4a Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 09:53:28 -0500 Subject: [PATCH 130/183] Add Sqlite Write Ahead Log to Sqlite --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index fe6126d6e..eeff2dc36 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ forge*changelog.txt # Sqlite databases *.sqlite *.sqlite-journal +*.sqlite-shm +*.sqlite-wal From b4b7738aa64e23a0282994a3c9e7558766a8b40e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 10:06:03 -0500 Subject: [PATCH 131/183] Fix API enums missing "DhApi" prefix and remove unused Enums/code --- .../api/enums/config/EBufferRebuildTimes.java | 56 --- ...sToAvoid.java => EDhApiBlocksToAvoid.java} | 7 +- ...de.java => EDhApiGLErrorHandlingMode.java} | 5 +- ...fileMode.java => EDhApiGlProfileMode.java} | 5 +- ...Method.java => EDhApiGpuUploadMethod.java} | 8 +- ...lity.java => EDhApiHorizontalQuality.java} | 7 +- ...ELodShading.java => EDhApiLodShading.java} | 4 +- ...ELoggerMode.java => EDhApiLoggerMode.java} | 10 +- ...ava => EDhApiMaxHorizontalResolution.java} | 26 +- ...e.java => EDhApiServerFolderNameMode.java} | 6 +- .../api/enums/config/EDhApiUpdateBranch.java | 14 + ...erdraw.java => EDhApiVanillaOverdraw.java} | 5 +- ...uality.java => EDhApiVerticalQuality.java} | 8 +- .../api/enums/config/EGenerationPriority.java | 48 --- .../enums/config/ELightGenerationMode.java | 40 --- .../api/enums/config/EOverdrawPrevention.java | 51 --- .../api/enums/config/EUpdateBranch.java | 7 - ...tyPreset.java => EDhApiQualityPreset.java} | 5 +- ...eadPreset.java => EDhApiThreadPreset.java} | 7 +- ...ndering.java => EDhApiDebugRendering.java} | 16 +- ...ColorMode.java => EDhApiFogColorMode.java} | 7 +- ...ogDistance.java => EDhApiFogDistance.java} | 5 +- ...ogDrawMode.java => EDhApiFogDrawMode.java} | 4 +- ...EFogFalloff.java => EDhApiFogFalloff.java} | 6 +- ...xMode.java => EDhApiHeightFogMixMode.java} | 6 +- ...tFogMode.java => EDhApiHeightFogMode.java} | 8 +- ...dererMode.java => EDhApiRendererMode.java} | 10 +- ...nsparency.java => EDhApiTransparency.java} | 7 +- .../both/IDhApiWorldGenerationConfig.java | 1 - .../client/IDhApiAmbientOcclusionConfig.java | 3 - .../config/client/IDhApiDebuggingConfig.java | 4 +- .../config/client/IDhApiFarFogConfig.java | 5 +- .../config/client/IDhApiFogConfig.java | 13 +- .../config/client/IDhApiGpuBuffersConfig.java | 4 +- .../config/client/IDhApiGraphicsConfig.java | 30 +- .../config/client/IDhApiHeightFogConfig.java | 12 +- .../client/IDhApiMultiplayerConfig.java | 4 +- .../RenderModeEnabledConverter.java | 12 +- .../config/client/DhApiDebuggingConfig.java | 6 +- .../config/client/DhApiFarFogConfig.java | 6 +- .../methods/config/client/DhApiFogConfig.java | 12 +- .../config/client/DhApiGpuBuffersConfig.java | 4 +- .../config/client/DhApiGraphicsConfig.java | 37 +- .../config/client/DhApiHeightFogConfig.java | 18 +- .../config/client/DhApiMultiplayerConfig.java | 6 +- .../common/DhApiWorldGenerationConfig.java | 1 - .../core/api/internal/ClientApi.java | 15 +- .../distanthorizons/core/config/Config.java | 322 +++++++----------- .../DebugColumnConfigEventHandler.java | 16 - .../QuickRenderToggleConfigEventHandler.java | 10 +- .../RenderCacheConfigEventHandler.java | 21 +- ...rawPreventionPresetConfigEventHandler.java | 86 ----- ...RenderQualityPresetConfigEventHandler.java | 90 ++--- .../ThreadPresetConfigEventHandler.java | 124 +++---- .../core/config/gui/OpenGLConfigScreen.java | 4 +- .../bufferBuilding/ColumnRenderBuffer.java | 13 +- .../ColumnRenderBufferBuilder.java | 4 +- .../bufferBuilding/CubicLodTemplate.java | 10 +- .../render/bufferBuilding/LodQuadBuilder.java | 6 +- .../FullDataToRenderDataTransformer.java | 4 +- .../structure/ClientOnlySaveStructure.java | 4 +- .../core/level/ClientLevelModule.java | 6 +- .../core/logging/ConfigBasedLogger.java | 10 +- .../core/logging/ConfigBasedSpamLogger.java | 10 +- .../core/render/LodQuadTree.java | 6 +- .../core/render/fog/FogSettings.java | 8 +- .../core/render/fog/LodFogConfig.java | 30 +- .../core/render/glObject/GLProxy.java | 30 +- .../core/render/glObject/buffer/GLBuffer.java | 8 +- .../glObject/buffer/GLVertexBuffer.java | 8 +- .../glObject/buffer/QuadElementBuffer.java | 6 +- .../core/render/renderer/DebugRenderer.java | 12 +- .../core/render/renderer/LodRenderer.java | 10 +- .../core/render/renderer/ScreenQuad.java | 4 +- .../core/render/renderer/TestRenderer.java | 10 +- .../render/renderer/shaders/FogShader.java | 5 +- .../distanthorizons/core/util/LodUtil.java | 9 +- .../modAccessor/AbstractOptifineAccessor.java | 10 +- .../modAccessor/IOptifineAccessor.java | 4 +- .../assets/distanthorizons/lang/en_us.json | 244 ++++++------- 80 files changed, 671 insertions(+), 1054 deletions(-) delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBufferRebuildTimes.java rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EBlocksToAvoid.java => EDhApiBlocksToAvoid.java} (87%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EGLErrorHandlingMode.java => EDhApiGLErrorHandlingMode.java} (91%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EGlProfileMode.java => EDhApiGlProfileMode.java} (92%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EGpuUploadMethod.java => EDhApiGpuUploadMethod.java} (92%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EHorizontalQuality.java => EDhApiHorizontalQuality.java} (91%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{ELodShading.java => EDhApiLodShading.java} (96%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{ELoggerMode.java => EDhApiLoggerMode.java} (92%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EMaxHorizontalResolution.java => EDhApiMaxHorizontalResolution.java} (80%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EServerFolderNameMode.java => EDhApiServerFolderNameMode.java} (95%) create mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiUpdateBranch.java rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EVanillaOverdraw.java => EDhApiVanillaOverdraw.java} (95%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/{EVerticalQuality.java => EDhApiVerticalQuality.java} (91%) delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGenerationPriority.java delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELightGenerationMode.java delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EOverdrawPrevention.java delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EUpdateBranch.java rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/{EQualityPreset.java => EDhApiQualityPreset.java} (94%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/{EThreadPreset.java => EDhApiThreadPreset.java} (92%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EDebugRendering.java => EDhApiDebugRendering.java} (86%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EFogColorMode.java => EDhApiFogColorMode.java} (95%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EFogDistance.java => EDhApiFogDistance.java} (94%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EFogDrawMode.java => EDhApiFogDrawMode.java} (96%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EFogFalloff.java => EDhApiFogFalloff.java} (93%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EHeightFogMixMode.java => EDhApiHeightFogMixMode.java} (94%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{EHeightFogMode.java => EDhApiHeightFogMode.java} (91%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{ERendererMode.java => EDhApiRendererMode.java} (87%) rename api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/{ETransparency.java => EDhApiTransparency.java} (89%) delete mode 100644 core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/OverdrawPreventionPresetConfigEventHandler.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBufferRebuildTimes.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBufferRebuildTimes.java deleted file mode 100644 index b34c005c7..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBufferRebuildTimes.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.api.enums.config; - -/** - * CONSTANT
- * FREQUENT
- * NORMAL
- * RARE

- * - * Determines how fast the buffers should be regenerated - * - * @author Leonardo Amato - * @version 9-25-2021 - */ -@Deprecated // not currently in use, if the config this enum represents is re-implemented, the deprecated flag can be removed -public enum EBufferRebuildTimes -{ - CONSTANT(0, 0, 0, 1), - - FREQUENT(1000, 500, 2500, 1), - - NORMAL(2000, 1000, 5000, 4), - - RARE(5000, 2000, 10000, 16); - - public final int playerMoveTimeout; - public final int renderedChunkTimeout; - public final int chunkChangeTimeout; - public final int playerMoveDistance; - - EBufferRebuildTimes(int playerMoveTimeout, int renderedChunkTimeout, int chunkChangeTimeout, int playerMoveDistance) - { - this.playerMoveTimeout = playerMoveTimeout; - this.renderedChunkTimeout = renderedChunkTimeout; - this.chunkChangeTimeout = chunkChangeTimeout; - this.playerMoveDistance = playerMoveDistance; - } -} diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBlocksToAvoid.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiBlocksToAvoid.java similarity index 87% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBlocksToAvoid.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiBlocksToAvoid.java index 2f26d3d71..92388135d 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EBlocksToAvoid.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiBlocksToAvoid.java @@ -23,9 +23,10 @@ package com.seibel.distanthorizons.api.enums.config; * NONE,
* NON_COLLIDING,
* - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EBlocksToAvoid +public enum EDhApiBlocksToAvoid { // Reminder: // when adding items up the API minor version @@ -36,6 +37,6 @@ public enum EBlocksToAvoid public final boolean noCollision; - EBlocksToAvoid(boolean noCollision) { this.noCollision = noCollision; } + EDhApiBlocksToAvoid(boolean noCollision) { this.noCollision = noCollision; } } \ No newline at end of file diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGLErrorHandlingMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGLErrorHandlingMode.java similarity index 91% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGLErrorHandlingMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGLErrorHandlingMode.java index 00719a140..5e5013613 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGLErrorHandlingMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGLErrorHandlingMode.java @@ -20,9 +20,10 @@ package com.seibel.distanthorizons.api.enums.config; /** - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EGLErrorHandlingMode +public enum EDhApiGLErrorHandlingMode { IGNORE, LOG, diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGlProfileMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGlProfileMode.java similarity index 92% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGlProfileMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGlProfileMode.java index b4247d66f..3e383e758 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGlProfileMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGlProfileMode.java @@ -20,9 +20,10 @@ package com.seibel.distanthorizons.api.enums.config; /** - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EGlProfileMode +public enum EDhApiGlProfileMode { CORE, COMPAT, diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGpuUploadMethod.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java similarity index 92% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGpuUploadMethod.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java index d87d8738d..8f3f5ec9c 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGpuUploadMethod.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGpuUploadMethod.java @@ -28,10 +28,10 @@ package com.seibel.distanthorizons.api.enums.config; * * @author Leetom * @author James Seibel - * @version 2022-7-2 - * @since API 1.0.0 + * @version 2024-4-6 + * @since API 1.1.0 */ -public enum EGpuUploadMethod +public enum EDhApiGpuUploadMethod { /** Picks the best option based on the GPU the user has. */ AUTO(false, false), @@ -60,7 +60,7 @@ public enum EGpuUploadMethod public final boolean useEarlyMapping; public final boolean useBufferStorage; - EGpuUploadMethod(boolean useEarlyMapping, boolean useBufferStorage) + EDhApiGpuUploadMethod(boolean useEarlyMapping, boolean useBufferStorage) { this.useEarlyMapping = useEarlyMapping; this.useBufferStorage = useBufferStorage; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java similarity index 91% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java index b180498d5..5ac005504 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EHorizontalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java @@ -26,9 +26,10 @@ package com.seibel.distanthorizons.api.enums.config; * HIGH
* UNLIMITED
* - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EHorizontalQuality +public enum EDhApiHorizontalQuality { // Reminder: // when adding items up the API minor version @@ -55,7 +56,7 @@ public enum EHorizontalQuality public final double quadraticBase; public final int distanceUnitInBlocks; - EHorizontalQuality(double quadraticBase, int distanceUnitInBlocks) + EDhApiHorizontalQuality(double quadraticBase, int distanceUnitInBlocks) { this.quadraticBase = quadraticBase; this.distanceUnitInBlocks = distanceUnitInBlocks; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java similarity index 96% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java index 125a26d6f..bfab3f7dc 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELodShading.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java @@ -24,10 +24,10 @@ package com.seibel.distanthorizons.api.enums.config; * OLD_LIGHTING
* NONE
* - * @since API 1.0.0 + * @since API 1.1.0 * @version 2024-4-6 */ -public enum ELodShading +public enum EDhApiLodShading { // Reminder: // when adding items up the API minor version diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELoggerMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLoggerMode.java similarity index 92% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELoggerMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLoggerMode.java index 3a16c88ed..eb9043a2e 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELoggerMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLoggerMode.java @@ -22,9 +22,10 @@ package com.seibel.distanthorizons.api.enums.config; import org.apache.logging.log4j.Level; /** - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum ELoggerMode +public enum EDhApiLoggerMode { DISABLED(Level.OFF, Level.OFF), LOG_ALL_TO_FILE(Level.ALL, Level.OFF), @@ -40,11 +41,14 @@ public enum ELoggerMode LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE(Level.INFO, Level.WARN), LOG_ERROR_TO_CHAT_AND_INFO_TO_FILE(Level.INFO, Level.ERROR), ; + public final Level levelForFile; public final Level levelForChat; - ELoggerMode(Level levelForFile, Level levelForChat) + + EDhApiLoggerMode(Level levelForFile, Level levelForChat) { this.levelForFile = levelForFile; this.levelForChat = levelForChat; } + } diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EMaxHorizontalResolution.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java similarity index 80% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EMaxHorizontalResolution.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java index e9139c20b..922932c1b 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EMaxHorizontalResolution.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiMaxHorizontalResolution.java @@ -33,10 +33,10 @@ import com.seibel.distanthorizons.coreapi.util.MathUtil; * * @author James Seibel * @author Leonardo Amato - * @version 2023-6-14 - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EMaxHorizontalResolution +public enum EDhApiMaxHorizontalResolution { /** render 256 LODs for each chunk */ BLOCK(16, 0), @@ -84,12 +84,12 @@ public enum EMaxHorizontalResolution * 2nd dimension: An array of all LodDetails that are less than or
* equal to that detailLevel */ - private static EMaxHorizontalResolution[][] lowerDetailArrays; + private static EDhApiMaxHorizontalResolution[][] lowerDetailArrays; - EMaxHorizontalResolution(int newLengthCount, int newDetailLevel) + EDhApiMaxHorizontalResolution(int newLengthCount, int newDetailLevel) { this.detailLevel = (byte) newDetailLevel; this.dataPointLengthCount = newLengthCount; @@ -129,20 +129,20 @@ public enum EMaxHorizontalResolution * Returns an array of all LodDetails that have a detail level * that is less than or equal to the given LodDetail */ - public static EMaxHorizontalResolution[] getSelfAndLowerDetails(EMaxHorizontalResolution detail) + public static EDhApiMaxHorizontalResolution[] getSelfAndLowerDetails(EDhApiMaxHorizontalResolution detail) { if (lowerDetailArrays == null) { // run first time setup - lowerDetailArrays = new EMaxHorizontalResolution[EMaxHorizontalResolution.values().length][]; + lowerDetailArrays = new EDhApiMaxHorizontalResolution[EDhApiMaxHorizontalResolution.values().length][]; // go through each LodDetail - for (EMaxHorizontalResolution currentDetail : EMaxHorizontalResolution.values()) + for (EDhApiMaxHorizontalResolution currentDetail : EDhApiMaxHorizontalResolution.values()) { - ArrayList lowerDetails = new ArrayList<>(); + ArrayList lowerDetails = new ArrayList<>(); // find the details lower than currentDetail - for (EMaxHorizontalResolution compareDetail : EMaxHorizontalResolution.values()) + for (EDhApiMaxHorizontalResolution compareDetail : EDhApiMaxHorizontalResolution.values()) { if (currentDetail.detailLevel <= compareDetail.detailLevel) { @@ -154,7 +154,7 @@ public enum EMaxHorizontalResolution Collections.sort(lowerDetails); Collections.reverse(lowerDetails); - lowerDetailArrays[currentDetail.detailLevel] = lowerDetails.toArray(new EMaxHorizontalResolution[lowerDetails.size()]); + lowerDetailArrays[currentDetail.detailLevel] = lowerDetails.toArray(new EDhApiMaxHorizontalResolution[lowerDetails.size()]); } } @@ -162,9 +162,9 @@ public enum EMaxHorizontalResolution } /** Returns what detail level should be used at a given distance and maxDistance. */ - public static EMaxHorizontalResolution getDetailForDistance(EMaxHorizontalResolution maxDetailLevel, int distance, int maxDistance) + public static EDhApiMaxHorizontalResolution getDetailForDistance(EDhApiMaxHorizontalResolution maxDetailLevel, int distance, int maxDistance) { - EMaxHorizontalResolution[] lowerDetails = getSelfAndLowerDetails(maxDetailLevel); + EDhApiMaxHorizontalResolution[] lowerDetails = getSelfAndLowerDetails(maxDetailLevel); int distanceBetweenDetails = maxDistance / lowerDetails.length; int index = MathUtil.clamp(0, distance / distanceBetweenDetails, lowerDetails.length - 1); diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiServerFolderNameMode.java similarity index 95% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiServerFolderNameMode.java index de7e9dcec..69a8ea619 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EServerFolderNameMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiServerFolderNameMode.java @@ -30,10 +30,10 @@ package com.seibel.distanthorizons.api.enums.config; * Determines how the multiplayer folders should be named. * * @author James Seibel - * @version 2024-4-2 - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EServerFolderNameMode +public enum EDhApiServerFolderNameMode { // Reminder: // when adding items up the API minor version diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiUpdateBranch.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiUpdateBranch.java new file mode 100644 index 000000000..e7eab618f --- /dev/null +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiUpdateBranch.java @@ -0,0 +1,14 @@ +package com.seibel.distanthorizons.api.enums.config; + +/** + * STABLE,
+ * NIGHTLY,

+ * + * @since API 1.1.0 + * @version 2024-4-6 + */ +public enum EDhApiUpdateBranch +{ + STABLE, + NIGHTLY +} diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVanillaOverdraw.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiVanillaOverdraw.java similarity index 95% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVanillaOverdraw.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiVanillaOverdraw.java index c9f9124ab..ed321d493 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVanillaOverdraw.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiVanillaOverdraw.java @@ -28,10 +28,11 @@ package com.seibel.distanthorizons.api.enums.config; * the vanilla Minecraft terrain. * * @author James Seibel - * @version 2022-6-30 + * @since API 1.1.0 + * @version 2024-4-6 */ @Deprecated // not currently in use, if the config this enum represents is re-implemented, the deprecated flag can be removed -public enum EVanillaOverdraw +public enum EDhApiVanillaOverdraw { // Reminder: // when adding items up the API minor version diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiVerticalQuality.java similarity index 91% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiVerticalQuality.java index 6f2d89040..0917ab82b 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EVerticalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiVerticalQuality.java @@ -29,10 +29,10 @@ import com.seibel.distanthorizons.coreapi.util.MathUtil; * EXTREME
* * @author Leonardo Amato - * @version 2023-2-5 - * @since API 1.0.0 + * @version 2024-4-6 + * @since API 1.1.0 */ -public enum EVerticalQuality +public enum EDhApiVerticalQuality { HEIGHT_MAP( new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}), LOW( new int[]{4, 3, 3, 2, 2, 1, 1, 1, 1, 1, 1}), @@ -47,7 +47,7 @@ public enum EVerticalQuality - EVerticalQuality(int[] maxVerticalData) { this.maxVerticalData = maxVerticalData; } + EDhApiVerticalQuality(int[] maxVerticalData) { this.maxVerticalData = maxVerticalData; } diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGenerationPriority.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGenerationPriority.java deleted file mode 100644 index 39271c3ee..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EGenerationPriority.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.api.enums.config; - -/** - * AUTO
- * Near_First
- * Far_First

- * - * Determines which LODs should have priority when generating - * outside the normal view distance. - * - * @author Leonardo Amato - * @version 12-1-2021 - */ -@Deprecated // not currently in use, if the config this enum represents is re-implemented, the deprecated flag can be removed -public enum EGenerationPriority -{ - // Reminder: - // when adding items up the API minor version - // when removing items up the API major version - - /** NEAR_FIRST when connected to servers and BALANCED when on single player */ - AUTO, - - NEAR_FIRST, - - BALANCED, - - FAR_FIRST -} diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELightGenerationMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELightGenerationMode.java deleted file mode 100644 index cc57d08b7..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/ELightGenerationMode.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.api.enums.config; - -/** - * DISTANT_HORIZONS,
- * MINECRAFT, - * - * @author Leetom - * @version 2023-6-7 - * @since API 1.0.0 - */ -public enum ELightGenerationMode -{ - // Reminder: - // when adding items up the API minor version - // when removing items up the API major version - - DISTANT_HORIZONS, - - MINECRAFT - -} diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EOverdrawPrevention.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EOverdrawPrevention.java deleted file mode 100644 index 46f82868b..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EOverdrawPrevention.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.api.enums.config; - -/** - * NONE
- * LIGHT
- * MEDIUM
- * HEAVY
- * - * CUSTOM
- * - * @since API 1.0.0 - * @deprecated will be removed when DH updates to MC 1.21
- * After removal a float value will be used to control overdraw instead. - */ -@Deprecated -public enum EOverdrawPrevention -{ - // Reminder: - // when adding items up the API minor version - // when removing items up the API major version - - NONE, - LIGHT, - MEDIUM, - HEAVY, - - /** - * Should not be passed in.
- * Is returned if the overdraw value doesn't match any of the enums defined here. - */ - CUSTOM; -} diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EUpdateBranch.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EUpdateBranch.java deleted file mode 100644 index 09da806d5..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EUpdateBranch.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.seibel.distanthorizons.api.enums.config; - -public enum EUpdateBranch -{ - STABLE, - NIGHTLY -} diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EQualityPreset.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EDhApiQualityPreset.java similarity index 94% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EQualityPreset.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EDhApiQualityPreset.java index 7936f3176..7bca56868 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EQualityPreset.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EDhApiQualityPreset.java @@ -30,9 +30,10 @@ import com.seibel.distanthorizons.api.enums.config.DisallowSelectingViaConfigGui * HIGH,
* EXTREME,
* - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EQualityPreset +public enum EDhApiQualityPreset { // Reminder: // when adding items up the API minor version diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EThreadPreset.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EDhApiThreadPreset.java similarity index 92% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EThreadPreset.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EDhApiThreadPreset.java index 8773ff1a4..b10468e4f 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EThreadPreset.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/quickOptions/EDhApiThreadPreset.java @@ -29,9 +29,10 @@ import com.seibel.distanthorizons.api.enums.config.DisallowSelectingViaConfigGui * BALANCED,
* AGGRESSIVE,
* - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum EThreadPreset +public enum EDhApiThreadPreset { // Reminder: // when adding items up the API minor version @@ -44,8 +45,6 @@ public enum EThreadPreset LOW_IMPACT, BALANCED, AGGRESSIVE, - - // temporarily removed due to stability concerns I_PAID_FOR_THE_WHOLE_CPU, } \ No newline at end of file diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDebugRendering.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java similarity index 86% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDebugRendering.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java index 62ee27786..646f9f25a 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDebugRendering.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiDebugRendering.java @@ -28,10 +28,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * * @author Leetom * @author James Seibel - * @version 2023-6-7 - * @since API 1.0.0 + * @version 2024-4-6 + * @since API 1.1.0 */ -public enum EDebugRendering +public enum EDhApiDebugRendering { // Reminder: // when adding items up the API minor version @@ -44,10 +44,6 @@ public enum EDebugRendering /** LOD colors are based on their detail */ SHOW_DETAIL, - @Deprecated - /** LOD colors are based on their gen mode. */ - SHOW_GENMODE, - /** Block Materials are often used by Iris shaders to determine how LODs should be rendered */ SHOW_BLOCK_MATERIAL, @@ -58,7 +54,7 @@ public enum EDebugRendering SHOW_RENDER_SOURCE_FLAG; - public static EDebugRendering next(EDebugRendering type) + public static EDhApiDebugRendering next(EDhApiDebugRendering type) { switch (type) { @@ -75,7 +71,7 @@ public enum EDebugRendering } } - public static EDebugRendering previous(EDebugRendering type) + public static EDhApiDebugRendering previous(EDhApiDebugRendering type) { switch (type) { @@ -84,8 +80,6 @@ public enum EDebugRendering case SHOW_RENDER_SOURCE_FLAG: return SHOW_OVERLAPPING_QUADS; case SHOW_OVERLAPPING_QUADS: - return SHOW_GENMODE; - case SHOW_GENMODE: return SHOW_DETAIL; default: return OFF; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogColorMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogColorMode.java similarity index 95% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogColorMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogColorMode.java index ab0831d49..8900574c5 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogColorMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogColorMode.java @@ -24,10 +24,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * USE_SKY_COLOR,
* * @author James Seibel - * @version 2022-6-9 - * @since API 1.0.0 + * @version 2024-4-6 + * @since API 1.1.0 */ -public enum EFogColorMode +public enum EDhApiFogColorMode { // Reminder: // when adding items: up the API minor version @@ -44,4 +44,5 @@ public enum EFogColorMode * https://www.curseforge.com/minecraft/mc-mods/clear-skies-forge-port */ USE_SKY_COLOR, + } diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogDistance.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java similarity index 94% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogDistance.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java index 5196ad696..d7b585802 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogDistance.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java @@ -26,9 +26,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * * @author James Seibel * @version 2022-6-2 - * @since API 1.0.0 + * @since API 1.1.0 */ -public enum EFogDistance +@Deprecated +public enum EDhApiFogDistance { // Reminder: // when adding items up the API minor version diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogDrawMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDrawMode.java similarity index 96% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogDrawMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDrawMode.java index 83c732142..1bc322109 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogDrawMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDrawMode.java @@ -25,10 +25,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * FOG_DISABLED
* * @author James Seibel + * @since API 1.1.0 * @version 2022-6-2 - * @since API 1.0.0 */ -public enum EFogDrawMode +public enum EDhApiFogDrawMode { // Reminder: // when adding items up the API minor version diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogFalloff.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogFalloff.java similarity index 93% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogFalloff.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogFalloff.java index c1de07ece..4f310bf17 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EFogFalloff.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogFalloff.java @@ -26,9 +26,9 @@ package com.seibel.distanthorizons.api.enums.rendering; * * @author Leetom * @version 2022-6-30 - * @since API 1.0.0 + * @since API 1.1.0 */ -public enum EFogFalloff +public enum EDhApiFogFalloff { // Reminder: // when adding items up the API minor version @@ -38,5 +38,5 @@ public enum EFogFalloff LINEAR, EXPONENTIAL, EXPONENTIAL_SQUARED, - // TEXTURE_BASED, // TODO: Impl this + } diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EHeightFogMixMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiHeightFogMixMode.java similarity index 94% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EHeightFogMixMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiHeightFogMixMode.java index b688ebda8..13ab79dbb 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EHeightFogMixMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiHeightFogMixMode.java @@ -32,10 +32,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * AVERAGE
* * @author Leetom - * @version 2022-4-14 - * @since API 1.0.0 + * @version 2024-4-6 + * @since API 1.1.0 */ -public enum EHeightFogMixMode +public enum EDhApiHeightFogMixMode { BASIC, IGNORE_HEIGHT, diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EHeightFogMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiHeightFogMode.java similarity index 91% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EHeightFogMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiHeightFogMode.java index 5aebcc8cd..99e538603 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EHeightFogMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiHeightFogMode.java @@ -28,10 +28,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * ABOVE_AND_BELOW_SET_HEIGHT,
* * @author Leetom - * @version 6-30-2022 - * @since API 1.0.0 + * @version 2024-4-6 + * @since API 1.1.0 */ -public enum EHeightFogMode +public enum EDhApiHeightFogMode { // Reminder: // when adding items up the API minor version @@ -49,7 +49,7 @@ public enum EHeightFogMode public final boolean above; public final boolean below; - EHeightFogMode(boolean basedOnCamera, boolean above, boolean below) + EDhApiHeightFogMode(boolean basedOnCamera, boolean above, boolean below) { this.basedOnCamera = basedOnCamera; this.above = above; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/ERendererMode.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiRendererMode.java similarity index 87% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/ERendererMode.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiRendererMode.java index a6a9c6c85..87a7099b1 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/ERendererMode.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiRendererMode.java @@ -24,10 +24,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * Debug
* Disabled
* - * @version 2022-6-2 - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum ERendererMode +public enum EDhApiRendererMode { // Reminder: // when adding items up the API minor version @@ -40,7 +40,7 @@ public enum ERendererMode /** Used by the config GUI to cycle through the available rendering options */ - public static ERendererMode next(ERendererMode type) + public static EDhApiRendererMode next(EDhApiRendererMode type) { switch (type) { @@ -54,7 +54,7 @@ public enum ERendererMode } /** Used by the config GUI to cycle through the available rendering options */ - public static ERendererMode previous(ERendererMode type) + public static EDhApiRendererMode previous(EDhApiRendererMode type) { switch (type) { diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/ETransparency.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiTransparency.java similarity index 89% rename from api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/ETransparency.java rename to api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiTransparency.java index aefc14473..495c8fe6e 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/ETransparency.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiTransparency.java @@ -24,9 +24,10 @@ package com.seibel.distanthorizons.api.enums.rendering; * FAKE,
* COMPLETE,
* - * @since API 1.0.0 + * @since API 1.1.0 + * @version 2024-4-6 */ -public enum ETransparency +public enum EDhApiTransparency { // Reminder: // when adding items up the API minor version @@ -40,7 +41,7 @@ public enum ETransparency public final boolean transparencyEnabled; public final boolean fakeTransparencyEnabled; - ETransparency(boolean transparencyEnabled, boolean fakeTransparencyEnabled) + EDhApiTransparency(boolean transparencyEnabled, boolean fakeTransparencyEnabled) { this.transparencyEnabled = transparencyEnabled; this.fakeTransparencyEnabled = fakeTransparencyEnabled; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/both/IDhApiWorldGenerationConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/both/IDhApiWorldGenerationConfig.java index d52733634..9ea48a2ac 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/both/IDhApiWorldGenerationConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/both/IDhApiWorldGenerationConfig.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.api.interfaces.config.both; -import com.seibel.distanthorizons.api.enums.config.ELightGenerationMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGeneratorMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiAmbientOcclusionConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiAmbientOcclusionConfig.java index d20bac76e..8be9af11a 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiAmbientOcclusionConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiAmbientOcclusionConfig.java @@ -19,9 +19,6 @@ package com.seibel.distanthorizons.api.interfaces.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogColorMode; -import com.seibel.distanthorizons.api.enums.rendering.EFogDistance; -import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiDebuggingConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiDebuggingConfig.java index 18f2aea84..ad45b6d61 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiDebuggingConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiDebuggingConfig.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.api.interfaces.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; @@ -33,7 +33,7 @@ import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; public interface IDhApiDebuggingConfig extends IDhApiConfigGroup { /** Can be used to debug the standard fake chunk rendering. */ - IDhApiConfigValue debugRendering(); + IDhApiConfigValue debugRendering(); /** If enabled debug keybindings can be used. */ IDhApiConfigValue debugKeybindings(); diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFarFogConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFarFogConfig.java index 95f0d0a3d..e50e76cff 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFarFogConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFarFogConfig.java @@ -19,8 +19,7 @@ package com.seibel.distanthorizons.api.interfaces.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogFalloff; -import com.seibel.distanthorizons.api.enums.rendering.*; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogFalloff; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; @@ -64,7 +63,7 @@ public interface IDhApiFarFogConfig extends IDhApiConfigGroup IDhApiConfigValue farFogMaxThickness(); /** Defines how the fog changes in thickness. */ - IDhApiConfigValue farFogFalloff(); + IDhApiConfigValue farFogFalloff(); /** Defines the fog density. */ IDhApiConfigValue farFogDensity(); diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java index f7ad27c84..e976ef8ad 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java @@ -19,10 +19,9 @@ package com.seibel.distanthorizons.api.interfaces.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogColorMode; -import com.seibel.distanthorizons.api.enums.rendering.EFogDistance; -import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; -import com.seibel.distanthorizons.api.enums.rendering.*; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogColorMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDistance; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; @@ -59,13 +58,13 @@ public interface IDhApiFogConfig extends IDhApiConfigGroup //====================// /** Defines at what distance fog is rendered on fake chunks. */ - IDhApiConfigValue distance(); + IDhApiConfigValue distance(); /** Should be used to enable/disable fog rendering. */ - IDhApiConfigValue drawMode(); + IDhApiConfigValue drawMode(); /** Can be used to enable support with mods that change vanilla MC's fog color. */ - IDhApiConfigValue color(); + IDhApiConfigValue color(); /** * If enabled attempts to disable vanilla MC's fog on real chunks.
diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGpuBuffersConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGpuBuffersConfig.java index 56f6febc7..4b0433f7a 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGpuBuffersConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGpuBuffersConfig.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.api.interfaces.config.client; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; @@ -34,7 +34,7 @@ public interface IDhApiGpuBuffersConfig extends IDhApiConfigGroup { /** Defines how geometry data is uploaded to the GPU. */ - IDhApiConfigValue gpuUploadMethod(); + IDhApiConfigValue gpuUploadMethod(); /** * Defines how long we should wait after uploading one diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGraphicsConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGraphicsConfig.java index eae828efc..eaf51d691 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGraphicsConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGraphicsConfig.java @@ -20,9 +20,8 @@ package com.seibel.distanthorizons.api.interfaces.config.client; import com.seibel.distanthorizons.api.enums.config.*; -import com.seibel.distanthorizons.api.enums.config.*; -import com.seibel.distanthorizons.api.enums.rendering.ERendererMode; -import com.seibel.distanthorizons.api.enums.rendering.ETransparency; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiRendererMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiTransparency; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; @@ -68,7 +67,7 @@ public interface IDhApiGraphicsConfig extends IDhApiConfigGroup * * Changing this config also changes {@link IDhApiGraphicsConfig#renderingEnabled()}'s value. */ - IDhApiConfigValue renderingMode(); + IDhApiConfigValue renderingMode(); @@ -77,18 +76,18 @@ public interface IDhApiGraphicsConfig extends IDhApiConfigGroup //==================// /** Defines how detailed fake chunks are in the horizontal direction */ - IDhApiConfigValue maxHorizontalResolution(); + IDhApiConfigValue maxHorizontalResolution(); /** Defines how detailed fake chunks are in the vertical direction */ - IDhApiConfigValue verticalQuality(); + IDhApiConfigValue verticalQuality(); /** Modifies the quadratic function fake chunks use for horizontal quality drop-off. */ - IDhApiConfigValue horizontalQuality(); + IDhApiConfigValue horizontalQuality(); - IDhApiConfigValue transparency(); + IDhApiConfigValue transparency(); /** Defines what blocks won't be rendered as LODs. */ - IDhApiConfigValue blocksToAvoid(); + IDhApiConfigValue blocksToAvoid(); /** * Defines if the color of avoided blocks will color the block below them.
@@ -112,17 +111,6 @@ public interface IDhApiGraphicsConfig extends IDhApiConfigGroup // advanced graphic settings // //===========================// - /** - * Sets the distance used by the near clip plane to reduce - * overdraw.
- * Disabling this reduces holes in the world due to the near clip plane - * being too close to the camera and the terrain not being covered by vanilla terrain. - * - * @deprecated Use {@link IDhApiGraphicsConfig#overdrawPreventionRadius()} instead. - */ - @Deprecated - IDhApiConfigValue overdrawPrevention(); - /** * Sets the radius used by the near clip shader to reduce * overdraw.
@@ -172,7 +160,7 @@ public interface IDhApiGraphicsConfig extends IDhApiConfigGroup * * @since API 1.1.0 */ - IDhApiConfigValue lodShading(); + IDhApiConfigValue lodShading(); /** * Sets whether LODs outside the view frustum culling will diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiHeightFogConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiHeightFogConfig.java index 4e1c98753..754cc01a9 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiHeightFogConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiHeightFogConfig.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.api.interfaces.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogFalloff; -import com.seibel.distanthorizons.api.enums.rendering.EHeightFogMixMode; -import com.seibel.distanthorizons.api.enums.rendering.EHeightFogMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogFalloff; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiHeightFogMixMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiHeightFogMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; @@ -40,10 +40,10 @@ public interface IDhApiHeightFogConfig extends IDhApiConfigGroup { /** Defines how the height fog mixes. */ - IDhApiConfigValue heightFogMixMode(); + IDhApiConfigValue heightFogMixMode(); /** Defines how the height fog is drawn relative to the camera or world. */ - IDhApiConfigValue heightFogMode(); + IDhApiConfigValue heightFogMode(); /** * Defines the height fog's base height if {@link IDhApiHeightFogConfig#heightFogMode()} @@ -64,7 +64,7 @@ public interface IDhApiHeightFogConfig extends IDhApiConfigGroup IDhApiConfigValue heightFogMaxThickness(); /** Defines how the height fog changes in thickness. */ - IDhApiConfigValue heightFogFalloff(); + IDhApiConfigValue heightFogFalloff(); /** Defines the height fog's density. */ IDhApiConfigValue heightFogDensity(); diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiMultiplayerConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiMultiplayerConfig.java index 39edf49a5..20397f392 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiMultiplayerConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiMultiplayerConfig.java @@ -20,7 +20,7 @@ package com.seibel.distanthorizons.api.interfaces.config.client; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; -import com.seibel.distanthorizons.api.enums.config.EServerFolderNameMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiServerFolderNameMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; /** @@ -37,7 +37,7 @@ public interface IDhApiMultiplayerConfig extends IDhApiConfigGroup * Defines how multiplayer server folders are named.
* Note: Changing this while connected to a multiplayer world will cause undefined behavior! */ - IDhApiConfigValue folderSavingMode(); + IDhApiConfigValue folderSavingMode(); /** * Defines the necessary similarity (as a percent) that two potential levels diff --git a/api/src/main/java/com/seibel/distanthorizons/coreapi/util/converters/RenderModeEnabledConverter.java b/api/src/main/java/com/seibel/distanthorizons/coreapi/util/converters/RenderModeEnabledConverter.java index 2bb47acfe..207d3c8b0 100644 --- a/api/src/main/java/com/seibel/distanthorizons/coreapi/util/converters/RenderModeEnabledConverter.java +++ b/api/src/main/java/com/seibel/distanthorizons/coreapi/util/converters/RenderModeEnabledConverter.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.coreapi.util.converters; -import com.seibel.distanthorizons.api.enums.rendering.ERendererMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiRendererMode; import com.seibel.distanthorizons.coreapi.interfaces.config.IConverter; /** @@ -28,17 +28,17 @@ import com.seibel.distanthorizons.coreapi.interfaces.config.IConverter; * @author James Seibel * @version 2022-6-30 */ -public class RenderModeEnabledConverter implements IConverter +public class RenderModeEnabledConverter implements IConverter { - @Override public ERendererMode convertToCoreType(Boolean renderingEnabled) + @Override public EDhApiRendererMode convertToCoreType(Boolean renderingEnabled) { - return renderingEnabled ? ERendererMode.DEFAULT : ERendererMode.DISABLED; + return renderingEnabled ? EDhApiRendererMode.DEFAULT : EDhApiRendererMode.DISABLED; } - @Override public Boolean convertToApiType(ERendererMode renderingMode) + @Override public Boolean convertToApiType(EDhApiRendererMode renderingMode) { - return renderingMode == ERendererMode.DEFAULT; + return renderingMode == EDhApiRendererMode.DEFAULT; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiDebuggingConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiDebuggingConfig.java index e34b964b9..925dfc3da 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiDebuggingConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiDebuggingConfig.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiDebuggingConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; public class DhApiDebuggingConfig implements IDhApiDebuggingConfig { @@ -33,8 +33,8 @@ public class DhApiDebuggingConfig implements IDhApiDebuggingConfig - public IDhApiConfigValue debugRendering() - { return new DhApiConfigValue(Config.Client.Advanced.Debugging.debugRendering); } + public IDhApiConfigValue debugRendering() + { return new DhApiConfigValue(Config.Client.Advanced.Debugging.debugRendering); } public IDhApiConfigValue debugKeybindings() { return new DhApiConfigValue(Config.Client.Advanced.Debugging.enableDebugKeybindings); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFarFogConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFarFogConfig.java index 1bacc727c..0393ccbdb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFarFogConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFarFogConfig.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.api.external.methods.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogFalloff; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogFalloff; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiFarFogConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; @@ -50,8 +50,8 @@ public class DhApiFarFogConfig implements IDhApiFarFogConfig { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogMax); } @Override - public IDhApiConfigValue farFogFalloff() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogFalloff); } + public IDhApiConfigValue farFogFalloff() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogFalloff); } @Override public IDhApiConfigValue farFogDensity() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java index 9889723f2..61a2d3d92 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.api.external.methods.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogColorMode; -import com.seibel.distanthorizons.api.enums.rendering.EFogDistance; -import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogColorMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDistance; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiFarFogConfig; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiFogConfig; @@ -53,15 +53,15 @@ public class DhApiFogConfig implements IDhApiFogConfig //====================// @Override - public IDhApiConfigValue distance() + public IDhApiConfigValue distance() { return new DhApiConfigValue<>(Config.Client.Advanced.Graphics.Fog.distance); } @Override - public IDhApiConfigValue drawMode() + public IDhApiConfigValue drawMode() { return new DhApiConfigValue<>(Config.Client.Advanced.Graphics.Fog.drawMode); } @Override - public IDhApiConfigValue color() + public IDhApiConfigValue color() { return new DhApiConfigValue<>(Config.Client.Advanced.Graphics.Fog.colorMode); } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGpuBuffersConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGpuBuffersConfig.java index 7069e5df4..22f4852b0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGpuBuffersConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGpuBuffersConfig.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiGpuBuffersConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; public class DhApiGpuBuffersConfig implements IDhApiGpuBuffersConfig { @@ -33,7 +33,7 @@ public class DhApiGpuBuffersConfig implements IDhApiGpuBuffersConfig - public IDhApiConfigValue gpuUploadMethod() + public IDhApiConfigValue gpuUploadMethod() { return new DhApiConfigValue<>(Config.Client.Advanced.GpuBuffers.gpuUploadMethod); } public IDhApiConfigValue gpuUploadPerMegabyteInMilliseconds() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGraphicsConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGraphicsConfig.java index 63b810521..27c1d48eb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGraphicsConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGraphicsConfig.java @@ -20,14 +20,14 @@ package com.seibel.distanthorizons.core.api.external.methods.config.client; import com.seibel.distanthorizons.api.enums.config.*; -import com.seibel.distanthorizons.api.enums.rendering.ETransparency; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiTransparency; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiAmbientOcclusionConfig; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiFogConfig; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiGraphicsConfig; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiNoiseTextureConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; -import com.seibel.distanthorizons.api.enums.rendering.ERendererMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiRendererMode; import com.seibel.distanthorizons.core.config.Config; public class DhApiGraphicsConfig implements IDhApiGraphicsConfig @@ -61,8 +61,8 @@ public class DhApiGraphicsConfig implements IDhApiGraphicsConfig { return new DhApiConfigValue(Config.Client.quickEnableRendering); } @Override - public IDhApiConfigValue renderingMode() - { return new DhApiConfigValue(Config.Client.Advanced.Debugging.rendererMode); } + public IDhApiConfigValue renderingMode() + { return new DhApiConfigValue(Config.Client.Advanced.Debugging.rendererMode); } @@ -71,24 +71,24 @@ public class DhApiGraphicsConfig implements IDhApiGraphicsConfig //==================// @Override - public IDhApiConfigValue maxHorizontalResolution() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution); } + public IDhApiConfigValue maxHorizontalResolution() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution); } @Override - public IDhApiConfigValue verticalQuality() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.verticalQuality); } + public IDhApiConfigValue verticalQuality() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.verticalQuality); } @Override - public IDhApiConfigValue horizontalQuality() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.horizontalQuality); } + public IDhApiConfigValue horizontalQuality() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.horizontalQuality); } @Override - public IDhApiConfigValue transparency() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.transparency); } + public IDhApiConfigValue transparency() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.transparency); } @Override - public IDhApiConfigValue blocksToAvoid() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.blocksToIgnore); } + public IDhApiConfigValue blocksToAvoid() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Quality.blocksToIgnore); } @Override public IDhApiConfigValue tintWithAvoidedBlocks() @@ -105,11 +105,6 @@ public class DhApiGraphicsConfig implements IDhApiGraphicsConfig // advanced graphic settings // //===========================// - @Deprecated - @Override - public IDhApiConfigValue overdrawPrevention() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.AdvancedGraphics.overdrawPreventionPreset); } - @Override public IDhApiConfigValue overdrawPreventionRadius() { return new DhApiConfigValue(Config.Client.Advanced.Graphics.AdvancedGraphics.overdrawPrevention); } @@ -143,8 +138,8 @@ public class DhApiGraphicsConfig implements IDhApiGraphicsConfig { return new DhApiConfigValue(Config.Client.Advanced.Graphics.AdvancedGraphics.lodBias); } @Override - public IDhApiConfigValue lodShading() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.AdvancedGraphics.lodShading); } + public IDhApiConfigValue lodShading() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.AdvancedGraphics.lodShading); } @Override public IDhApiConfigValue disableFrustumCulling() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiHeightFogConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiHeightFogConfig.java index 866137bc0..ecd1090a2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiHeightFogConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiHeightFogConfig.java @@ -19,9 +19,9 @@ package com.seibel.distanthorizons.core.api.external.methods.config.client; -import com.seibel.distanthorizons.api.enums.rendering.EFogFalloff; -import com.seibel.distanthorizons.api.enums.rendering.EHeightFogMixMode; -import com.seibel.distanthorizons.api.enums.rendering.EHeightFogMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogFalloff; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiHeightFogMixMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiHeightFogMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiHeightFogConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; @@ -36,12 +36,12 @@ public class DhApiHeightFogConfig implements IDhApiHeightFogConfig @Override - public IDhApiConfigValue heightFogMixMode() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMixMode); } + public IDhApiConfigValue heightFogMixMode() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMixMode); } @Override - public IDhApiConfigValue heightFogMode() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMode); } + public IDhApiConfigValue heightFogMode() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMode); } @Override public IDhApiConfigValue heightFogBaseHeight() @@ -64,8 +64,8 @@ public class DhApiHeightFogConfig implements IDhApiHeightFogConfig { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMax); } @Override - public IDhApiConfigValue heightFogFalloff() - { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogFalloff); } + public IDhApiConfigValue heightFogFalloff() + { return new DhApiConfigValue(Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogFalloff); } @Override public IDhApiConfigValue heightFogDensity() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiplayerConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiplayerConfig.java index e09d70244..1c16671d0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiplayerConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiMultiplayerConfig.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiMultiplayerConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.api.enums.config.EServerFolderNameMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiServerFolderNameMode; public class DhApiMultiplayerConfig implements IDhApiMultiplayerConfig { @@ -33,8 +33,8 @@ public class DhApiMultiplayerConfig implements IDhApiMultiplayerConfig - public IDhApiConfigValue folderSavingMode() - { return new DhApiConfigValue(Config.Client.Advanced.Multiplayer.serverFolderNameMode); } + public IDhApiConfigValue folderSavingMode() + { return new DhApiConfigValue(Config.Client.Advanced.Multiplayer.serverFolderNameMode); } public IDhApiConfigValue multiverseSimilarityRequirement() { return new DhApiConfigValue(Config.Client.Advanced.Multiplayer.multiverseSimilarityRequiredPercent); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/common/DhApiWorldGenerationConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/common/DhApiWorldGenerationConfig.java index 121909395..547f395b1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/common/DhApiWorldGenerationConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/common/DhApiWorldGenerationConfig.java @@ -19,7 +19,6 @@ package com.seibel.distanthorizons.core.api.external.methods.config.common; -import com.seibel.distanthorizons.api.enums.config.ELightGenerationMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.both.IDhApiWorldGenerationConfig; import com.seibel.distanthorizons.api.objects.config.DhApiConfigValue; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index b4d0b3f04..48b1bf012 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -21,7 +21,6 @@ package com.seibel.distanthorizons.core.api.internal; import com.seibel.distanthorizons.api.DhApi; import com.seibel.distanthorizons.api.enums.rendering.EDhApiRenderPass; -import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiFramebuffer; import com.seibel.distanthorizons.api.methods.events.abstractEvents.*; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.level.IKeyedClientLevelManager; @@ -32,10 +31,9 @@ import com.seibel.distanthorizons.core.world.*; import com.seibel.distanthorizons.coreapi.DependencyInjection.ApiEventInjector; import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.coreapi.DependencyInjection.OverrideInjector; import com.seibel.distanthorizons.coreapi.ModInfo; -import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; -import com.seibel.distanthorizons.api.enums.rendering.ERendererMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiRendererMode; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.logging.ConfigBasedSpamLogger; @@ -54,7 +52,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.lwjgl.glfw.GLFW; -import org.lwjgl.opengl.GL32; import java.util.HashMap; import java.util.HashSet; @@ -547,7 +544,7 @@ public class ClientApi if (!renderingDeferredLayer) { - if (Config.Client.Advanced.Debugging.rendererMode.get() == ERendererMode.DEFAULT) + if (Config.Client.Advanced.Debugging.rendererMode.get() == EDhApiRendererMode.DEFAULT) { this.renderingCancelledForThisFrame = ApiEventInjector.INSTANCE.fireAllEvents(DhApiBeforeRenderEvent.class, renderEventParam); if (!this.renderingCancelledForThisFrame) @@ -560,7 +557,7 @@ public class ClientApi ApiEventInjector.INSTANCE.fireAllEvents(DhApiAfterRenderEvent.class, renderEventParam); } } - else if (Config.Client.Advanced.Debugging.rendererMode.get() == ERendererMode.DEBUG) + else if (Config.Client.Advanced.Debugging.rendererMode.get() == EDhApiRendererMode.DEBUG) { profiler.push("Render Debug"); ClientApi.testRenderer.render(); @@ -617,12 +614,12 @@ public class ClientApi if (glfwKey == GLFW.GLFW_KEY_F8) { - Config.Client.Advanced.Debugging.debugRendering.set(EDebugRendering.next(Config.Client.Advanced.Debugging.debugRendering.get())); + Config.Client.Advanced.Debugging.debugRendering.set(EDhApiDebugRendering.next(Config.Client.Advanced.Debugging.debugRendering.get())); MC.sendChatMessage("F8: Set debug mode to " + Config.Client.Advanced.Debugging.debugRendering.get()); } else if (glfwKey == GLFW.GLFW_KEY_F6) { - Config.Client.Advanced.Debugging.rendererMode.set(ERendererMode.next(Config.Client.Advanced.Debugging.rendererMode.get())); + Config.Client.Advanced.Debugging.rendererMode.set(EDhApiRendererMode.next(Config.Client.Advanced.Debugging.rendererMode.get())); MC.sendChatMessage("F6: Set rendering to " + Config.Client.Advanced.Debugging.rendererMode.get()); } else if (glfwKey == GLFW.GLFW_KEY_P) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 1594afe79..2cf5dd26f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -74,8 +74,8 @@ public class Config public static ConfigLinkedEntry quickLodChunkRenderDistance = new ConfigLinkedEntry(Advanced.Graphics.Quality.lodChunkRenderDistanceRadius); - public static ConfigEntry qualityPresetSetting = new ConfigEntry.Builder() - .set(EQualityPreset.MEDIUM) // the default value is set via the listener when accessed + public static ConfigEntry qualityPresetSetting = new ConfigEntry.Builder() + .set(EDhApiQualityPreset.MEDIUM) // the default value is set via the listener when accessed .comment("" + "Changing this setting will modify a number of different settings that will change the \n" + "visual fidelity of the rendered LODs.\n" @@ -86,8 +86,8 @@ public class Config .addListener(RenderQualityPresetConfigEventHandler.INSTANCE) .build(); - public static ConfigEntry threadPresetSetting = new ConfigEntry.Builder() - .set(EThreadPreset.LOW_IMPACT) // the default value is set via the listener when accessed + public static ConfigEntry threadPresetSetting = new ConfigEntry.Builder() + .set(EDhApiThreadPreset.LOW_IMPACT) // the default value is set via the listener when accessed .comment("" + "Changing this setting will modify a number of different settings that will change \n" + "the load that Distant Horizons is allowed to put on your CPU. \n" @@ -140,20 +140,20 @@ public class Config public static class Quality { - public static ConfigEntry maxHorizontalResolution = new ConfigEntry.Builder() - .set(EMaxHorizontalResolution.BLOCK) + public static ConfigEntry maxHorizontalResolution = new ConfigEntry.Builder() + .set(EDhApiMaxHorizontalResolution.BLOCK) .comment("" + "What is the maximum detail LODs should be drawn at? \n" + "Higher settings will increase memory and GPU usage. \n" + "\n" - + EMaxHorizontalResolution.CHUNK + ": render 1 LOD for each Chunk. \n" - + EMaxHorizontalResolution.HALF_CHUNK + ": render 4 LODs for each Chunk. \n" - + EMaxHorizontalResolution.FOUR_BLOCKS + ": render 16 LODs for each Chunk. \n" - + EMaxHorizontalResolution.TWO_BLOCKS + ": render 64 LODs for each Chunk. \n" - + EMaxHorizontalResolution.BLOCK + ": render 256 LODs for each Chunk (width of one block). \n" + + EDhApiMaxHorizontalResolution.CHUNK + ": render 1 LOD for each Chunk. \n" + + EDhApiMaxHorizontalResolution.HALF_CHUNK + ": render 4 LODs for each Chunk. \n" + + EDhApiMaxHorizontalResolution.FOUR_BLOCKS + ": render 16 LODs for each Chunk. \n" + + EDhApiMaxHorizontalResolution.TWO_BLOCKS + ": render 64 LODs for each Chunk. \n" + + EDhApiMaxHorizontalResolution.BLOCK + ": render 256 LODs for each Chunk (width of one block). \n" + "\n" - + "Lowest Quality: " + EMaxHorizontalResolution.CHUNK + "\n" - + "Highest Quality: " + EMaxHorizontalResolution.BLOCK) + + "Lowest Quality: " + EDhApiMaxHorizontalResolution.CHUNK + "\n" + + "Highest Quality: " + EDhApiMaxHorizontalResolution.BLOCK) .setPerformance(EConfigEntryPerformance.MEDIUM) .build(); @@ -163,21 +163,21 @@ public class Config .setPerformance(EConfigEntryPerformance.HIGH) .build(); - public static ConfigEntry verticalQuality = new ConfigEntry.Builder() - .set(EVerticalQuality.MEDIUM) + public static ConfigEntry verticalQuality = new ConfigEntry.Builder() + .set(EDhApiVerticalQuality.MEDIUM) .comment("" + "This indicates how well LODs will represent \n" + "overhangs, caves, floating islands, etc. \n" + "Higher options will make the world more accurate, but" + "will increase memory and GPU usage. \n" + "\n" - + "Lowest Quality: " + EVerticalQuality.HEIGHT_MAP + "\n" - + "Highest Quality: " + EVerticalQuality.EXTREME) + + "Lowest Quality: " + EDhApiVerticalQuality.HEIGHT_MAP + "\n" + + "Highest Quality: " + EDhApiVerticalQuality.EXTREME) .setPerformance(EConfigEntryPerformance.VERY_HIGH) .build(); - public static ConfigEntry horizontalQuality = new ConfigEntry.Builder() - .set(EHorizontalQuality.MEDIUM) + public static ConfigEntry horizontalQuality = new ConfigEntry.Builder() + .set(EDhApiHorizontalQuality.MEDIUM) .comment("" + "This indicates how quickly LODs decrease in quality the further away they are. \n" + "Higher settings will render higher quality fake chunks farther away, \n" @@ -185,25 +185,25 @@ public class Config .setPerformance(EConfigEntryPerformance.MEDIUM) .build(); - public static ConfigEntry transparency = new ConfigEntry.Builder() - .set(ETransparency.COMPLETE) + public static ConfigEntry transparency = new ConfigEntry.Builder() + .set(EDhApiTransparency.COMPLETE) .comment("" + "How should LOD transparency be handled. \n" + "\n" - + ETransparency.COMPLETE + ": LODs will render transparent. \n" - + ETransparency.FAKE + ": LODs will be opaque, but shaded to match the blocks underneath. \n" - + ETransparency.DISABLED + ": LODs will be opaque. \n" + + EDhApiTransparency.COMPLETE + ": LODs will render transparent. \n" + + EDhApiTransparency.FAKE + ": LODs will be opaque, but shaded to match the blocks underneath. \n" + + EDhApiTransparency.DISABLED + ": LODs will be opaque. \n" + "") .setPerformance(EConfigEntryPerformance.MEDIUM) .build(); - public static ConfigEntry blocksToIgnore = new ConfigEntry.Builder() - .set(EBlocksToAvoid.NON_COLLIDING) + public static ConfigEntry blocksToIgnore = new ConfigEntry.Builder() + .set(EDhApiBlocksToAvoid.NON_COLLIDING) .comment("" + "What blocks shouldn't be rendered as LODs? \n" + "\n" - + EBlocksToAvoid.NONE + ": Represent all blocks in the LODs \n" - + EBlocksToAvoid.NON_COLLIDING + ": Only represent solid blocks in the LODs (tall grass, torches, etc. won't count for a LOD's height) \n" + + EDhApiBlocksToAvoid.NONE + ": Represent all blocks in the LODs \n" + + EDhApiBlocksToAvoid.NON_COLLIDING + ": Only represent solid blocks in the LODs (tall grass, torches, etc. won't count for a LOD's height) \n" + "") .setPerformance(EConfigEntryPerformance.NONE) .build(); @@ -236,33 +236,33 @@ public class Config public static class Fog { - public static ConfigEntry drawMode = new ConfigEntry.Builder() - .set(EFogDrawMode.FOG_ENABLED) + public static ConfigEntry drawMode = new ConfigEntry.Builder() + .set(EDhApiFogDrawMode.FOG_ENABLED) .comment("" + "When should fog be drawn? \n" + "\n" - + EFogDrawMode.USE_OPTIFINE_SETTING + ": Use whatever Fog setting Optifine is using.\n" - + "If Optifine isn't installed this defaults to " + EFogDrawMode.FOG_ENABLED + ". \n" - + EFogDrawMode.FOG_ENABLED + ": Never draw fog on the LODs \n" - + EFogDrawMode.FOG_DISABLED + ": Always draw fast fog on the LODs \n" + + EDhApiFogDrawMode.USE_OPTIFINE_SETTING + ": Use whatever Fog setting Optifine is using.\n" + + "If Optifine isn't installed this defaults to " + EDhApiFogDrawMode.FOG_ENABLED + ". \n" + + EDhApiFogDrawMode.FOG_ENABLED + ": Never draw fog on the LODs \n" + + EDhApiFogDrawMode.FOG_DISABLED + ": Always draw fast fog on the LODs \n" + "\n" + "Disabling fog will improve GPU performance.") .setPerformance(EConfigEntryPerformance.VERY_LOW) .build(); - public static ConfigEntry distance = new ConfigEntry.Builder() - .set(EFogDistance.FAR) + public static ConfigEntry distance = new ConfigEntry.Builder() + .set(EDhApiFogDistance.FAR) .comment("At what distance should Fog be drawn on the LODs?") .setPerformance(EConfigEntryPerformance.NONE) .build(); - public static ConfigEntry colorMode = new ConfigEntry.Builder() - .set(EFogColorMode.USE_WORLD_FOG_COLOR) + public static ConfigEntry colorMode = new ConfigEntry.Builder() + .set(EDhApiFogColorMode.USE_WORLD_FOG_COLOR) .comment("" + "What color should fog use? \n" + "\n" - + EFogColorMode.USE_WORLD_FOG_COLOR + ": Use the world's fog color. \n" - + EFogColorMode.USE_SKY_COLOR + ": Use the sky's color.") + + EDhApiFogColorMode.USE_WORLD_FOG_COLOR + ": Use the world's fog color. \n" + + EDhApiFogColorMode.USE_SKY_COLOR + ": Use the sky's color.") .setPerformance(EConfigEntryPerformance.NONE) .build(); @@ -322,14 +322,14 @@ public class Config + "1.0: Fully opaque fog.") .build(); - public static ConfigEntry farFogFalloff = new ConfigEntry.Builder() - .set(EFogFalloff.EXPONENTIAL_SQUARED) + public static ConfigEntry farFogFalloff = new ConfigEntry.Builder() + .set(EDhApiFogFalloff.EXPONENTIAL_SQUARED) .comment("" + "How should the fog thickness should be calculated? \n" + "\n" - + EFogFalloff.LINEAR + ": Linear based on distance (will ignore 'density')\n" - + EFogFalloff.EXPONENTIAL + ": 1/(e^(distance*density)) \n" - + EFogFalloff.EXPONENTIAL_SQUARED + ": 1/(e^((distance*density)^2)") + + EDhApiFogFalloff.LINEAR + ": Linear based on distance (will ignore 'density')\n" + + EDhApiFogFalloff.EXPONENTIAL + ": 1/(e^(distance*density)) \n" + + EDhApiFogFalloff.EXPONENTIAL_SQUARED + ": 1/(e^((distance*density)^2)") .build(); public static ConfigEntry farFogDensity = new ConfigEntry.Builder() @@ -345,37 +345,37 @@ public class Config public static class HeightFog { - public static ConfigEntry heightFogMixMode = new ConfigEntry.Builder() - .set(EHeightFogMixMode.BASIC) + public static ConfigEntry heightFogMixMode = new ConfigEntry.Builder() + .set(EDhApiHeightFogMixMode.BASIC) .comment("" + "How should height effect the fog thickness? \n" + "Note: height fog is combined with the other fog settings. \n" + "\n" - + EHeightFogMixMode.BASIC + ": No special height fog effect. Fog is calculated based on camera distance \n" - + EHeightFogMixMode.IGNORE_HEIGHT + ": Ignore height completely. Fog is only calculated with horizontal distance \n" - + EHeightFogMixMode.ADDITION + ": heightFog + farFog \n" - + EHeightFogMixMode.MAX + ": max(heightFog, farFog) \n" - + EHeightFogMixMode.MULTIPLY + ": heightFog * farFog \n" - + EHeightFogMixMode.INVERSE_MULTIPLY + ": 1 - (1-heightFog) * (1-farFog) \n" - + EHeightFogMixMode.LIMITED_ADDITION + ": farFog + max(farFog, heightFog) \n" - + EHeightFogMixMode.MULTIPLY_ADDITION + ": farFog + farFog * heightFog \n" - + EHeightFogMixMode.INVERSE_MULTIPLY_ADDITION + ": farFog + 1 - (1-heightFog) * (1-farFog) \n" - + EHeightFogMixMode.AVERAGE + ": farFog*0.5 + heightFog*0.5 \n" + + EDhApiHeightFogMixMode.BASIC + ": No special height fog effect. Fog is calculated based on camera distance \n" + + EDhApiHeightFogMixMode.IGNORE_HEIGHT + ": Ignore height completely. Fog is only calculated with horizontal distance \n" + + EDhApiHeightFogMixMode.ADDITION + ": heightFog + farFog \n" + + EDhApiHeightFogMixMode.MAX + ": max(heightFog, farFog) \n" + + EDhApiHeightFogMixMode.MULTIPLY + ": heightFog * farFog \n" + + EDhApiHeightFogMixMode.INVERSE_MULTIPLY + ": 1 - (1-heightFog) * (1-farFog) \n" + + EDhApiHeightFogMixMode.LIMITED_ADDITION + ": farFog + max(farFog, heightFog) \n" + + EDhApiHeightFogMixMode.MULTIPLY_ADDITION + ": farFog + farFog * heightFog \n" + + EDhApiHeightFogMixMode.INVERSE_MULTIPLY_ADDITION + ": farFog + 1 - (1-heightFog) * (1-farFog) \n" + + EDhApiHeightFogMixMode.AVERAGE + ": farFog*0.5 + heightFog*0.5 \n" + "\n" - + "Note: height fog settings are ignored if '" + EHeightFogMixMode.BASIC + "' or '" + EHeightFogMixMode.IGNORE_HEIGHT + "' are selected.") + + "Note: height fog settings are ignored if '" + EDhApiHeightFogMixMode.BASIC + "' or '" + EDhApiHeightFogMixMode.IGNORE_HEIGHT + "' are selected.") .build(); - public static ConfigEntry heightFogMode = new ConfigEntry.Builder() - .set(EHeightFogMode.ABOVE_AND_BELOW_CAMERA) + public static ConfigEntry heightFogMode = new ConfigEntry.Builder() + .set(EDhApiHeightFogMode.ABOVE_AND_BELOW_CAMERA) .comment("" + "Where should the height fog start? \n" + "\n" - + EHeightFogMode.ABOVE_CAMERA + ": Height fog starts at the camera and goes towards the sky \n" - + EHeightFogMode.BELOW_CAMERA + ": Height fog starts at the camera and goes towards the void \n" - + EHeightFogMode.ABOVE_AND_BELOW_CAMERA + ": Height fog starts from the camera to goes towards both the sky and void \n" - + EHeightFogMode.ABOVE_SET_HEIGHT + ": Height fog starts from a set height and goes towards the sky \n" - + EHeightFogMode.BELOW_SET_HEIGHT + ": Height fog starts from a set height and goes towards the void \n" - + EHeightFogMode.ABOVE_AND_BELOW_SET_HEIGHT + ": Height fog starts from a set height and goes towards both the sky and void") + + EDhApiHeightFogMode.ABOVE_CAMERA + ": Height fog starts at the camera and goes towards the sky \n" + + EDhApiHeightFogMode.BELOW_CAMERA + ": Height fog starts at the camera and goes towards the void \n" + + EDhApiHeightFogMode.ABOVE_AND_BELOW_CAMERA + ": Height fog starts from the camera to goes towards both the sky and void \n" + + EDhApiHeightFogMode.ABOVE_SET_HEIGHT + ": Height fog starts from a set height and goes towards the sky \n" + + EDhApiHeightFogMode.BELOW_SET_HEIGHT + ": Height fog starts from a set height and goes towards the void \n" + + EDhApiHeightFogMode.ABOVE_AND_BELOW_SET_HEIGHT + ": Height fog starts from a set height and goes towards both the sky and void") .build(); public static ConfigEntry heightFogBaseHeight = new ConfigEntry.Builder() @@ -419,14 +419,14 @@ public class Config + "1.0: Fully opaque fog.") .build(); - public static ConfigEntry heightFogFalloff = new ConfigEntry.Builder() - .set(EFogFalloff.EXPONENTIAL_SQUARED) + public static ConfigEntry heightFogFalloff = new ConfigEntry.Builder() + .set(EDhApiFogFalloff.EXPONENTIAL_SQUARED) .comment("" + "How should the height fog thickness should be calculated? \n" + "\n" - + EFogFalloff.LINEAR + ": Linear based on height (will ignore 'density')\n" - + EFogFalloff.EXPONENTIAL + ": 1/(e^(height*density)) \n" - + EFogFalloff.EXPONENTIAL_SQUARED + ": 1/(e^((height*density)^2)") + + EDhApiFogFalloff.LINEAR + ": Linear based on height (will ignore 'density')\n" + + EDhApiFogFalloff.EXPONENTIAL + ": 1/(e^(height*density)) \n" + + EDhApiFogFalloff.EXPONENTIAL_SQUARED + ": 1/(e^((height*density)^2)") .build(); public static ConfigEntry heightFogDensity = new ConfigEntry.Builder() @@ -536,18 +536,6 @@ public class Config public static class AdvancedGraphics { - /** - * @deprecated Use overdrawPrevention instead, will be removed when DH updates to MC 1.21
- * After removal a float value will be used to control overdraw instead.
- */ - @Deprecated - public static ConfigEntry overdrawPreventionPreset = new ConfigEntry.Builder() - .set(EOverdrawPrevention.MEDIUM) - .comment("") - .setAppearance(EConfigEntryAppearance.ONLY_IN_API) - .setPerformance(EConfigEntryPerformance.NONE) - .build(); - public static ConfigEntry overdrawPrevention = new ConfigEntry.Builder() .setMinDefaultMax(0.0, 0.4, 1.0) .comment("" @@ -628,15 +616,15 @@ public class Config + "If set to 0 the mod wont overwrite vanilla's default (which so happens to also be 0)") .build(); - public static ConfigEntry lodShading = new ConfigEntry.Builder() - .set(ELodShading.AUTO) + public static ConfigEntry lodShading = new ConfigEntry.Builder() + .set(EDhApiLodShading.AUTO) .comment("" + "How should LODs be shaded? \n" + "\n" - + ELodShading.AUTO + ": Uses the same side shading as vanilla Minecraft blocks. \n" - + ELodShading.FORCED + ": Simulates Minecraft's block shading for LODs. \n" + + EDhApiLodShading.AUTO + ": Uses the same side shading as vanilla Minecraft blocks. \n" + + EDhApiLodShading.ENABLED + ": Simulates Minecraft's block shading for LODs. \n" + " Can be used to force LOD shading when using some shaders. \n" - + ELodShading.NONE + ": All LOD sides will be rendered with the same brightness. \n" + + EDhApiLodShading.DISABLED + ": All LOD sides will be rendered with the same brightness. \n" + "") .setPerformance(EConfigEntryPerformance.NONE) .build(); @@ -730,36 +718,6 @@ public class Config + "") .build(); - // deprecated and not implemented, can be made public if we ever re-implement it - @Deprecated - private static ConfigEntry generationPriority = new ConfigEntry.Builder() - .set(EGenerationPriority.NEAR_FIRST) - .comment("" - + "In what priority should fake chunks be generated outside the vanilla render distance? \n" - + "\n" - + EGenerationPriority.FAR_FIRST + " \n" - + "Fake chunks are generated from lowest to highest detail \n" - + " with a priority for far away regions. \n" - + "This fills in the world fastest, but you will have large low detail \n" - + " blocks for a while while the generation happens. \n" - + "\n" - + EGenerationPriority.NEAR_FIRST + " \n" - + "Fake chunks are generated around the player \n" - + " in a spiral, similar to vanilla minecraft. \n" - + "Best used when on a server since we can't generate \n" - + " fake chunks. \n" - + "\n" - + EGenerationPriority.BALANCED + " \n" - + "A mix between " + EGenerationPriority.NEAR_FIRST + "and" + EGenerationPriority.FAR_FIRST + ". \n" - + "First prioritise completing nearby highest detail chunks, \n" - + " then focus on filling in the low detail areas away from the player. \n" - + "\n" - + EGenerationPriority.AUTO + " \n" - + "Uses " + EGenerationPriority.BALANCED + " when on a single player world \n" - + " and " + EGenerationPriority.NEAR_FIRST + " when connected to a server.") - .setPerformance(EConfigEntryPerformance.NONE) - .build(); - } public static class LodBuilding @@ -846,16 +804,16 @@ public class Config public static class Multiplayer { - public static ConfigEntry serverFolderNameMode = new ConfigEntry.Builder() - .set(EServerFolderNameMode.NAME_ONLY) + public static ConfigEntry serverFolderNameMode = new ConfigEntry.Builder() + .set(EDhApiServerFolderNameMode.NAME_ONLY) .comment("" + "How should multiplayer save folders should be named? \n" + "\n" - + EServerFolderNameMode.NAME_ONLY + ": Example: \"Minecraft Server\" \n" - + EServerFolderNameMode.IP_ONLY + ": Example: \"192.168.1.40\" \n" - + EServerFolderNameMode.NAME_IP + ": Example: \"Minecraft Server IP 192.168.1.40\" \n" - + EServerFolderNameMode.NAME_IP_PORT + ": Example: \"Minecraft Server IP 192.168.1.40:25565\"" - + EServerFolderNameMode.NAME_IP_PORT_MC_VERSION + ": Example: \"Minecraft Server IP 192.168.1.40:25565 GameVersion 1.16.5\"") + + EDhApiServerFolderNameMode.NAME_ONLY + ": Example: \"Minecraft Server\" \n" + + EDhApiServerFolderNameMode.IP_ONLY + ": Example: \"192.168.1.40\" \n" + + EDhApiServerFolderNameMode.NAME_IP + ": Example: \"Minecraft Server IP 192.168.1.40\" \n" + + EDhApiServerFolderNameMode.NAME_IP_PORT + ": Example: \"Minecraft Server IP 192.168.1.40:25565\"" + + EDhApiServerFolderNameMode.NAME_IP_PORT_MC_VERSION + ": Example: \"Minecraft Server IP 192.168.1.40:25565 GameVersion 1.16.5\"") .build(); public static ConfigEntry multiverseSimilarityRequiredPercent = new ConfigEntry.Builder() @@ -1017,27 +975,27 @@ public class Config public static class GpuBuffers { - public static ConfigEntry gpuUploadMethod = new ConfigEntry.Builder() - .set(EGpuUploadMethod.AUTO) + public static ConfigEntry gpuUploadMethod = new ConfigEntry.Builder() + .set(EDhApiGpuUploadMethod.AUTO) .comment("" + "What method should be used to upload geometry to the GPU? \n" + "\n" - + EGpuUploadMethod.AUTO + ": Picks the best option based on the GPU you have. \n" + + EDhApiGpuUploadMethod.AUTO + ": Picks the best option based on the GPU you have. \n" + "\n" - + EGpuUploadMethod.BUFFER_STORAGE + ": Default if OpenGL 4.5 is supported. \n" + + EDhApiGpuUploadMethod.BUFFER_STORAGE + ": Default if OpenGL 4.5 is supported. \n" + " Fast rendering, no stuttering. \n" + "\n" - + EGpuUploadMethod.SUB_DATA + ": Backup option for NVIDIA. \n" + + EDhApiGpuUploadMethod.SUB_DATA + ": Backup option for NVIDIA. \n" + " Fast rendering but may stutter when uploading. \n" + "\n" - + EGpuUploadMethod.BUFFER_MAPPING + ": Slow rendering but won't stutter when uploading. \n" + + EDhApiGpuUploadMethod.BUFFER_MAPPING + ": Slow rendering but won't stutter when uploading. \n" + " Generally the best option for integrated GPUs. \n" + " Default option for AMD/Intel if OpenGL 4.5 isn't supported. \n" + " May end up storing buffers in System memory. \n" + " Fast rendering if in GPU memory, slow if in system memory, \n" + " but won't stutter when uploading. \n" + "\n" - + EGpuUploadMethod.DATA + ": Fast rendering but will stutter when uploading. \n" + + EDhApiGpuUploadMethod.DATA + ": Fast rendering but will stutter when uploading. \n" + " Backup option for AMD/Intel. \n" + " Fast rendering but may stutter when uploading. \n" + "\n" @@ -1070,15 +1028,6 @@ public class Config + "") .build(); - // deprecated and not implemented, can be made public if we ever re-implement it - @Deprecated - private static ConfigEntry rebuildTimes = new ConfigEntry.Builder() - .set(EBufferRebuildTimes.NORMAL) - .comment("" - + "How frequently should vertex buffers (geometry) be rebuilt and sent to the GPU? \n" - + "Higher settings may cause stuttering, but will prevent holes in the world") - .build(); - } public static class AutoUpdater @@ -1097,9 +1046,9 @@ public class Config + "Should Distant Horizons silently, automatically download and install new versions?") .build(); - public static ConfigEntry updateBranch = new ConfigEntry.Builder() + public static ConfigEntry updateBranch = new ConfigEntry.Builder() .set( - ModInfo.IS_DEV_BUILD? EUpdateBranch.NIGHTLY: EUpdateBranch.STABLE // If it's already a nightly build, then download the nightly build ofc + ModInfo.IS_DEV_BUILD? EDhApiUpdateBranch.NIGHTLY: EDhApiUpdateBranch.STABLE // If it's already a nightly build, then download the nightly build ofc ) .comment("" + " If DH should use the nightly (provided by Gitlab), or stable (provided by Modrinth) build") @@ -1110,64 +1059,64 @@ public class Config { // TODO add change all option // TODO default to error chat and info file - public static ConfigEntry logWorldGenEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logWorldGenEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about the world generation process. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logWorldGenPerformance = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_FILE) + public static ConfigEntry logWorldGenPerformance = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_FILE) .comment("" + "If enabled, the mod will log performance about the world generation process. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logWorldGenLoadEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_FILE) + public static ConfigEntry logWorldGenLoadEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_FILE) .comment("" + "If enabled, the mod will log information about the world generation process. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logLodBuilderEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logLodBuilderEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about the LOD generation process. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logRendererBufferEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logRendererBufferEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about the renderer buffer process. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logRendererGLEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logRendererGLEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about the renderer OpenGL process. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logFileReadWriteEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logFileReadWriteEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about file read/write operations. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logFileSubDimEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logFileSubDimEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about file sub-dimension operations. \n" + "This can be useful for debugging.") .build(); - public static ConfigEntry logNetworkEvent = new ConfigEntry.Builder() - .set(ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) + public static ConfigEntry logNetworkEvent = new ConfigEntry.Builder() + .set(EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE) .comment("" + "If enabled, the mod will log information about network operations. \n" + "This can be useful for debugging.") @@ -1177,25 +1126,25 @@ public class Config public static class Debugging { - public static ConfigEntry rendererMode = new ConfigEntry.Builder() - .set(ERendererMode.DEFAULT) + public static ConfigEntry rendererMode = new ConfigEntry.Builder() + .set(EDhApiRendererMode.DEFAULT) .comment("" + "What renderer is active? \n" + "\n" - + ERendererMode.DEFAULT + ": Default lod renderer \n" - + ERendererMode.DEBUG + ": Debug testing renderer \n" - + ERendererMode.DISABLED + ": Disable rendering") + + EDhApiRendererMode.DEFAULT + ": Default lod renderer \n" + + EDhApiRendererMode.DEBUG + ": Debug testing renderer \n" + + EDhApiRendererMode.DISABLED + ": Disable rendering") .build(); - public static ConfigEntry debugRendering = new ConfigEntry.Builder() - .set(EDebugRendering.OFF) + public static ConfigEntry debugRendering = new ConfigEntry.Builder() + .set(EDhApiDebugRendering.OFF) .comment("" + "Should specialized colors/rendering modes be used? \n" + "\n" - + EDebugRendering.OFF + ": LODs will be drawn with their normal colors. \n" - + EDebugRendering.SHOW_DETAIL + ": LODs' color will be based on their detail level. \n" - + EDebugRendering.SHOW_BLOCK_MATERIAL + ": LODs' color will be based on their material. \n" - + EDebugRendering.SHOW_OVERLAPPING_QUADS + ": LODs will be drawn with total white, but overlapping quads will be drawn with red. \n" + + EDhApiDebugRendering.OFF + ": LODs will be drawn with their normal colors. \n" + + EDhApiDebugRendering.SHOW_DETAIL + ": LODs' color will be based on their detail level. \n" + + EDhApiDebugRendering.SHOW_BLOCK_MATERIAL + ": LODs' color will be based on their material. \n" + + EDhApiDebugRendering.SHOW_OVERLAPPING_QUADS + ": LODs will be drawn with total white, but overlapping quads will be drawn with red. \n" + "") .build(); @@ -1319,15 +1268,15 @@ public class Config + "") .build(); - public static ConfigEntry glErrorHandlingMode = new ConfigEntry.Builder() - .set(ModInfo.IS_DEV_BUILD ? EGLErrorHandlingMode.LOG : EGLErrorHandlingMode.IGNORE) + public static ConfigEntry glErrorHandlingMode = new ConfigEntry.Builder() + .set(ModInfo.IS_DEV_BUILD ? EDhApiGLErrorHandlingMode.LOG : EDhApiGLErrorHandlingMode.IGNORE) .comment("" + "Defines how OpenGL errors are handled. \n" + "May incorrectly catch OpenGL errors thrown by other mods. \n" + "\n" - + EGLErrorHandlingMode.IGNORE + ": Do nothing. \n" - + EGLErrorHandlingMode.LOG + ": write an error to the log. \n" - + EGLErrorHandlingMode.LOG_THROW + ": write to the log and throw an exception. \n" + + EDhApiGLErrorHandlingMode.IGNORE + ": Do nothing. \n" + + EDhApiGLErrorHandlingMode.LOG + ": write an error to the log. \n" + + EDhApiGLErrorHandlingMode.LOG_THROW + ": write to the log and throw an exception. \n" + " Warning: this should only be enabled when debugging the LOD renderer \n" + " as it may break Minecraft's renderer when an exception is thrown. \n" + "") @@ -1360,14 +1309,14 @@ public class Config "") .build(); - public static ConfigEntry glProfileMode = new ConfigEntry.Builder() - .set(EGlProfileMode.CORE) + public static ConfigEntry glProfileMode = new ConfigEntry.Builder() + .set(EDhApiGlProfileMode.CORE) .comment("" + "Can be changed if you experience crashing when loading into a world.\n" + "\n" + "Defines the OpenGL context type Distant Horizon's will create. \n" + - "Generally this should be left as ["+EGlProfileMode.CORE+"] unless there is an issue with your GPU driver. \n" + - "Possible values: ["+ StringUtil.join("],[", EGlProfileMode.values())+"] \n" + + "Generally this should be left as ["+ EDhApiGlProfileMode.CORE+"] unless there is an issue with your GPU driver. \n" + + "Possible values: ["+ StringUtil.join("],[", EDhApiGlProfileMode.values())+"] \n" + "") .build(); public static ConfigEntry enableGlForwardCompatibilityMode = new ConfigEntry.Builder() @@ -1515,23 +1464,12 @@ public class Config { complicatedListenerSetupComplete = true; - try - { - // listener can only be added after the config has finished initializing - Client.Advanced.Graphics.AdvancedGraphics.overdrawPreventionPreset.addListener(OverdrawPreventionPresetConfigEventHandler.INSTANCE); - } - catch (Exception e) - { - LOGGER.error("Unexpected exception when running config delayed listener setup. Error: [" + e.getMessage() + "].", e); - } - try { // TODO automatically get all instances of AbstractPresetConfigEventHandler and fire "setUiOnlyConfigValues" ThreadPresetConfigEventHandler.INSTANCE.setUiOnlyConfigValues(); RenderQualityPresetConfigEventHandler.INSTANCE.setUiOnlyConfigValues(); QuickRenderToggleConfigEventHandler.INSTANCE.setUiOnlyConfigValues(); - OverdrawPreventionPresetConfigEventHandler.INSTANCE.setUiOnlyConfigValues(); RenderCacheConfigEventHandler.getInstance(); } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/DebugColumnConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/DebugColumnConfigEventHandler.java index 9e014f45f..93f55820d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/DebugColumnConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/DebugColumnConfigEventHandler.java @@ -20,24 +20,8 @@ package com.seibel.distanthorizons.core.config.eventHandlers; import com.seibel.distanthorizons.api.DhApi; -import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; -import com.seibel.distanthorizons.api.enums.config.EMaxHorizontalResolution; -import com.seibel.distanthorizons.api.enums.config.EVerticalQuality; -import com.seibel.distanthorizons.api.enums.config.quickOptions.EQualityPreset; -import com.seibel.distanthorizons.api.enums.rendering.ETransparency; import com.seibel.distanthorizons.api.interfaces.render.IDhApiRenderProxy; -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.config.ConfigEntryWithPresetOptions; -import com.seibel.distanthorizons.core.config.eventHandlers.presets.AbstractPresetConfigEventHandler; -import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.config.listeners.IConfigListener; -import com.seibel.distanthorizons.coreapi.interfaces.config.IConfigEntry; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; public class DebugColumnConfigEventHandler implements IConfigListener { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/QuickRenderToggleConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/QuickRenderToggleConfigEventHandler.java index 1991603e3..8b6f3cb6d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/QuickRenderToggleConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/QuickRenderToggleConfigEventHandler.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.config.eventHandlers; -import com.seibel.distanthorizons.api.enums.rendering.ERendererMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiRendererMode; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.config.Config; @@ -28,15 +28,15 @@ public class QuickRenderToggleConfigEventHandler public static QuickRenderToggleConfigEventHandler INSTANCE = new QuickRenderToggleConfigEventHandler(); private final ConfigChangeListener quickRenderChangeListener; - private final ConfigChangeListener rendererModeChangeListener; + private final ConfigChangeListener rendererModeChangeListener; /** private since we only ever need one handler at a time */ private QuickRenderToggleConfigEventHandler() { - this.quickRenderChangeListener = new ConfigChangeListener<>(Config.Client.quickEnableRendering, (val) -> { Config.Client.Advanced.Debugging.rendererMode.set(Config.Client.quickEnableRendering.get() ? ERendererMode.DEFAULT : ERendererMode.DISABLED); }); - this.rendererModeChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Debugging.rendererMode, (val) -> { Config.Client.quickEnableRendering.set(Config.Client.Advanced.Debugging.rendererMode.get() != ERendererMode.DISABLED); }); + this.quickRenderChangeListener = new ConfigChangeListener<>(Config.Client.quickEnableRendering, (val) -> { Config.Client.Advanced.Debugging.rendererMode.set(Config.Client.quickEnableRendering.get() ? EDhApiRendererMode.DEFAULT : EDhApiRendererMode.DISABLED); }); + this.rendererModeChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Debugging.rendererMode, (val) -> { Config.Client.quickEnableRendering.set(Config.Client.Advanced.Debugging.rendererMode.get() != EDhApiRendererMode.DISABLED); }); } /** @@ -45,7 +45,7 @@ public class QuickRenderToggleConfigEventHandler */ public void setUiOnlyConfigValues() { - Config.Client.quickEnableRendering.set(Config.Client.Advanced.Debugging.rendererMode.get() != ERendererMode.DISABLED); + Config.Client.quickEnableRendering.set(Config.Client.Advanced.Debugging.rendererMode.get() != EDhApiRendererMode.DISABLED); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java index 1a169aa97..6e3901462 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java @@ -20,13 +20,12 @@ package com.seibel.distanthorizons.core.config.eventHandlers; import com.seibel.distanthorizons.api.DhApi; -import com.seibel.distanthorizons.api.enums.config.EBlocksToAvoid; -import com.seibel.distanthorizons.api.enums.config.ELodShading; -import com.seibel.distanthorizons.api.enums.config.EMaxHorizontalResolution; -import com.seibel.distanthorizons.api.enums.config.EVerticalQuality; -import com.seibel.distanthorizons.api.enums.rendering.ETransparency; +import com.seibel.distanthorizons.api.enums.config.EDhApiBlocksToAvoid; +import com.seibel.distanthorizons.api.enums.config.EDhApiLodShading; +import com.seibel.distanthorizons.api.enums.config.EDhApiMaxHorizontalResolution; +import com.seibel.distanthorizons.api.enums.config.EDhApiVerticalQuality; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiTransparency; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; -import com.seibel.distanthorizons.core.config.listeners.IConfigListener; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.util.TimerUtil; @@ -45,15 +44,15 @@ public class RenderCacheConfigEventHandler // previous values used to check if a watched setting was actually modified - private final ConfigChangeListener horizontalResolutionChangeListener; - private final ConfigChangeListener verticalQualityChangeListener; - private final ConfigChangeListener transparencyChangeListener; - private final ConfigChangeListener blocksToIgnoreChangeListener; + private final ConfigChangeListener horizontalResolutionChangeListener; + private final ConfigChangeListener verticalQualityChangeListener; + private final ConfigChangeListener transparencyChangeListener; + private final ConfigChangeListener blocksToIgnoreChangeListener; private final ConfigChangeListener tintWithAvoidedBlocksChangeListener; private final ConfigChangeListener brightnessMultiplierChangeListener; private final ConfigChangeListener saturationMultiplierChangeListener; - private final ConfigChangeListener lodShadingChangeListener; + private final ConfigChangeListener lodShadingChangeListener; /** how long to wait in milliseconds before applying the config changes */ private static final long TIMEOUT_IN_MS = 4_000L; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/OverdrawPreventionPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/OverdrawPreventionPresetConfigEventHandler.java deleted file mode 100644 index 216238d53..000000000 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/OverdrawPreventionPresetConfigEventHandler.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.core.config.eventHandlers.presets; - -import com.seibel.distanthorizons.api.enums.config.EMaxHorizontalResolution; -import com.seibel.distanthorizons.api.enums.config.EOverdrawPrevention; -import com.seibel.distanthorizons.api.enums.config.quickOptions.EQualityPreset; -import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.core.config.ConfigEntryWithPresetOptions; -import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; -import com.seibel.distanthorizons.coreapi.interfaces.config.IConfigEntry; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -public class OverdrawPreventionPresetConfigEventHandler extends AbstractPresetConfigEventHandler -{ - public static final OverdrawPreventionPresetConfigEventHandler INSTANCE = new OverdrawPreventionPresetConfigEventHandler(); - - private static final Logger LOGGER = LogManager.getLogger(); - - - private final ConfigEntryWithPresetOptions overdrawPrevention = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.AdvancedGraphics.overdrawPrevention, - new HashMap() - {{ - this.put(EOverdrawPrevention.HEAVY, 0.6); - this.put(EOverdrawPrevention.MEDIUM, 0.4); - this.put(EOverdrawPrevention.LIGHT, 0.25); - this.put(EOverdrawPrevention.NONE, 0.0); - }}); - - - - //==============// - // constructors // - //==============// - - /** private since we only ever need one handler at a time */ - private OverdrawPreventionPresetConfigEventHandler() - { - // add each config used by this preset - this.configList.add(this.overdrawPrevention); - - - for (ConfigEntryWithPresetOptions config : this.configList) - { - // ignore try-using, the listener should only ever be added once and should never be removed - new ConfigChangeListener<>(config.configEntry, (val) -> { this.onConfigValueChanged(); }); - } - } - - - - //==============// - // enum getters // - //==============// - - @Override - protected IConfigEntry getPresetConfigEntry() { return Config.Client.Advanced.Graphics.AdvancedGraphics.overdrawPreventionPreset; } - - @Override - protected List getPresetEnumList() { return Arrays.asList(EOverdrawPrevention.values()); } - @Override - protected EOverdrawPrevention getCustomPresetEnum() { return EOverdrawPrevention.CUSTOM; } - -} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/RenderQualityPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/RenderQualityPresetConfigEventHandler.java index 3ac0f6f78..85f1e1b40 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/RenderQualityPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/RenderQualityPresetConfigEventHandler.java @@ -19,11 +19,11 @@ package com.seibel.distanthorizons.core.config.eventHandlers.presets; -import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; -import com.seibel.distanthorizons.api.enums.config.EMaxHorizontalResolution; -import com.seibel.distanthorizons.api.enums.config.EVerticalQuality; -import com.seibel.distanthorizons.api.enums.config.quickOptions.EQualityPreset; -import com.seibel.distanthorizons.api.enums.rendering.ETransparency; +import com.seibel.distanthorizons.api.enums.config.EDhApiHorizontalQuality; +import com.seibel.distanthorizons.api.enums.config.EDhApiMaxHorizontalResolution; +import com.seibel.distanthorizons.api.enums.config.EDhApiVerticalQuality; +import com.seibel.distanthorizons.api.enums.config.quickOptions.EDhApiQualityPreset; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiTransparency; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.ConfigEntryWithPresetOptions; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; @@ -33,57 +33,57 @@ import org.apache.logging.log4j.Logger; import java.util.*; -public class RenderQualityPresetConfigEventHandler extends AbstractPresetConfigEventHandler +public class RenderQualityPresetConfigEventHandler extends AbstractPresetConfigEventHandler { public static final RenderQualityPresetConfigEventHandler INSTANCE = new RenderQualityPresetConfigEventHandler(); private static final Logger LOGGER = LogManager.getLogger(); - private final ConfigEntryWithPresetOptions drawResolution = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution, - new HashMap() + private final ConfigEntryWithPresetOptions drawResolution = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.maxHorizontalResolution, + new HashMap() {{ - this.put(EQualityPreset.MINIMUM, EMaxHorizontalResolution.TWO_BLOCKS); - this.put(EQualityPreset.LOW, EMaxHorizontalResolution.BLOCK); - this.put(EQualityPreset.MEDIUM, EMaxHorizontalResolution.BLOCK); - this.put(EQualityPreset.HIGH, EMaxHorizontalResolution.BLOCK); - this.put(EQualityPreset.EXTREME, EMaxHorizontalResolution.BLOCK); + this.put(EDhApiQualityPreset.MINIMUM, EDhApiMaxHorizontalResolution.TWO_BLOCKS); + this.put(EDhApiQualityPreset.LOW, EDhApiMaxHorizontalResolution.BLOCK); + this.put(EDhApiQualityPreset.MEDIUM, EDhApiMaxHorizontalResolution.BLOCK); + this.put(EDhApiQualityPreset.HIGH, EDhApiMaxHorizontalResolution.BLOCK); + this.put(EDhApiQualityPreset.EXTREME, EDhApiMaxHorizontalResolution.BLOCK); }}); - private final ConfigEntryWithPresetOptions verticalQuality = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.verticalQuality, - new HashMap() + private final ConfigEntryWithPresetOptions verticalQuality = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.verticalQuality, + new HashMap() {{ - this.put(EQualityPreset.MINIMUM, EVerticalQuality.HEIGHT_MAP); - this.put(EQualityPreset.LOW, EVerticalQuality.LOW); - this.put(EQualityPreset.MEDIUM, EVerticalQuality.MEDIUM); - this.put(EQualityPreset.HIGH, EVerticalQuality.HIGH); - this.put(EQualityPreset.EXTREME, EVerticalQuality.EXTREME); + this.put(EDhApiQualityPreset.MINIMUM, EDhApiVerticalQuality.HEIGHT_MAP); + this.put(EDhApiQualityPreset.LOW, EDhApiVerticalQuality.LOW); + this.put(EDhApiQualityPreset.MEDIUM, EDhApiVerticalQuality.MEDIUM); + this.put(EDhApiQualityPreset.HIGH, EDhApiVerticalQuality.HIGH); + this.put(EDhApiQualityPreset.EXTREME, EDhApiVerticalQuality.EXTREME); }}); - private final ConfigEntryWithPresetOptions horizontalQuality = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, - new HashMap() + private final ConfigEntryWithPresetOptions horizontalQuality = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.horizontalQuality, + new HashMap() {{ - this.put(EQualityPreset.MINIMUM, EHorizontalQuality.LOWEST); - this.put(EQualityPreset.LOW, EHorizontalQuality.LOW); - this.put(EQualityPreset.MEDIUM, EHorizontalQuality.MEDIUM); - this.put(EQualityPreset.HIGH, EHorizontalQuality.HIGH); - this.put(EQualityPreset.EXTREME, EHorizontalQuality.EXTREME); + this.put(EDhApiQualityPreset.MINIMUM, EDhApiHorizontalQuality.LOWEST); + this.put(EDhApiQualityPreset.LOW, EDhApiHorizontalQuality.LOW); + this.put(EDhApiQualityPreset.MEDIUM, EDhApiHorizontalQuality.MEDIUM); + this.put(EDhApiQualityPreset.HIGH, EDhApiHorizontalQuality.HIGH); + this.put(EDhApiQualityPreset.EXTREME, EDhApiHorizontalQuality.EXTREME); }}); - private final ConfigEntryWithPresetOptions transparency = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.transparency, - new HashMap() + private final ConfigEntryWithPresetOptions transparency = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Quality.transparency, + new HashMap() {{ - this.put(EQualityPreset.MINIMUM, ETransparency.DISABLED); - this.put(EQualityPreset.LOW, ETransparency.DISABLED); // should be fake if/when fake is fixed - this.put(EQualityPreset.MEDIUM, ETransparency.COMPLETE); - this.put(EQualityPreset.HIGH, ETransparency.COMPLETE); - this.put(EQualityPreset.EXTREME, ETransparency.COMPLETE); + this.put(EDhApiQualityPreset.MINIMUM, EDhApiTransparency.DISABLED); + this.put(EDhApiQualityPreset.LOW, EDhApiTransparency.DISABLED); // should be fake if/when fake is fixed + this.put(EDhApiQualityPreset.MEDIUM, EDhApiTransparency.COMPLETE); + this.put(EDhApiQualityPreset.HIGH, EDhApiTransparency.COMPLETE); + this.put(EDhApiQualityPreset.EXTREME, EDhApiTransparency.COMPLETE); }}); - private final ConfigEntryWithPresetOptions ssaoEnabled = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Ssao.enabled, - new HashMap() + private final ConfigEntryWithPresetOptions ssaoEnabled = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.Graphics.Ssao.enabled, + new HashMap() {{ - this.put(EQualityPreset.MINIMUM, false); - this.put(EQualityPreset.LOW, false); - this.put(EQualityPreset.MEDIUM, true); - this.put(EQualityPreset.HIGH, true); - this.put(EQualityPreset.EXTREME, true); + this.put(EDhApiQualityPreset.MINIMUM, false); + this.put(EDhApiQualityPreset.LOW, false); + this.put(EDhApiQualityPreset.MEDIUM, true); + this.put(EDhApiQualityPreset.HIGH, true); + this.put(EDhApiQualityPreset.EXTREME, true); }}); @@ -103,7 +103,7 @@ public class RenderQualityPresetConfigEventHandler extends AbstractPresetConfigE this.configList.add(this.ssaoEnabled); - for (ConfigEntryWithPresetOptions config : this.configList) + for (ConfigEntryWithPresetOptions config : this.configList) { // ignore try-using, the listener should only ever be added once and should never be removed new ConfigChangeListener<>(config.configEntry, (val) -> { this.onConfigValueChanged(); }); @@ -117,11 +117,11 @@ public class RenderQualityPresetConfigEventHandler extends AbstractPresetConfigE //==============// @Override - protected IConfigEntry getPresetConfigEntry() { return Config.Client.qualityPresetSetting; } + protected IConfigEntry getPresetConfigEntry() { return Config.Client.qualityPresetSetting; } @Override - protected List getPresetEnumList() { return Arrays.asList(EQualityPreset.values()); } + protected List getPresetEnumList() { return Arrays.asList(EDhApiQualityPreset.values()); } @Override - protected EQualityPreset getCustomPresetEnum() { return EQualityPreset.CUSTOM; } + protected EDhApiQualityPreset getCustomPresetEnum() { return EDhApiQualityPreset.CUSTOM; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index fee7bb252..0a25d0695 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.config.eventHandlers.presets; -import com.seibel.distanthorizons.api.enums.config.quickOptions.EThreadPreset; +import com.seibel.distanthorizons.api.enums.config.quickOptions.EDhApiThreadPreset; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.ConfigEntryWithPresetOptions; @@ -32,7 +32,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; -public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHandler +public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHandler { public static final ThreadPresetConfigEventHandler INSTANCE = new ThreadPresetConfigEventHandler(); @@ -42,90 +42,90 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan public static int getWorldGenDefaultThreadCount() { return getThreadCountByPercent(0.15); } - private final ConfigEntryWithPresetOptions worldGenThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, - new HashMap() + private final ConfigEntryWithPresetOptions worldGenThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 1); - this.put(EThreadPreset.LOW_IMPACT, getWorldGenDefaultThreadCount()); - this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.25)); - this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.5)); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1); + this.put(EDhApiThreadPreset.LOW_IMPACT, getWorldGenDefaultThreadCount()); + this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.25)); + this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.5)); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); public static double getWorldGenDefaultRunTimeRatio() { return LOW_THREAD_COUNT_CPU ? 0.5 : 0.75; } - private final ConfigEntryWithPresetOptions worldGenRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, - new HashMap() + private final ConfigEntryWithPresetOptions worldGenRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, LOW_THREAD_COUNT_CPU ? 0.1 : 0.25); - this.put(EThreadPreset.LOW_IMPACT, getWorldGenDefaultRunTimeRatio()); - this.put(EThreadPreset.BALANCED, LOW_THREAD_COUNT_CPU ? 0.5 : 0.75); - this.put(EThreadPreset.AGGRESSIVE, LOW_THREAD_COUNT_CPU ? 0.75 : 1.0); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, LOW_THREAD_COUNT_CPU ? 0.1 : 0.25); + this.put(EDhApiThreadPreset.LOW_IMPACT, getWorldGenDefaultRunTimeRatio()); + this.put(EDhApiThreadPreset.BALANCED, LOW_THREAD_COUNT_CPU ? 0.5 : 0.75); + this.put(EDhApiThreadPreset.AGGRESSIVE, LOW_THREAD_COUNT_CPU ? 0.75 : 1.0); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); }}); public static int getFileHandlerDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions fileHandlerThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, - new HashMap() + private final ConfigEntryWithPresetOptions fileHandlerThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfFileHandlerThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 1); - this.put(EThreadPreset.LOW_IMPACT, getFileHandlerDefaultThreadCount()); - this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.2)); - this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1); + this.put(EDhApiThreadPreset.LOW_IMPACT, getFileHandlerDefaultThreadCount()); + this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.2)); + this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); public static double getFileHandlerDefaultRunTimeRatio() { return 0.5; } - private final ConfigEntryWithPresetOptions fileHandlerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, - new HashMap() + private final ConfigEntryWithPresetOptions fileHandlerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); - this.put(EThreadPreset.LOW_IMPACT, getFileHandlerDefaultRunTimeRatio()); - this.put(EThreadPreset.BALANCED, 0.75); - this.put(EThreadPreset.AGGRESSIVE, 1.0); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EDhApiThreadPreset.LOW_IMPACT, getFileHandlerDefaultRunTimeRatio()); + this.put(EDhApiThreadPreset.BALANCED, 0.75); + this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); }}); public static int getUpdatePropagatorDefaultThreadCount() { return getThreadCountByPercent(0.5); } - private final ConfigEntryWithPresetOptions UpdatePropagatorThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads, - new HashMap() + private final ConfigEntryWithPresetOptions UpdatePropagatorThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 1); - this.put(EThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultThreadCount()); - this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.75)); - this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.75)); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1); + this.put(EDhApiThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultThreadCount()); + this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.75)); + this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.75)); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); public static double getUpdatePropagatorDefaultRunTimeRatio() { return 0.25; } - private final ConfigEntryWithPresetOptions UpdatePropagatorRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads, - new HashMap() + private final ConfigEntryWithPresetOptions UpdatePropagatorRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 0.25); - this.put(EThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultRunTimeRatio()); - this.put(EThreadPreset.BALANCED, 0.5); - this.put(EThreadPreset.AGGRESSIVE, 1.0); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EDhApiThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultRunTimeRatio()); + this.put(EDhApiThreadPreset.BALANCED, 0.5); + this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); }}); public static int getLodBuilderDefaultThreadCount() { return getThreadCountByPercent(0.1); } - private final ConfigEntryWithPresetOptions lodBuilderThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads, - new HashMap() + private final ConfigEntryWithPresetOptions lodBuilderThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfLodBuilderThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 1); - this.put(EThreadPreset.LOW_IMPACT, getLodBuilderDefaultThreadCount()); - this.put(EThreadPreset.BALANCED, getThreadCountByPercent(0.2)); - this.put(EThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1); + this.put(EDhApiThreadPreset.LOW_IMPACT, getLodBuilderDefaultThreadCount()); + this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.2)); + this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.4)); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); public static double getLodBuilderDefaultRunTimeRatio() { return LOW_THREAD_COUNT_CPU ? 0.25 : 0.5; } - private final ConfigEntryWithPresetOptions lodBuilderRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForLodBuilderThreads, - new HashMap() + private final ConfigEntryWithPresetOptions lodBuilderRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForLodBuilderThreads, + new HashMap() {{ - this.put(EThreadPreset.MINIMAL_IMPACT, 0.1); - this.put(EThreadPreset.LOW_IMPACT, getLodBuilderDefaultRunTimeRatio()); - this.put(EThreadPreset.BALANCED, LOW_THREAD_COUNT_CPU ? 0.5 : 0.75); - this.put(EThreadPreset.AGGRESSIVE, 1.0); - this.put(EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.1); + this.put(EDhApiThreadPreset.LOW_IMPACT, getLodBuilderDefaultRunTimeRatio()); + this.put(EDhApiThreadPreset.BALANCED, LOW_THREAD_COUNT_CPU ? 0.5 : 0.75); + this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0); + this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); }}); @@ -151,7 +151,7 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.configList.add(this.lodBuilderRunTime); - for (ConfigEntryWithPresetOptions config : this.configList) + for (ConfigEntryWithPresetOptions config : this.configList) { // ignore try-using, the listeners should only ever be added once and should never be removed new ConfigChangeListener<>(config.configEntry, (val) -> { this.onConfigValueChanged(); }); @@ -196,11 +196,11 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan //==============// @Override - protected IConfigEntry getPresetConfigEntry() { return Config.Client.threadPresetSetting; } + protected IConfigEntry getPresetConfigEntry() { return Config.Client.threadPresetSetting; } @Override - protected List getPresetEnumList() { return Arrays.asList(EThreadPreset.values()); } + protected List getPresetEnumList() { return Arrays.asList(EDhApiThreadPreset.values()); } @Override - protected EThreadPreset getCustomPresetEnum() { return EThreadPreset.CUSTOM; } + protected EDhApiThreadPreset getCustomPresetEnum() { return EDhApiThreadPreset.CUSTOM; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/gui/OpenGLConfigScreen.java b/core/src/main/java/com/seibel/distanthorizons/core/config/gui/OpenGLConfigScreen.java index 71dad5506..35a6cf353 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/gui/OpenGLConfigScreen.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/gui/OpenGLConfigScreen.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.config.gui; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.glObject.GLState; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; @@ -76,7 +76,7 @@ public class OpenGLConfigScreen extends AbstractScreen buffer.rewind(); GLVertexBuffer vbo = new GLVertexBuffer(false); vbo.bind(); - vbo.uploadBuffer(buffer, 4, EGpuUploadMethod.DATA, vertices.length * Float.BYTES); + vbo.uploadBuffer(buffer, 4, EDhApiGpuUploadMethod.DATA, vertices.length * Float.BYTES); return vbo; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java index 796aebd4c..05eb3b340 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java @@ -25,13 +25,12 @@ import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EGLProxyContext; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; -import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.objects.StatsMap; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.core.util.*; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Logger; @@ -88,7 +87,7 @@ public class ColumnRenderBuffer implements AutoCloseable //==================// /** Should be run on a DH thread. */ - public void uploadBuffer(LodQuadBuilder builder, EGpuUploadMethod gpuUploadMethod) throws InterruptedException + public void uploadBuffer(LodQuadBuilder builder, EDhApiGpuUploadMethod gpuUploadMethod) throws InterruptedException { LodUtil.assertTrue(Thread.currentThread().getName().startsWith(ThreadUtil.THREAD_NAME_PREFIX), "Buffer uploading needs to be done on a DH thread to prevent locking up any MC threads."); @@ -147,7 +146,7 @@ public class ColumnRenderBuffer implements AutoCloseable } } - private void uploadBuffersUsingUploadMethod(LodQuadBuilder builder, EGpuUploadMethod gpuUploadMethod) throws InterruptedException + private void uploadBuffersUsingUploadMethod(LodQuadBuilder builder, EDhApiGpuUploadMethod gpuUploadMethod) throws InterruptedException { if (gpuUploadMethod.useEarlyMapping) { @@ -163,7 +162,7 @@ public class ColumnRenderBuffer implements AutoCloseable - private void uploadBuffersMapped(LodQuadBuilder builder, EGpuUploadMethod method) + private void uploadBuffersMapped(LodQuadBuilder builder, EDhApiGpuUploadMethod method) { // opaque vbos // @@ -199,7 +198,7 @@ public class ColumnRenderBuffer implements AutoCloseable } } - private void uploadBuffersDirect(LodQuadBuilder builder, EGpuUploadMethod method) throws InterruptedException + private void uploadBuffersDirect(LodQuadBuilder builder, EDhApiGpuUploadMethod method) throws InterruptedException { this.vbos = ColumnRenderBufferBuilder.resizeBuffer(this.vbos, builder.getCurrentNeededOpaqueVertexBufferCount()); uploadBuffersDirect(this.vbos, builder.makeOpaqueVertexBuffers(), method); @@ -207,7 +206,7 @@ public class ColumnRenderBuffer implements AutoCloseable this.vbosTransparent = ColumnRenderBufferBuilder.resizeBuffer(this.vbosTransparent, builder.getCurrentNeededTransparentVertexBufferCount()); uploadBuffersDirect(this.vbosTransparent, builder.makeTransparentVertexBuffers(), method); } - private static void uploadBuffersDirect(GLVertexBuffer[] vbos, Iterator iter, EGpuUploadMethod method) throws InterruptedException + private static void uploadBuffersDirect(GLVertexBuffer[] vbos, Iterator iter, EDhApiGpuUploadMethod method) throws InterruptedException { long remainingMS = 0; long MBPerMS = Config.Client.Advanced.GpuBuffers.gpuUploadPerMegabyteInMilliseconds.get(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 6a7ab9380..6598c2e06 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding; -import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; @@ -152,7 +152,7 @@ public class ColumnRenderBufferBuilder private static void makeLodRenderData(LodQuadBuilder quadBuilder, ColumnRenderSource renderSource, ColumnRenderSource[] adjRegions) { // Variable initialization - EDebugRendering debugMode = Config.Client.Advanced.Debugging.debugRendering.get(); + EDhApiDebugRendering debugMode = Config.Client.Advanced.Debugging.debugRendering.get(); // can be used to limit which section positions are build and thus, rendered // useful when debugging a specific section diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java index dbfebb5d5..ff0ecfd42 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java @@ -25,7 +25,7 @@ import com.seibel.distanthorizons.core.pos.DhLodPos; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; -import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.coreapi.util.BitShiftUtil; @@ -42,7 +42,7 @@ public class CubicLodTemplate public static void addLodToBuffer( long data, long topData, long bottomData, ColumnArrayView[][] adjColumnViews, byte detailLevel, int offsetPosX, int offsetOosZ, LodQuadBuilder quadBuilder, - EDebugRendering debugging, ColumnRenderSource.DebugSourceFlag debugSource) + EDhApiDebugRendering debugging, ColumnRenderSource.DebugSourceFlag debugSource) { DhLodPos blockOffsetPos = new DhLodPos(detailLevel, offsetPosX, offsetOosZ).convertToDetailLevel(LodUtil.BLOCK_DETAIL_LEVEL); @@ -88,12 +88,6 @@ public class CubicLodTemplate break; } case SHOW_DETAIL: - case SHOW_GENMODE: - { - color = LodUtil.DEBUG_DETAIL_LEVEL_COLORS[detailLevel]; - fullBright = true; - break; - } case SHOW_BLOCK_MATERIAL: { switch (blockMaterialId) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index 29acdd418..87f16a9c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -27,7 +27,7 @@ import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.util.ColorUtil; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; @@ -524,7 +524,7 @@ public class LodQuadBuilder } - public BufferFiller makeOpaqueBufferFiller(EGpuUploadMethod method) + public BufferFiller makeOpaqueBufferFiller(EDhApiGpuUploadMethod method) { return new BufferFiller() { @@ -604,7 +604,7 @@ public class LodQuadBuilder }; } - public BufferFiller makeTransparentBufferFiller(EGpuUploadMethod method) + public BufferFiller makeTransparentBufferFiller(EDhApiGpuUploadMethod method) { return new BufferFiller() { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java index 65c3636e0..2aabb2e71 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/FullDataToRenderDataTransformer.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.dataObjects.transformers; -import com.seibel.distanthorizons.api.enums.config.EBlocksToAvoid; +import com.seibel.distanthorizons.api.enums.config.EDhApiBlocksToAvoid; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; @@ -163,7 +163,7 @@ public class FullDataToRenderDataTransformer int blockX, int blockZ, ColumnArrayView renderColumnData, LongArrayList fullColumnData) { - boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EBlocksToAvoid.NON_COLLIDING); + boolean avoidSolidBlocks = (Config.Client.Advanced.Graphics.Quality.blocksToIgnore.get() == EDhApiBlocksToAvoid.NON_COLLIDING); boolean colorBelowWithAvoidedBlocks = Config.Client.Advanced.Graphics.Quality.tintWithAvoidedBlocks.get(); HashSet blockStatesToIgnore = WRAPPER_FACTORY.getRendererIgnoredBlocks(level.getLevelWrapper()); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java b/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java index a0b900f8d..6e9809934 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/structure/ClientOnlySaveStructure.java @@ -23,7 +23,7 @@ import com.google.common.net.PercentEscaper; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.subDimMatching.SubDimensionLevelMatcher; import com.seibel.distanthorizons.core.config.Config; -import com.seibel.distanthorizons.api.enums.config.EServerFolderNameMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiServerFolderNameMode; import com.seibel.distanthorizons.core.level.IServerKeyedClientLevel; import com.seibel.distanthorizons.core.util.objects.ParsedIp; import com.seibel.distanthorizons.core.util.LodUtil; @@ -264,7 +264,7 @@ public class ClientOnlySaveStructure extends AbstractSaveStructure // determine the auto folder name format - EServerFolderNameMode folderNameMode = Config.Client.Advanced.Multiplayer.serverFolderNameMode.get(); + EDhApiServerFolderNameMode folderNameMode = Config.Client.Advanced.Multiplayer.serverFolderNameMode.get(); String serverName = MC_CLIENT.getCurrentServerName().replaceAll(INVALID_FILE_CHARACTERS_REGEX, ""); String serverMcVersion = MC_CLIENT.getCurrentServerVersion().replaceAll(INVALID_FILE_CHARACTERS_REGEX, ""); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 6b424652e..31fcd1158 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.level; -import com.seibel.distanthorizons.api.enums.rendering.EDebugRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; @@ -80,7 +80,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle // tick methods // //==============// - private EDebugRendering lastDebugRendering = EDebugRendering.OFF; + private EDhApiDebugRendering lastDebugRendering = EDhApiDebugRendering.OFF; public void clientTick() { @@ -123,7 +123,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle clientRenderState.quadtree.tick(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos())); boolean isBuffersDirty = false; - EDebugRendering newDebugRendering = Config.Client.Advanced.Debugging.debugRendering.get(); + EDhApiDebugRendering newDebugRendering = Config.Client.Advanced.Debugging.debugRendering.get(); if (newDebugRendering != lastDebugRendering) { lastDebugRendering = newDebugRendering; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java b/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java index 9471d77af..798080895 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedLogger.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.logging; -import com.seibel.distanthorizons.api.enums.config.ELoggerMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiLoggerMode; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Level; @@ -50,11 +50,11 @@ public class ConfigBasedLogger }); } - private ELoggerMode mode; - private final Supplier getter; + private EDhApiLoggerMode mode; + private final Supplier getter; private final Logger logger; - public ConfigBasedLogger(Logger logger, Supplier configQuery) + public ConfigBasedLogger(Logger logger, Supplier configQuery) { getter = configQuery; mode = getter.get(); @@ -85,7 +85,7 @@ public class ConfigBasedLogger public boolean canMaybeLog() { - return mode != ELoggerMode.DISABLED; + return mode != EDhApiLoggerMode.DISABLED; } public void log(Level level, String str, Object... param) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedSpamLogger.java b/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedSpamLogger.java index 4c2ba312c..82f9258dc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedSpamLogger.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/logging/ConfigBasedSpamLogger.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.logging; -import com.seibel.distanthorizons.api.enums.config.ELoggerMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiLoggerMode; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Level; @@ -52,13 +52,13 @@ public class ConfigBasedSpamLogger }); } - private ELoggerMode mode; - private final Supplier getter; + private EDhApiLoggerMode mode; + private final Supplier getter; private final int maxLogCount; private final AtomicInteger logTries = new AtomicInteger(0); private final Logger logger; - public ConfigBasedSpamLogger(Logger logger, Supplier configQuery, int maxLogPerSec) + public ConfigBasedSpamLogger(Logger logger, Supplier configQuery, int maxLogPerSec) { getter = configQuery; mode = getter.get(); @@ -74,7 +74,7 @@ public class ConfigBasedSpamLogger public boolean canMaybeLog() { - return mode != ELoggerMode.DISABLED && logTries.get() < maxLogCount; + return mode != EDhApiLoggerMode.DISABLED && logTries.get() < maxLogCount; } public void update() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index bc9761931..203ceee3d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render; -import com.seibel.distanthorizons.api.enums.config.EHorizontalQuality; +import com.seibel.distanthorizons.api.enums.config.EDhApiHorizontalQuality; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource; @@ -67,7 +67,7 @@ public class LodQuadTree extends QuadTree implements AutoClose */ private final ConcurrentLinkedQueue sectionsToReload = new ConcurrentLinkedQueue<>(); private final IDhClientLevel level; //FIXME: Proper hierarchy to remove this reference! - private final ConfigChangeListener horizontalScaleChangeListener; + private final ConfigChangeListener horizontalScaleChangeListener; private final ReentrantLock treeReadWriteLock = new ReentrantLock(); private final AtomicBoolean fullDataRetrievalQueueRunning = new AtomicBoolean(false); @@ -369,7 +369,7 @@ public class LodQuadTree extends QuadTree implements AutoClose private byte getDetailLevelFromDistance(double distance) { // special case, never drop the quality - if (Config.Client.Advanced.Graphics.Quality.horizontalQuality.get() == EHorizontalQuality.UNLIMITED) + if (Config.Client.Advanced.Graphics.Quality.horizontalQuality.get() == EDhApiHorizontalQuality.UNLIMITED) { return this.maxRenderDetailLevel; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/fog/FogSettings.java b/core/src/main/java/com/seibel/distanthorizons/core/render/fog/FogSettings.java index 98fa739ca..12db4070d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/fog/FogSettings.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/fog/FogSettings.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render.fog; -import com.seibel.distanthorizons.api.enums.rendering.EFogFalloff; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogFalloff; import java.util.Objects; @@ -31,7 +31,7 @@ import java.util.Objects; public class FogSettings { /** a FogSetting object with 0 for every value */ - public static final FogSettings EMPTY = new FogSettings(0, 0, 0, 0, 0, EFogFalloff.LINEAR); + public static final FogSettings EMPTY = new FogSettings(0, 0, 0, 0, 0, EDhApiFogFalloff.LINEAR); public final double start; @@ -39,9 +39,9 @@ public class FogSettings public final double min; public final double max; public final double density; - public final EFogFalloff fogType; + public final EDhApiFogFalloff fogType; - public FogSettings(double start, double end, double min, double max, double density, EFogFalloff fogType) + public FogSettings(double start, double end, double min, double max, double density, EDhApiFogFalloff fogType) { this.start = start; this.end = end; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java index a67e992fe..5bc86eb2e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java @@ -48,8 +48,8 @@ public class LodFogConfig public final FogSettings farFogSetting; public final FogSettings heightFogSetting; - public final EHeightFogMixMode heightFogMixMode; - public final EHeightFogMode heightFogMode; + public final EDhApiHeightFogMixMode heightFogMixMode; + public final EDhApiHeightFogMode heightFogMode; public final float heightFogHeight; final boolean drawNearFog; @@ -66,8 +66,8 @@ public class LodFogConfig public static LodFogConfig generateFogConfig() { - EFogDrawMode fogMode = Config.Client.Advanced.Graphics.Fog.drawMode.get(); - if (fogMode == EFogDrawMode.USE_OPTIFINE_SETTING && OPTIFINE != null) + EDhApiFogDrawMode fogMode = Config.Client.Advanced.Graphics.Fog.drawMode.get(); + if (fogMode == EDhApiFogDrawMode.USE_OPTIFINE_SETTING && OPTIFINE != null) { fogMode = OPTIFINE.getFogDrawMode(); } @@ -75,7 +75,7 @@ public class LodFogConfig } /** sets all fog options from the config */ - private LodFogConfig(EFogDrawMode fogDrawMode) + private LodFogConfig(EDhApiFogDrawMode fogDrawMode) { // TODO: Move these out of here earthCurveRatio = Config.Client.Advanced.Graphics.AdvancedGraphics.earthCurveRatio.get(); @@ -86,12 +86,12 @@ public class LodFogConfig noiseDropoff = Config.Client.Advanced.Graphics.NoiseTextureSettings.noiseDropoff.get(); - if (fogDrawMode != EFogDrawMode.FOG_DISABLED) + if (fogDrawMode != EDhApiFogDrawMode.FOG_DISABLED) { - EFogDistance fogDistance = Config.Client.Advanced.Graphics.Fog.distance.get(); - drawNearFog = (fogDistance == EFogDistance.NEAR || fogDistance == EFogDistance.NEAR_AND_FAR); + EDhApiFogDistance fogDistance = Config.Client.Advanced.Graphics.Fog.distance.get(); + drawNearFog = (fogDistance == EDhApiFogDistance.NEAR || fogDistance == EDhApiFogDistance.NEAR_AND_FAR); - if (fogDistance == EFogDistance.FAR || fogDistance == EFogDistance.NEAR_AND_FAR) + if (fogDistance == EDhApiFogDistance.FAR || fogDistance == EDhApiFogDistance.NEAR_AND_FAR) { // far fog should be drawn @@ -105,7 +105,7 @@ public class LodFogConfig ); heightFogMixMode = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMixMode.get(); - if (heightFogMixMode == EHeightFogMixMode.IGNORE_HEIGHT || heightFogMixMode == EHeightFogMixMode.BASIC) + if (heightFogMixMode == EDhApiHeightFogMixMode.IGNORE_HEIGHT || heightFogMixMode == EDhApiHeightFogMixMode.BASIC) { // basic fog mixing @@ -238,7 +238,7 @@ public class LodFogConfig str.append("" + "float calculateFarFogDepth(float horizontal, float dist, float nearFogStart) \n" + "{ \n" + - " return " + (heightFogMixMode == EHeightFogMixMode.BASIC ? + " return " + (heightFogMixMode == EDhApiHeightFogMixMode.BASIC ? "(dist - nearFogStart)/(1.0 - nearFogStart);" : "(horizontal - nearFogStart)/(1.0 - nearFogStart);") + "} \n"); @@ -286,7 +286,7 @@ public class LodFogConfig return str; } - private static String getFarFogMethod(EFogFalloff fogType) + private static String getFarFogMethod(EDhApiFogFalloff fogType) { switch (fogType) { @@ -302,7 +302,7 @@ public class LodFogConfig } } - private static String getHeightDepthMethod(EHeightFogMode heightMode, float heightFogHeight) + private static String getHeightDepthMethod(EDhApiHeightFogMode heightMode, float heightFogHeight) { String str = ""; if (!heightMode.basedOnCamera) @@ -334,7 +334,7 @@ public class LodFogConfig * Example:
* " return linearFog(dist, heightFogStart, heightFogLength, heightFogMin, heightFogRange);" */ - private static String getHeightFogMethod(EFogFalloff fogType) + private static String getHeightFogMethod(EDhApiFogFalloff fogType) { switch (fogType) { @@ -354,7 +354,7 @@ public class LodFogConfig * creates a line in the format
* " return max(1.0-near, far);" */ - private static String getMixFogLine(EHeightFogMixMode heightFogMode, boolean drawNearFog) + private static String getMixFogLine(EDhApiHeightFogMixMode heightFogMode, boolean drawNearFog) { String str = " return "; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java index b53f782a5..c49f74ce9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/GLProxy.java @@ -20,9 +20,9 @@ package com.seibel.distanthorizons.core.render.glObject; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.seibel.distanthorizons.api.enums.config.EGLErrorHandlingMode; -import com.seibel.distanthorizons.api.enums.config.EGlProfileMode; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGLErrorHandlingMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiGlProfileMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EGLProxyContext; @@ -107,7 +107,7 @@ public class GLProxy public boolean bufferStorageSupported = false; // ~OpenGL 4.4 public boolean VertexAttributeBufferBindingSupported = false; // ~OpenGL 4.3 - private final EGpuUploadMethod preferredUploadMethod; + private final EDhApiGpuUploadMethod preferredUploadMethod; public final GLMessage.Builder vanillaDebugMessageBuilder = GLMessage.Builder.DEFAULT_MESSAGE_BUILDER; public final GLMessage.Builder lodBuilderDebugMessageBuilder = GLMessage.Builder.DEFAULT_MESSAGE_BUILDER; @@ -209,7 +209,7 @@ public class GLProxy GLFW.glfwWindowHint(GLFW.GLFW_OPENGL_FORWARD_COMPAT, forwardCompatEnabled ? GLFW.GLFW_TRUE : GLFW.GLFW_FALSE); int profileModeInt; - EGlProfileMode profileModeEnum = Config.Client.Advanced.Debugging.OpenGl.glProfileMode.get(); + EDhApiGlProfileMode profileModeEnum = Config.Client.Advanced.Debugging.OpenGl.glProfileMode.get(); switch (profileModeEnum) { case CORE: @@ -322,12 +322,12 @@ public class GLProxy if (vendor.contains("NVIDIA") || vendor.contains("GEFORCE")) { // NVIDIA card - this.preferredUploadMethod = this.bufferStorageSupported ? EGpuUploadMethod.BUFFER_STORAGE : EGpuUploadMethod.SUB_DATA; + this.preferredUploadMethod = this.bufferStorageSupported ? EDhApiGpuUploadMethod.BUFFER_STORAGE : EDhApiGpuUploadMethod.SUB_DATA; } else { // AMD or Intel card - this.preferredUploadMethod = this.bufferStorageSupported ? EGpuUploadMethod.BUFFER_STORAGE : EGpuUploadMethod.DATA; + this.preferredUploadMethod = this.bufferStorageSupported ? EDhApiGpuUploadMethod.BUFFER_STORAGE : EDhApiGpuUploadMethod.DATA; } GL_LOGGER.info("GPU Vendor [" + vendor + "], Preferred upload method is [" + this.preferredUploadMethod + "]."); @@ -445,16 +445,16 @@ public class GLProxy return instance; } - public EGpuUploadMethod getGpuUploadMethod() + public EDhApiGpuUploadMethod getGpuUploadMethod() { - EGpuUploadMethod method = Config.Client.Advanced.GpuBuffers.gpuUploadMethod.get(); - if (!this.bufferStorageSupported && method == EGpuUploadMethod.BUFFER_STORAGE) + EDhApiGpuUploadMethod method = Config.Client.Advanced.GpuBuffers.gpuUploadMethod.get(); + if (!this.bufferStorageSupported && method == EDhApiGpuUploadMethod.BUFFER_STORAGE) { // if buffer storage isn't supported // default to DATA since that is the most compatible - method = EGpuUploadMethod.DATA; + method = EDhApiGpuUploadMethod.DATA; } - return method == EGpuUploadMethod.AUTO ? this.preferredUploadMethod : method; + return method == EDhApiGpuUploadMethod.AUTO ? this.preferredUploadMethod : method; } @@ -567,8 +567,8 @@ public class GLProxy private static void logMessage(GLMessage msg) { - EGLErrorHandlingMode errorHandlingMode = Config.Client.Advanced.Debugging.OpenGl.glErrorHandlingMode.get(); - if (errorHandlingMode == EGLErrorHandlingMode.IGNORE) + EDhApiGLErrorHandlingMode errorHandlingMode = Config.Client.Advanced.Debugging.OpenGl.glErrorHandlingMode.get(); + if (errorHandlingMode == EDhApiGLErrorHandlingMode.IGNORE) { return; } @@ -581,7 +581,7 @@ public class GLProxy GL_LOGGER.error("GL ERROR " + msg.id + " from " + msg.source + ": " + msg.message); - if (errorHandlingMode == EGLErrorHandlingMode.LOG_THROW) + if (errorHandlingMode == EDhApiGLErrorHandlingMode.LOG_THROW) { throw new RuntimeException("GL ERROR: " + msg); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java index da1d5f942..112a13cd0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLBuffer.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render.glObject.buffer; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.core.enums.EGLProxyContext; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.render.glObject.GLProxy; @@ -163,7 +163,7 @@ public class GLBuffer implements AutoCloseable * Assumes the GL Context is already bound.
* Will create the VBO if one exist. */ - public void uploadBuffer(ByteBuffer bb, EGpuUploadMethod uploadMethod, int maxExpansionSize, int bufferHint) + public void uploadBuffer(ByteBuffer bb, EDhApiGpuUploadMethod uploadMethod, int maxExpansionSize, int bufferHint) { LodUtil.assertTrue(!uploadMethod.useEarlyMapping, "UploadMethod signal that this should use Mapping instead of uploadBuffer!"); int bbSize = bb.limit() - bb.position(); @@ -242,7 +242,7 @@ public class GLBuffer implements AutoCloseable // buffer mapping // //================// - public ByteBuffer mapBuffer(int targetSize, EGpuUploadMethod uploadMethod, int maxExpensionSize, int bufferHint, int mapFlags) + public ByteBuffer mapBuffer(int targetSize, EDhApiGpuUploadMethod uploadMethod, int maxExpensionSize, int bufferHint, int mapFlags) { LodUtil.assertTrue(targetSize != 0, "MapBuffer targetSize is 0"); LodUtil.assertTrue(uploadMethod.useEarlyMapping, "Upload method must be one that use early mappings in order to call mapBuffer"); @@ -312,7 +312,7 @@ public class GLBuffer implements AutoCloseable * Makes sure the buffer exists and is of the correct format * before uploading. */ - private void createOrChangeBufferTypeForUpload(EGpuUploadMethod uploadMethod) + private void createOrChangeBufferTypeForUpload(EDhApiGpuUploadMethod uploadMethod) { // create/change the buffer type if necessary if (uploadMethod.useBufferStorage != this.bufferStorage) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLVertexBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLVertexBuffer.java index 23bf21b0f..05fbe1287 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLVertexBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/GLVertexBuffer.java @@ -21,11 +21,9 @@ package com.seibel.distanthorizons.core.render.glObject.buffer; import java.nio.ByteBuffer; -import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import org.apache.logging.log4j.Logger; import org.lwjgl.opengl.GL32; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; /** * This is a container for a OpenGL @@ -63,7 +61,7 @@ public class GLVertexBuffer extends GLBuffer @Override public int getBufferBindingTarget() { return GL32.GL_ARRAY_BUFFER; } - public void uploadBuffer(ByteBuffer byteBuffer, int vertCount, EGpuUploadMethod uploadMethod, int maxExpensionSize) + public void uploadBuffer(ByteBuffer byteBuffer, int vertCount, EDhApiGpuUploadMethod uploadMethod, int maxExpensionSize) { if (vertCount < 0) { @@ -79,7 +77,7 @@ public class GLVertexBuffer extends GLBuffer this.vertexCount = vertCount; } - public ByteBuffer mapBuffer(int targetSize, EGpuUploadMethod uploadMethod, int maxExpensionSize) + public ByteBuffer mapBuffer(int targetSize, EDhApiGpuUploadMethod uploadMethod, int maxExpensionSize) { return super.mapBuffer(targetSize, uploadMethod, maxExpensionSize, uploadMethod.useBufferStorage ? GL32.GL_MAP_WRITE_BIT : diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/QuadElementBuffer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/QuadElementBuffer.java index 900bf4f90..915d70c3d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/QuadElementBuffer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/glObject/buffer/QuadElementBuffer.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render.glObject.buffer; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.render.glObject.GLEnums; import com.seibel.distanthorizons.core.render.glObject.GLProxy; @@ -160,13 +160,13 @@ public class QuadElementBuffer extends GLElementBuffer { bind(); - super.uploadBuffer(buffer, EGpuUploadMethod.DATA, + super.uploadBuffer(buffer, EDhApiGpuUploadMethod.DATA, indicesCount * GLEnums.getTypeSize(type), GL32.GL_STATIC_DRAW); } else { bind(); - super.uploadBuffer(buffer, EGpuUploadMethod.BUFFER_STORAGE, + super.uploadBuffer(buffer, EDhApiGpuUploadMethod.BUFFER_STORAGE, indicesCount * GLEnums.getTypeSize(type), 0); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java index 92fec9b71..c6fa77bbc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/DebugRenderer.java @@ -19,8 +19,8 @@ package com.seibel.distanthorizons.core.render.renderer; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; -import com.seibel.distanthorizons.api.enums.config.ELoggerMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiLoggerMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.config.types.ConfigEntry; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; @@ -56,8 +56,8 @@ public class DebugRenderer { public static DebugRenderer INSTANCE = new DebugRenderer(); - public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT); - public static final ConfigBasedSpamLogger SPAM_LOGGER = new ConfigBasedSpamLogger(LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT, 1); + public static final ConfigBasedLogger LOGGER = new ConfigBasedLogger(LogManager.getLogger(TestRenderer.class), () -> EDhApiLoggerMode.LOG_ALL_TO_CHAT); + public static final ConfigBasedSpamLogger SPAM_LOGGER = new ConfigBasedSpamLogger(LogManager.getLogger(TestRenderer.class), () -> EDhApiLoggerMode.LOG_ALL_TO_CHAT, 1); private static final IMinecraftRenderWrapper MC_RENDER = SingletonInjector.INSTANCE.get(IMinecraftRenderWrapper.class); @@ -174,7 +174,7 @@ public class DebugRenderer this.boxBuffer = new GLVertexBuffer(false); this.boxBuffer.bind(); - this.boxBuffer.uploadBuffer(buffer, 8, EGpuUploadMethod.DATA, box_vertices.length * Float.BYTES); + this.boxBuffer.uploadBuffer(buffer, 8, EDhApiGpuUploadMethod.DATA, box_vertices.length * Float.BYTES); buffer = ByteBuffer.allocateDirect(box_outline_indices.length * Integer.BYTES); buffer.order(ByteOrder.nativeOrder()); @@ -182,7 +182,7 @@ public class DebugRenderer buffer.rewind(); this.boxOutlineBuffer = new GLElementBuffer(false); - this.boxOutlineBuffer.uploadBuffer(buffer, EGpuUploadMethod.DATA, box_outline_indices.length * Integer.BYTES, GL32.GL_STATIC_DRAW); + this.boxOutlineBuffer.uploadBuffer(buffer, EDhApiGpuUploadMethod.DATA, box_outline_indices.length * Integer.BYTES, GL32.GL_STATIC_DRAW); } public void render(Mat4f transform) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index 6b8d983e8..78de044ef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render.renderer; -import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiFramebuffer; import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiShaderProgram; import com.seibel.distanthorizons.api.methods.events.abstractEvents.*; @@ -43,7 +43,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftCli import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.misc.ILightMapWrapper; -import com.seibel.distanthorizons.api.enums.rendering.EFogColorMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogColorMode; import com.seibel.distanthorizons.core.render.fog.LodFogConfig; import com.seibel.distanthorizons.core.wrapperInterfaces.modAccessor.AbstractOptifineAccessor; import com.seibel.distanthorizons.core.wrapperInterfaces.modAccessor.IIrisAccessor; @@ -384,7 +384,7 @@ public class LodRenderer } - if (Config.Client.Advanced.Graphics.Fog.drawMode.get() != EFogDrawMode.FOG_DISABLED) + if (Config.Client.Advanced.Graphics.Fog.drawMode.get() != EDhApiFogDrawMode.FOG_DISABLED) { profiler.popPush("LOD Fog"); @@ -520,7 +520,7 @@ public class LodRenderer GL32.glDepthMask(true); // Apparently the depth mask state is stored in the FBO, so glState fails to restore it... - if (Config.Client.Advanced.Graphics.Fog.drawMode.get() != EFogDrawMode.FOG_DISABLED) + if (Config.Client.Advanced.Graphics.Fog.drawMode.get() != EDhApiFogDrawMode.FOG_DISABLED) { profiler.popPush("LOD Fog"); FogShader.INSTANCE.render(partialTicks); @@ -796,7 +796,7 @@ public class LodRenderer { Color fogColor; - if (Config.Client.Advanced.Graphics.Fog.colorMode.get() == EFogColorMode.USE_SKY_COLOR) + if (Config.Client.Advanced.Graphics.Fog.colorMode.get() == EDhApiFogColorMode.USE_SKY_COLOR) { fogColor = MC_RENDER.getSkyColor(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/ScreenQuad.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/ScreenQuad.java index c250d69d3..2f9abd294 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/ScreenQuad.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/ScreenQuad.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render.renderer; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.render.glObject.vertexAttribute.AbstractVertexAttribute; import com.seibel.distanthorizons.core.render.glObject.vertexAttribute.VertexPointer; @@ -93,7 +93,7 @@ public class ScreenQuad this.boxBuffer = new GLVertexBuffer(false); this.boxBuffer.bind(); - this.boxBuffer.uploadBuffer(buffer, box_vertices.length, EGpuUploadMethod.DATA, box_vertices.length * Float.BYTES); + this.boxBuffer.uploadBuffer(buffer, box_vertices.length, EDhApiGpuUploadMethod.DATA, box_vertices.length * Float.BYTES); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/TestRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/TestRenderer.java index 2c871f892..5d13b95d5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/TestRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/TestRenderer.java @@ -19,8 +19,8 @@ package com.seibel.distanthorizons.core.render.renderer; -import com.seibel.distanthorizons.api.enums.config.EGpuUploadMethod; -import com.seibel.distanthorizons.api.enums.config.ELoggerMode; +import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; +import com.seibel.distanthorizons.api.enums.config.EDhApiLoggerMode; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.logging.ConfigBasedSpamLogger; @@ -45,9 +45,9 @@ public class TestRenderer public TestRenderer() { } public static final ConfigBasedLogger logger = new ConfigBasedLogger( - LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT); + LogManager.getLogger(TestRenderer.class), () -> EDhApiLoggerMode.LOG_ALL_TO_CHAT); public static final ConfigBasedSpamLogger spamLogger = new ConfigBasedSpamLogger( - LogManager.getLogger(TestRenderer.class), () -> ELoggerMode.LOG_ALL_TO_CHAT, 1); + LogManager.getLogger(TestRenderer.class), () -> EDhApiLoggerMode.LOG_ALL_TO_CHAT, 1); private static final IMinecraftRenderWrapper MC_RENDER = SingletonInjector.INSTANCE.get(IMinecraftRenderWrapper.class); ShaderProgram basicShader; @@ -91,7 +91,7 @@ public class TestRenderer buffer.rewind(); GLVertexBuffer vbo = new GLVertexBuffer(false); vbo.bind(); - vbo.uploadBuffer(buffer, 4, EGpuUploadMethod.DATA, vertices.length * Float.BYTES); + vbo.uploadBuffer(buffer, 4, EDhApiGpuUploadMethod.DATA, vertices.length * Float.BYTES); return vbo; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/shaders/FogShader.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/shaders/FogShader.java index 2d5a3491c..e5a1fc255 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/shaders/FogShader.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/shaders/FogShader.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.render.renderer.shaders; -import com.seibel.distanthorizons.api.enums.rendering.EFogColorMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogColorMode; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.render.fog.LodFogConfig; @@ -29,7 +29,6 @@ import com.seibel.distanthorizons.core.render.glObject.shader.ShaderProgram; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; import com.seibel.distanthorizons.core.render.renderer.ScreenQuad; import com.seibel.distanthorizons.core.util.LodUtil; -import com.seibel.distanthorizons.core.util.RenderUtil; import com.seibel.distanthorizons.core.wrapperInterfaces.IVersionConstants; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.coreapi.util.math.Mat4f; @@ -124,7 +123,7 @@ public class FogShader extends AbstractShaderRenderer { Color fogColor; - if (Config.Client.Advanced.Graphics.Fog.colorMode.get() == EFogColorMode.USE_SKY_COLOR) + if (Config.Client.Advanced.Graphics.Fog.colorMode.get() == EDhApiFogColorMode.USE_SKY_COLOR) { fogColor = MC_RENDER.getSkyColor(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java index 28bfdd83e..7b43cf6a8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/LodUtil.java @@ -24,7 +24,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CompletionException; import java.util.concurrent.RejectedExecutionException; -import com.seibel.distanthorizons.api.enums.config.EVanillaOverdraw; +import com.seibel.distanthorizons.api.enums.config.EDhApiVanillaOverdraw; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhChunkPos; @@ -179,10 +179,11 @@ public class LodUtil public static int computeOverdrawOffset() { int chunkRenderDist = MC_RENDER.getRenderDistance() + 1; - EVanillaOverdraw overdraw = EVanillaOverdraw.ALWAYS; //Config.Client.Advanced.Graphics.AdvancedGraphics.vanillaOverdraw.get(); - if (overdraw == EVanillaOverdraw.ALWAYS) return Integer.MAX_VALUE; + EDhApiVanillaOverdraw overdraw = EDhApiVanillaOverdraw.ALWAYS; //Config.Client.Advanced.Graphics.AdvancedGraphics.vanillaOverdraw.get(); + if (overdraw == EDhApiVanillaOverdraw.ALWAYS) return Integer.MAX_VALUE; + int offset; - if (overdraw == EVanillaOverdraw.NEVER) + if (overdraw == EDhApiVanillaOverdraw.NEVER) { offset = 0; //Config.Client.Advanced.Graphics.AdvancedGraphics.overdrawOffset.get(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/AbstractOptifineAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/AbstractOptifineAccessor.java index 29683a624..4355e790b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/AbstractOptifineAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/AbstractOptifineAccessor.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.wrapperInterfaces.modAccessor; -import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; @@ -75,14 +75,14 @@ public abstract class AbstractOptifineAccessor implements IOptifineAccessor //===================// @Override - public EFogDrawMode getFogDrawMode() + public EDhApiFogDrawMode getFogDrawMode() { if (this.ofFogField == null) { // either optifine isn't installed, // the variable name was changed, or // the setup method wasn't called yet. - return EFogDrawMode.FOG_ENABLED; + return EDhApiFogDrawMode.FOG_ENABLED; } int returnNum = 0; @@ -105,9 +105,9 @@ public abstract class AbstractOptifineAccessor implements IOptifineAccessor // normal options case 1: // fast case 2: // fancy - return EFogDrawMode.FOG_ENABLED; + return EDhApiFogDrawMode.FOG_ENABLED; case 3: // off - return EFogDrawMode.FOG_DISABLED; + return EDhApiFogDrawMode.FOG_DISABLED; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/IOptifineAccessor.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/IOptifineAccessor.java index 35e6e8dbe..26a46c34a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/IOptifineAccessor.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/modAccessor/IOptifineAccessor.java @@ -19,7 +19,7 @@ package com.seibel.distanthorizons.core.wrapperInterfaces.modAccessor; -import com.seibel.distanthorizons.api.enums.rendering.EFogDrawMode; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.core.pos.DhChunkPos; import java.util.HashSet; @@ -31,7 +31,7 @@ public interface IOptifineAccessor extends IModAccessor HashSet getNormalRenderedChunks(); /** Get what type of fog optifine is currently set to render. */ - EFogDrawMode getFogDrawMode(); + EDhApiFogDrawMode getFogDrawMode(); /** * Returns the percentage multiplier of the screen's current resolution.
diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index f8d189706..d8f6bdc19 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -616,142 +616,142 @@ - "distanthorizons.config.enum.EQualityPreset.CUSTOM": + "distanthorizons.config.enum.EDhApiQualityPreset.CUSTOM": "Custom", - "distanthorizons.config.enum.EQualityPreset.MINIMUM": + "distanthorizons.config.enum.EDhApiQualityPreset.MINIMUM": "Minimum", - "distanthorizons.config.enum.EQualityPreset.LOW": + "distanthorizons.config.enum.EDhApiQualityPreset.LOW": "Low", - "distanthorizons.config.enum.EQualityPreset.MEDIUM": + "distanthorizons.config.enum.EDhApiQualityPreset.MEDIUM": "Medium", - "distanthorizons.config.enum.EQualityPreset.HIGH": + "distanthorizons.config.enum.EDhApiQualityPreset.HIGH": "High", - "distanthorizons.config.enum.EQualityPreset.EXTREME": + "distanthorizons.config.enum.EDhApiQualityPreset.EXTREME": "Extreme", - "distanthorizons.config.enum.EThreadPreset.CUSTOM": + "distanthorizons.config.enum.EDhApiThreadPreset.CUSTOM": "Custom", - "distanthorizons.config.enum.EThreadPreset.MINIMAL_IMPACT": + "distanthorizons.config.enum.EDhApiThreadPreset.MINIMAL_IMPACT": "Minimal Impact", - "distanthorizons.config.enum.EThreadPreset.LOW_IMPACT": + "distanthorizons.config.enum.EDhApiThreadPreset.LOW_IMPACT": "Low Impact", - "distanthorizons.config.enum.EThreadPreset.BALANCED": + "distanthorizons.config.enum.EDhApiThreadPreset.BALANCED": "Balanced", - "distanthorizons.config.enum.EThreadPreset.AGGRESSIVE": + "distanthorizons.config.enum.EDhApiThreadPreset.AGGRESSIVE": "Aggressive", - "distanthorizons.config.enum.EThreadPreset.I_PAID_FOR_THE_WHOLE_CPU": + "distanthorizons.config.enum.EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU": "I Paid For The Whole CPU", - "distanthorizons.config.enum.EMaxHorizontalResolution.BLOCK": + "distanthorizons.config.enum.EDhApiMaxHorizontalResolution.BLOCK": "Block", - "distanthorizons.config.enum.EMaxHorizontalResolution.TWO_BLOCKS": + "distanthorizons.config.enum.EDhApiMaxHorizontalResolution.TWO_BLOCKS": "2 blocks", - "distanthorizons.config.enum.EMaxHorizontalResolution.FOUR_BLOCKS": + "distanthorizons.config.enum.EDhApiMaxHorizontalResolution.FOUR_BLOCKS": "4 blocks", - "distanthorizons.config.enum.EMaxHorizontalResolution.HALF_CHUNK": + "distanthorizons.config.enum.EDhApiMaxHorizontalResolution.HALF_CHUNK": "Half a chunk", - "distanthorizons.config.enum.EMaxHorizontalResolution.CHUNK": + "distanthorizons.config.enum.EDhApiMaxHorizontalResolution.CHUNK": "Chunk", - "distanthorizons.config.enum.EVerticalQuality.HEIGHT_MAP": + "distanthorizons.config.enum.EDhApiVerticalQuality.HEIGHT_MAP": "1. Height Map", - "distanthorizons.config.enum.EVerticalQuality.LOW": + "distanthorizons.config.enum.EDhApiVerticalQuality.LOW": "2. Low", - "distanthorizons.config.enum.EVerticalQuality.MEDIUM": + "distanthorizons.config.enum.EDhApiVerticalQuality.MEDIUM": "3. Medium", - "distanthorizons.config.enum.EVerticalQuality.HIGH": + "distanthorizons.config.enum.EDhApiVerticalQuality.HIGH": "4. High", - "distanthorizons.config.enum.EVerticalQuality.VERY_HIGH": + "distanthorizons.config.enum.EDhApiVerticalQuality.VERY_HIGH": "5. Very High", - "distanthorizons.config.enum.EVerticalQuality.EXTREME": + "distanthorizons.config.enum.EDhApiVerticalQuality.EXTREME": "6. Extreme", - "distanthorizons.config.enum.EVerticalQuality.PIXEL_ART": + "distanthorizons.config.enum.EDhApiVerticalQuality.PIXEL_ART": "7. Pixel Art", - "distanthorizons.config.enum.EHorizontalQuality.LOWEST": + "distanthorizons.config.enum.EDhApiHorizontalQuality.LOWEST": "Lowest", - "distanthorizons.config.enum.EHorizontalQuality.LOW": + "distanthorizons.config.enum.EDhApiHorizontalQuality.LOW": "Low", - "distanthorizons.config.enum.EHorizontalQuality.MEDIUM": + "distanthorizons.config.enum.EDhApiHorizontalQuality.MEDIUM": "Medium", - "distanthorizons.config.enum.EHorizontalQuality.HIGH": + "distanthorizons.config.enum.EDhApiHorizontalQuality.HIGH": "High", - "distanthorizons.config.enum.EHorizontalQuality.EXTREME": + "distanthorizons.config.enum.EDhApiHorizontalQuality.EXTREME": "Extreme", - "distanthorizons.config.enum.EHorizontalQuality.UNLIMITED": + "distanthorizons.config.enum.EDhApiHorizontalQuality.UNLIMITED": "Unlimited", - "distanthorizons.config.enum.ETransparency.DISABLED": + "distanthorizons.config.enum.EDhApiTransparency.DISABLED": "Disabled", - "distanthorizons.config.enum.ETransparency.FAKE": + "distanthorizons.config.enum.EDhApiTransparency.FAKE": "Fake", - "distanthorizons.config.enum.ETransparency.COMPLETE": + "distanthorizons.config.enum.EDhApiTransparency.COMPLETE": "Complete", - "distanthorizons.config.enum.EFogDistance.NEAR": + "distanthorizons.config.enum.EDhApiFogDistance.NEAR": "Near", - "distanthorizons.config.enum.EFogDistance.FAR": + "distanthorizons.config.enum.EDhApiFogDistance.FAR": "Far", - "distanthorizons.config.enum.EFogDistance.NEAR_AND_FAR": + "distanthorizons.config.enum.EDhApiFogDistance.NEAR_AND_FAR": "Near and far", - "distanthorizons.config.enum.EFogDrawMode.USE_OPTIFINE_SETTING": + "distanthorizons.config.enum.EDhApiFogDrawMode.USE_OPTIFINE_SETTING": "Use modded settings", - "distanthorizons.config.enum.EFogDrawMode.FOG_ENABLED": + "distanthorizons.config.enum.EDhApiFogDrawMode.FOG_ENABLED": "Enabled", - "distanthorizons.config.enum.EFogDrawMode.FOG_DISABLED": + "distanthorizons.config.enum.EDhApiFogDrawMode.FOG_DISABLED": "Disabled", - "distanthorizons.config.enum.EFogColorMode.USE_WORLD_FOG_COLOR": + "distanthorizons.config.enum.EDhApiFogColorMode.USE_WORLD_FOG_COLOR": "Use world fog", - "distanthorizons.config.enum.EFogColorMode.USE_SKY_COLOR": + "distanthorizons.config.enum.EDhApiFogColorMode.USE_SKY_COLOR": "Use sky color", - "distanthorizons.config.enum.EFogFalloff.LINEAR": + "distanthorizons.config.enum.EDhApiFogFalloff.LINEAR": "Linear", - "distanthorizons.config.enum.EFogFalloff.EXPONENTIAL": + "distanthorizons.config.enum.EDhApiFogFalloff.EXPONENTIAL": "Exponential", - "distanthorizons.config.enum.EFogFalloff.EXPONENTIAL_SQUARED": + "distanthorizons.config.enum.EDhApiFogFalloff.EXPONENTIAL_SQUARED": "Exponential squared", - "distanthorizons.config.enum.EHeightFogMixMode.BASIC": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.BASIC": "Basic", - "distanthorizons.config.enum.EHeightFogMixMode.IGNORE_HEIGHT": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.IGNORE_HEIGHT": "Ignore Height", - "distanthorizons.config.enum.EHeightFogMixMode.ADDITION": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.ADDITION": "Addition", - "distanthorizons.config.enum.EHeightFogMixMode.MAX": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.MAX": "Max", - "distanthorizons.config.enum.EHeightFogMixMode.MULTIPLY": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.MULTIPLY": "Multiply", - "distanthorizons.config.enum.EHeightFogMixMode.INVERSE_MULTIPLY": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.INVERSE_MULTIPLY": "Inverse Multiply", - "distanthorizons.config.enum.EHeightFogMixMode.LIMITED_ADDITION": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.LIMITED_ADDITION": "Limited Addition", - "distanthorizons.config.enum.EHeightFogMixMode.MULTIPLY_ADDITION": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.MULTIPLY_ADDITION": "Multiply Addition", - "distanthorizons.config.enum.EHeightFogMixMode.INVERSE_MULTIPLY_ADDITION": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.INVERSE_MULTIPLY_ADDITION": "Inverse Multiply Addition", - "distanthorizons.config.enum.EHeightFogMixMode.AVERAGE": + "distanthorizons.config.enum.EDhApiHeightFogMixMode.AVERAGE": "Average", - "distanthorizons.config.enum.EHeightFogMode.ABOVE_CAMERA": + "distanthorizons.config.enum.EDhApiHeightFogMode.ABOVE_CAMERA": "Above Camera", - "distanthorizons.config.enum.EHeightFogMode.BELOW_CAMERA": + "distanthorizons.config.enum.EDhApiHeightFogMode.BELOW_CAMERA": "Below Camera", - "distanthorizons.config.enum.EHeightFogMode.ABOVE_AND_BELOW_CAMERA": + "distanthorizons.config.enum.EDhApiHeightFogMode.ABOVE_AND_BELOW_CAMERA": "Above And Below Camera", - "distanthorizons.config.enum.EHeightFogMode.ABOVE_SET_HEIGHT": + "distanthorizons.config.enum.EDhApiHeightFogMode.ABOVE_SET_HEIGHT": "Above Set Height", - "distanthorizons.config.enum.EHeightFogMode.BELOW_SET_HEIGHT": + "distanthorizons.config.enum.EDhApiHeightFogMode.BELOW_SET_HEIGHT": "Below Set Height", - "distanthorizons.config.enum.EHeightFogMode.ABOVE_AND_BELOW_SET_HEIGHT": + "distanthorizons.config.enum.EDhApiHeightFogMode.ABOVE_AND_BELOW_SET_HEIGHT": "Above And Below Set Height", - "distanthorizons.config.enum.EVanillaOverdraw.NEVER": + "distanthorizons.config.enum.EDhApiVanillaOverdraw.NEVER": "Never", - "distanthorizons.config.enum.EVanillaOverdraw.DYNAMIC": + "distanthorizons.config.enum.EDhApiVanillaOverdraw.DYNAMIC": "Dynamic", - "distanthorizons.config.enum.EVanillaOverdraw.ALWAYS": + "distanthorizons.config.enum.EDhApiVanillaOverdraw.ALWAYS": "Always", "distanthorizons.config.enum.EDhApiDistantGeneratorMode.NONE": @@ -783,135 +783,135 @@ "distanthorizons.config.enum.EDhApiWorldCompressionMode.VISUALLY_EQUAL": "2. Visually Equal", - "distanthorizons.config.enum.ELightGenerationMode.DISTANT_HORIZONS": + "distanthorizons.config.enum.EDhApiLightGenerationMode.DISTANT_HORIZONS": "Distant Horizons", - "distanthorizons.config.enum.ELightGenerationMode.MINECRAFT": + "distanthorizons.config.enum.EDhApiLightGenerationMode.MINECRAFT": "Minecraft", - "distanthorizons.config.enum.EGenerationPriority.AUTO": + "distanthorizons.config.enum.EDhApiGenerationPriority.AUTO": "Auto", - "distanthorizons.config.enum.EGenerationPriority.NEAR_FIRST": + "distanthorizons.config.enum.EDhApiGenerationPriority.NEAR_FIRST": "Near first", - "distanthorizons.config.enum.EGenerationPriority.BALANCED": + "distanthorizons.config.enum.EDhApiGenerationPriority.BALANCED": "Balanced", - "distanthorizons.config.enum.EGenerationPriority.FAR_FIRST": + "distanthorizons.config.enum.EDhApiGenerationPriority.FAR_FIRST": "Far first", - "distanthorizons.config.enum.EBlocksToAvoid.NONE": + "distanthorizons.config.enum.EDhApiBlocksToAvoid.NONE": "None", - "distanthorizons.config.enum.EBlocksToAvoid.NON_COLLIDING": + "distanthorizons.config.enum.EDhApiBlocksToAvoid.NON_COLLIDING": "Non-Colliding", - "distanthorizons.config.enum.EOverdrawPrevention.NONE": + "distanthorizons.config.enum.EDhApiOverdrawPrevention.NONE": "None", - "distanthorizons.config.enum.EOverdrawPrevention.LIGHT": + "distanthorizons.config.enum.EDhApiOverdrawPrevention.LIGHT": "Light", - "distanthorizons.config.enum.EOverdrawPrevention.MEDIUM": + "distanthorizons.config.enum.EDhApiOverdrawPrevention.MEDIUM": "Medium", - "distanthorizons.config.enum.EOverdrawPrevention.HEAVY": + "distanthorizons.config.enum.EDhApiOverdrawPrevention.HEAVY": "Heavy", - "distanthorizons.config.enum.EServerFolderNameMode.NAME_ONLY": + "distanthorizons.config.enum.EDhApiServerFolderNameMode.NAME_ONLY": "Name Only", - "distanthorizons.config.enum.EServerFolderNameMode.IP_ONLY": + "distanthorizons.config.enum.EDhApiServerFolderNameMode.IP_ONLY": "IP Only", - "distanthorizons.config.enum.EServerFolderNameMode.NAME_IP": + "distanthorizons.config.enum.EDhApiServerFolderNameMode.NAME_IP": "Name and IP", - "distanthorizons.config.enum.EServerFolderNameMode.NAME_IP_PORT": + "distanthorizons.config.enum.EDhApiServerFolderNameMode.NAME_IP_PORT": "Name, IP, Port", - "distanthorizons.config.enum.EServerFolderNameMode.NAME_IP_PORT_MC_VERSION": + "distanthorizons.config.enum.EDhApiServerFolderNameMode.NAME_IP_PORT_MC_VERSION": "Name, IP, Port, MC version", - "distanthorizons.config.enum.ERendererMode.DEFAULT": + "distanthorizons.config.enum.EDhApiRendererMode.DEFAULT": "Default", - "distanthorizons.config.enum.ERendererMode.DEBUG": + "distanthorizons.config.enum.EDhApiRendererMode.DEBUG": "Debug", - "distanthorizons.config.enum.ERendererMode.DISABLED": + "distanthorizons.config.enum.EDhApiRendererMode.DISABLED": "Disabled", - "distanthorizons.config.enum.EDebugRendering.OFF": + "distanthorizons.config.enum.EDhApiDebugRendering.OFF": "Off", - "distanthorizons.config.enum.EDebugRendering.SHOW_DETAIL": + "distanthorizons.config.enum.EDhApiDebugRendering.SHOW_DETAIL": "Show detail", - "distanthorizons.config.enum.EDebugRendering.SHOW_GENMODE": + "distanthorizons.config.enum.EDhApiDebugRendering.SHOW_GENMODE": "Show generation mode", - "distanthorizons.config.enum.EDebugRendering.SHOW_BLOCK_MATERIAL": + "distanthorizons.config.enum.EDhApiDebugRendering.SHOW_BLOCK_MATERIAL": "Show Material", - "distanthorizons.config.enum.EDebugRendering.SHOW_OVERLAPPING_QUADS": + "distanthorizons.config.enum.EDhApiDebugRendering.SHOW_OVERLAPPING_QUADS": "Show overlapping quads", - "distanthorizons.config.enum.EDebugRendering.SHOW_RENDER_SOURCE_FLAG": + "distanthorizons.config.enum.EDhApiDebugRendering.SHOW_RENDER_SOURCE_FLAG": "Show render source flag", - "distanthorizons.config.enum.EGLErrorHandlingMode.IGNORE": + "distanthorizons.config.enum.EDhApiGLErrorHandlingMode.IGNORE": "Ignore", - "distanthorizons.config.enum.EGLErrorHandlingMode.LOG": + "distanthorizons.config.enum.EDhApiGLErrorHandlingMode.LOG": "Log", - "distanthorizons.config.enum.EGLErrorHandlingMode.LOG_THROW": + "distanthorizons.config.enum.EDhApiGLErrorHandlingMode.LOG_THROW": "Log-Throw", - "distanthorizons.config.enum.EGlProfileMode.CORE": + "distanthorizons.config.enum.EDhApiGlProfileMode.CORE": "Core", - "distanthorizons.config.enum.EGlProfileMode.COMPAT": + "distanthorizons.config.enum.EDhApiGlProfileMode.COMPAT": "Compat", - "distanthorizons.config.enum.EGlProfileMode.ANY": + "distanthorizons.config.enum.EDhApiGlProfileMode.ANY": "Any", - "distanthorizons.config.enum.ELoggerMode.DISABLED": + "distanthorizons.config.enum.EDhApiLoggerMode.DISABLED": "Disabled", - "distanthorizons.config.enum.ELoggerMode.LOG_ALL_TO_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_ALL_TO_FILE": "File: All, Chat: Off", - "distanthorizons.config.enum.ELoggerMode.LOG_ERROR_TO_CHAT": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_ERROR_TO_CHAT": "File: All, Chat: Error", - "distanthorizons.config.enum.ELoggerMode.LOG_WARNING_TO_CHAT": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_WARNING_TO_CHAT": "File: All, Chat: Warning", - "distanthorizons.config.enum.ELoggerMode.LOG_INFO_TO_CHAT": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_INFO_TO_CHAT": "File: All, Chat: Info", - "distanthorizons.config.enum.ELoggerMode.LOG_DEBUG_TO_CHAT": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_DEBUG_TO_CHAT": "File: All, Chat: Debug", - "distanthorizons.config.enum.ELoggerMode.LOG_ALL_TO_CHAT": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_ALL_TO_CHAT": "File: All, Chat: All", - "distanthorizons.config.enum.ELoggerMode.LOG_ERROR_TO_CHAT_AND_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_ERROR_TO_CHAT_AND_FILE": "File: Error, Chat: Error", - "distanthorizons.config.enum.ELoggerMode.LOG_WARNING_TO_CHAT_AND_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_FILE": "File: Warning, Chat: Warning", - "distanthorizons.config.enum.ELoggerMode.LOG_INFO_TO_CHAT_AND_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_INFO_TO_CHAT_AND_FILE": "File: Info, Chat: Info", - "distanthorizons.config.enum.ELoggerMode.LOG_DEBUG_TO_CHAT_AND_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_DEBUG_TO_CHAT_AND_FILE": "File: Debug, Chat: Debug", - "distanthorizons.config.enum.ELoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_WARNING_TO_CHAT_AND_INFO_TO_FILE": "File: Info, Chat: Warning", - "distanthorizons.config.enum.ELoggerMode.LOG_ERROR_TO_CHAT_AND_INFO_TO_FILE": + "distanthorizons.config.enum.EDhApiLoggerMode.LOG_ERROR_TO_CHAT_AND_INFO_TO_FILE": "File: Info, Chat: Error", - "distanthorizons.config.enum.EGpuUploadMethod.AUTO": + "distanthorizons.config.enum.EDhApiGpuUploadMethod.AUTO": "Auto", - "distanthorizons.config.enum.EGpuUploadMethod.BUFFER_STORAGE": + "distanthorizons.config.enum.EDhApiGpuUploadMethod.BUFFER_STORAGE": "Buffer storage", - "distanthorizons.config.enum.EGpuUploadMethod.SUB_DATA": + "distanthorizons.config.enum.EDhApiGpuUploadMethod.SUB_DATA": "Sub data", - "distanthorizons.config.enum.EGpuUploadMethod.BUFFER_MAPPING": + "distanthorizons.config.enum.EDhApiGpuUploadMethod.BUFFER_MAPPING": "Buffer mapping", - "distanthorizons.config.enum.EGpuUploadMethod.DATA": + "distanthorizons.config.enum.EDhApiGpuUploadMethod.DATA": "Data", - "distanthorizons.config.enum.EBufferRebuildTimes.CONSTANT": + "distanthorizons.config.enum.EDhApiBufferRebuildTimes.CONSTANT": "Constant", - "distanthorizons.config.enum.EBufferRebuildTimes.FREQUENT": + "distanthorizons.config.enum.EDhApiBufferRebuildTimes.FREQUENT": "Frequent", - "distanthorizons.config.enum.EBufferRebuildTimes.NORMAL": + "distanthorizons.config.enum.EDhApiBufferRebuildTimes.NORMAL": "Normal", - "distanthorizons.config.enum.EBufferRebuildTimes.RARE": + "distanthorizons.config.enum.EDhApiBufferRebuildTimes.RARE": "Rare", - "distanthorizons.config.enum.ELodShading.AUTO": + "distanthorizons.config.enum.EDhApiLodShading.AUTO": "Auto", - "distanthorizons.config.enum.ELodShading.ENABLED": + "distanthorizons.config.enum.EDhApiLodShading.ENABLED": "Enabled", - "distanthorizons.config.enum.ELodShading.DISABLED": + "distanthorizons.config.enum.EDhApiLodShading.DISABLED": "Disabled", - "distanthorizons.config.enum.EUpdateBranch.STABLE": + "distanthorizons.config.enum.EDhApiUpdateBranch.STABLE": "Stable", - "distanthorizons.config.enum.EUpdateBranch.NIGHTLY": + "distanthorizons.config.enum.EDhApiUpdateBranch.NIGHTLY": "Nightly" } From a751507f19dd5a806562bf6212be40a19b6ac0ab Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 12:37:18 -0500 Subject: [PATCH 132/183] Have grass fade to dirt for walls --- .../render/bufferBuilding/ColumnBox.java | 32 +++++++++++++------ .../ColumnRenderBufferBuilder.java | 2 +- .../bufferBuilding/CubicLodTemplate.java | 3 ++ .../render/bufferBuilding/LodQuadBuilder.java | 27 ++++++++++++++-- .../distanthorizons/core/pos/DhBlockPos.java | 6 ++++ .../distanthorizons/core/util/ColorUtil.java | 1 + .../block/IBlockStateWrapper.java | 2 +- .../world/IClientLevelWrapper.java | 3 ++ 8 files changed, 63 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java index d7535a524..8d52addd6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java @@ -22,7 +22,9 @@ package com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.util.ColorUtil; +import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; +import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.dataObjects.render.columnViews.ColumnArrayView; import com.seibel.distanthorizons.core.render.renderer.LodRenderer; @@ -59,6 +61,12 @@ public class ColumnBox color = ColorUtil.setAlpha(color, 255); } + // try to prevent displaying grass gradients for dirt blocks underneath solid blocks + if (!isTopTransparent && irisBlockMaterialId == IBlockStateWrapper.IrisBlockMaterial.GRASS) + { + irisBlockMaterialId = IBlockStateWrapper.IrisBlockMaterial.DIRT; + } + // cave culling prevention // prevents certain faces from being culled underground that should be allowed @@ -124,7 +132,7 @@ public class ColumnBox // add an adjacent face if this is opaque face or transparent over the void if (!isTransparent || overVoid) { - builder.addQuadAdj(EDhDirection.NORTH, x, minY, z, xSize, ySize, color, irisBlockMaterialId, (byte) 15, blockLight); + builder.addQuadAdj(EDhDirection.NORTH, x, minY, z, xSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } } else if (adjDataNorth.length == 1) @@ -151,7 +159,7 @@ public class ColumnBox if (adjDataSouth == null) { if (!isTransparent || overVoid) - builder.addQuadAdj(EDhDirection.SOUTH, x, minY, maxZ, xSize, ySize, color, irisBlockMaterialId, (byte) 15, blockLight); + builder.addQuadAdj(EDhDirection.SOUTH, x, minY, maxZ, xSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } else if (adjDataSouth.length == 1) { @@ -178,7 +186,7 @@ public class ColumnBox if (adjDataWest == null) { if (!isTransparent || overVoid) - builder.addQuadAdj(EDhDirection.WEST, x, minY, z, zSize, ySize, color, irisBlockMaterialId, (byte) 15, blockLight); + builder.addQuadAdj(EDhDirection.WEST, x, minY, z, zSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } else if (adjDataWest.length == 1) { @@ -204,7 +212,7 @@ public class ColumnBox if (adjData[EDhDirection.EAST.ordinal() - 2] == null) { if (!isTransparent || overVoid) - builder.addQuadAdj(EDhDirection.EAST, maxX, minY, z, zSize, ySize, color, irisBlockMaterialId, (byte) 15, blockLight); + builder.addQuadAdj(EDhDirection.EAST, maxX, minY, z, zSize, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); } else if (adjDataEast.length == 1) { @@ -236,7 +244,7 @@ public class ColumnBox if (adjColumnView == null || adjColumnView.size == 0 || RenderDataPointUtil.isVoid(adjColumnView.get(0))) { // there isn't any data adjacent to this LOD, add the vertical quad - builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, ySize, color, irisBlockMaterialId, (byte) 15, blockLight); + builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, ySize, color, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, blockLight); return; } @@ -393,7 +401,7 @@ public class ColumnBox // The input face is completely inside the adj's face, don't render it if (debugOverlapColor != 0) { - builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, ySize, debugOverlapColor, irisBlockMaterialId, (byte) 15, (byte) 15); + builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, ySize, debugOverlapColor, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT); } } else @@ -402,7 +410,7 @@ public class ColumnBox if (adjYMax > yMin && debugOverlapColor != 0) { - builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, (short) (adjYMax - yMin), debugOverlapColor, irisBlockMaterialId, (byte) 15, (byte) 15); + builder.addQuadAdj(direction, x, yMin, z, horizontalWidth, (short) (adjYMax - yMin), debugOverlapColor, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT); } // if this is the only face, use the yMax and break, @@ -448,7 +456,7 @@ public class ColumnBox // the adj data intersects the higher part of the current data if (debugOverlapColor != 0) { - builder.addQuadAdj(direction, x, adjYMin, z, horizontalWidth, (short) (yMax - adjYMin), debugOverlapColor, irisBlockMaterialId, (byte) 15, (byte) 15); + builder.addQuadAdj(direction, x, adjYMin, z, horizontalWidth, (short) (yMax - adjYMin), debugOverlapColor, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT); } // we start the creation of a new face @@ -459,7 +467,7 @@ public class ColumnBox // _______&&: y < depth ______ < yMax if (debugOverlapColor != 0) { - builder.addQuadAdj(direction, x, adjYMin, z, horizontalWidth, (short) (adjYMax - adjYMin), debugOverlapColor, irisBlockMaterialId, (byte) 15, (byte) 15); + builder.addQuadAdj(direction, x, adjYMin, z, horizontalWidth, (short) (adjYMax - adjYMin), debugOverlapColor, irisBlockMaterialId, LodUtil.MAX_MC_LIGHT, LodUtil.MAX_MC_LIGHT); } if (firstFace) @@ -474,6 +482,12 @@ public class ColumnBox throw new RuntimeException("Loop error"); if (previousAdjDepth > adjYMax) { + if (irisBlockMaterialId == IBlockStateWrapper.IrisBlockMaterial.GRASS) + { + // this LOD is underneath another, grass will never show here + irisBlockMaterialId = IBlockStateWrapper.IrisBlockMaterial.DIRT; + } + builder.addQuadAdj(direction, x, adjYMax, z, horizontalWidth, (short) (previousAdjDepth - adjYMax), color, irisBlockMaterialId, RenderDataPointUtil.getLightSky(adjPoint), blockLight); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 6598c2e06..1cef462cb 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -102,7 +102,7 @@ public class ColumnRenderBufferBuilder long builderStartTime = System.currentTimeMillis(); - LodQuadBuilder builder = new LodQuadBuilder(enableSkyLightCulling, (short) (skyLightCullingBelow - clientLevel.getMinY()), enableTransparency); + LodQuadBuilder builder = new LodQuadBuilder(enableSkyLightCulling, (short) (skyLightCullingBelow - clientLevel.getMinY()), enableTransparency, clientLevel.getClientLevelWrapper()); makeLodRenderData(builder, renderSource, adjData); long builderEndTime = System.currentTimeMillis(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java index ff0ecfd42..9bb9c2076 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java @@ -133,6 +133,9 @@ public class CubicLodTemplate case IBlockStateWrapper.IrisBlockMaterial.WATER: color = ColorUtil.BLUE; break; + case IBlockStateWrapper.IrisBlockMaterial.GRASS: + color = ColorUtil.LIGHT_GREEN; + break; case IBlockStateWrapper.IrisBlockMaterial.ILLUMINATED: color = ColorUtil.YELLOW; break; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index 87f16a9c7..dabcea970 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -23,11 +23,15 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.*; +import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.util.ColorUtil; import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod; +import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; @@ -41,6 +45,7 @@ import org.apache.logging.log4j.Logger; public class LodQuadBuilder { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); public final boolean skipQuadsWithZeroSkylight; public final short skyLightCullingBelow; @@ -51,6 +56,7 @@ public class LodQuadBuilder private final ArrayList[] transparentQuads = (ArrayList[]) new ArrayList[6]; private final boolean doTransparency; + private final IClientLevelWrapper clientLevelWrapper; @@ -111,7 +117,7 @@ public class LodQuadBuilder // constructor // //=============// - public LodQuadBuilder(boolean enableSkylightCulling, short skyLightCullingBelow, boolean doTransparency) + public LodQuadBuilder(boolean enableSkylightCulling, short skyLightCullingBelow, boolean doTransparency, IClientLevelWrapper clientLevelWrapper) { this.doTransparency = doTransparency; for (int i = 0; i < 6; i++) @@ -122,6 +128,7 @@ public class LodQuadBuilder this.skipQuadsWithZeroSkylight = enableSkylightCulling; this.skyLightCullingBelow = skyLightCullingBelow; + this.clientLevelWrapper = clientLevelWrapper; } @@ -253,8 +260,24 @@ public class LodQuadBuilder default: throw new IllegalArgumentException("Invalid Axis enum: " + axis); } + + + int color = quad.color; + if (quad.irisBlockMaterialId == IBlockStateWrapper.IrisBlockMaterial.GRASS + && + ( + (quad.direction.getAxis().isHorizontal() && quadBase[i][1] == 0) // TODO what does "quadBase[i][1] == 0" mean? + || quad.direction == EDhDirection.DOWN) + ) + { + // for horizontal and bottom faces of grass blocks, use the dirt color to + // prevent green cliff walls + color = this.clientLevelWrapper.getDirtBlockColor(); + color = ColorUtil.applyShade(color, MC.getShade(quad.direction)); + } + this.putVertex(bb, (short) (quad.x + dx), (short) (quad.y + dy), (short) (quad.z + dz), - quad.hasError ? ColorUtil.RED : quad.color, + quad.hasError ? ColorUtil.RED : color, quad.hasError ? 0 : normalIndex, quad.hasError ? 0 : quad.irisBlockMaterialId, quad.hasError ? 15 : quad.skyLight, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhBlockPos.java b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhBlockPos.java index 7fe2bad0a..ecb4e3010 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/pos/DhBlockPos.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/pos/DhBlockPos.java @@ -45,10 +45,16 @@ public class DhBlockPos public static final int PACKED_Z_OFFSET = PACKED_Y_LENGTH; public static final int PACKED_X_OFFSET = PACKED_Y_LENGTH + PACKED_Z_LENGTH; + /** Useful for methods that need a position passed in but won't actually be used */ + public static final DhBlockPos ZERO = new DhBlockPos(0, 0, 0); + + public int x; public int y; public int z; + + public DhBlockPos(int x, int y, int z) { this.x = x; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java index 89ec90fa9..c0ffb7554 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java @@ -41,6 +41,7 @@ public class ColorUtil public static final int RED = rgbToInt(255, 0, 0); public static final int DARK_RED = rgbToInt(100, 0, 0); public static final int GREEN = rgbToInt(0, 255, 0); + public static final int LIGHT_GREEN = rgbToInt(80, 255, 80); public static final int BLUE = rgbToInt(0, 0, 255); public static final int YELLOW = rgbToInt(255, 255, 0); public static final int CYAN = rgbToInt(0, 255, 255); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java index f6f7f1dce..b26481a87 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/block/IBlockStateWrapper.java @@ -47,7 +47,7 @@ public interface IBlockStateWrapper extends IDhApiBlockStateWrapper public static final byte TERRACOTTA = 10; public static final byte NETHER_STONE = 11; public static final byte WATER = 12; - // unlisted numbers are unused + public static final byte GRASS = 13; /** shouldn't normally be needed, but just in case */ public static final byte AIR = 14; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java index 8ef1e3a01..60d74d4bf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/wrapperInterfaces/world/IClientLevelWrapper.java @@ -34,4 +34,7 @@ public interface IClientLevelWrapper extends ILevelWrapper int computeBaseColor(DhBlockPos pos, IBiomeWrapper biome, IBlockStateWrapper blockState); + /** @return -1 if there was a problem getting the color */ + int getDirtBlockColor(); + } From 10d9282df77f649a8e563d007e1197b6193c8efd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 12:37:31 -0500 Subject: [PATCH 133/183] Fix a couple FullDataPointUtilV2 references --- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 2 +- .../seibel/distanthorizons/core/util/RenderDataPointUtil.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 9ebb4d892..8e20be290 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -44,7 +44,7 @@ import java.util.Arrays; /** * This data source contains every datapoint over its given {@link DhSectionPos}.

* - * @see FullDataPointUtilV2 + * @see FullDataPointUtil * @see FullDataSourceV1 */ public class FullDataSourceV2 implements IDataSource diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java index 19eb93460..72bb1864f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderDataPointUtil.java @@ -30,7 +30,7 @@ import com.seibel.distanthorizons.coreapi.ModInfo; * A helper class that is used to access the data from a long * formatted as a render data point.

* - * To access data from a long formatted as a full data point see: {@link FullDataPointUtilV2} + * To access data from a long formatted as a full data point see: {@link FullDataPointUtil} * * DataPoint Format:
* @@ -56,7 +56,7 @@ import com.seibel.distanthorizons.coreapi.ModInfo; * BL BL BL BL SL SL SL SL |
* * - * @see FullDataPointUtilV2 + * @see FullDataPointUtil */ public class RenderDataPointUtil { From f310f1b316a8810fe3a3d4a907731c083e13183a Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 18:03:54 -0500 Subject: [PATCH 134/183] Remove Unlimited horizontal quality setting The setting was un-maintainable and would cause major issues for anything but the smallest render distances. --- .../api/enums/config/EDhApiHorizontalQuality.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java index 5ac005504..9bb431172 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java @@ -45,11 +45,7 @@ public enum EDhApiHorizontalQuality MEDIUM(2.0f, 12), HIGH(2.2f, 24), EXTREME(2.4f, 64), - - /** @deprecated this setting is unmaintainable at high render distances. */ - @Deprecated - @DisallowSelectingViaConfigGui - UNLIMITED(-1, -1); + ; From 8846ca5b5e6505ad8e17111006134f37a5937473 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 18:18:41 -0500 Subject: [PATCH 135/183] Improve shutdown thread pool handling --- .../core/api/internal/ServerApi.java | 5 - .../core/api/internal/SharedApi.java | 109 +++++++++--------- .../ColumnRenderBufferBuilder.java | 31 +++-- .../transformers/ChunkToLodBuilder.java | 22 ++-- .../file/AbstractNewDataSourceHandler.java | 11 +- .../FullDataSourceProviderV1.java | 11 +- .../FullDataSourceProviderV2.java | 1 + 7 files changed, 112 insertions(+), 78 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java index d85dd8639..0d1e97862 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ServerApi.java @@ -38,11 +38,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.ILevelWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapper; import org.apache.logging.log4j.Logger; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.ThreadPoolExecutor; - /** * This holds the methods that should be called by the host mod loader (Fabric, * Forge, etc.). Specifically server events. diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java index f04bd98c1..c8ffe6de8 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/SharedApi.java @@ -46,6 +46,7 @@ import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; /** Contains code and variables used by both {@link ClientApi} and {@link ServerApi} */ @@ -291,71 +292,75 @@ public class SharedApi return; } - executor.execute(() -> + try { - //LOGGER.trace(chunkWrapper.getChunkPos() + " " + executor.getActiveCount() + " / " + executor.getQueue().size() + " - " + executor.getCompletedTaskCount()); - - try + executor.execute(() -> { - // Save or populate the chunk wrapper's lighting - // this is done so we don't have to worry about MC unloading the lighting data for this chunk - boolean onlyUseDhLighting = Config.Client.Advanced.LodBuilding.onlyUseDhLightingEngine.get(); - if (!onlyUseDhLighting && chunkWrapper.isLightCorrect()) + //LOGGER.trace(chunkWrapper.getChunkPos() + " " + executor.getActiveCount() + " / " + executor.getQueue().size() + " - " + executor.getCompletedTaskCount()); + + try { - try + // Save or populate the chunk wrapper's lighting + // this is done so we don't have to worry about MC unloading the lighting data for this chunk + boolean onlyUseDhLighting = Config.Client.Advanced.LodBuilding.onlyUseDhLightingEngine.get(); + if (!onlyUseDhLighting && chunkWrapper.isLightCorrect()) { - // If MC's lighting engine isn't thread safe this may cause the server thread to lag - chunkWrapper.bakeDhLightingUsingMcLightingEngine(); - } - catch (IllegalStateException e) - { - LOGGER.warn("Chunk light baking error: " + e.getMessage(), e); - } - } - else - { - // generate the chunk's lighting, using neighboring chunks if present - - ArrayList nearbyChunkList; - if (neighbourChunkList != null) - { - nearbyChunkList = neighbourChunkList; + try + { + // If MC's lighting engine isn't thread safe this may cause the server thread to lag + chunkWrapper.bakeDhLightingUsingMcLightingEngine(); + } + catch (IllegalStateException e) + { + LOGGER.warn("Chunk light baking error: " + e.getMessage(), e); + } } else { - nearbyChunkList = new ArrayList<>(1); - nearbyChunkList.add(chunkWrapper); + // generate the chunk's lighting, using neighboring chunks if present + + ArrayList nearbyChunkList; + if (neighbourChunkList != null) + { + nearbyChunkList = neighbourChunkList; + } + else + { + nearbyChunkList = new ArrayList<>(1); + nearbyChunkList.add(chunkWrapper); + } + + DhLightingEngine.INSTANCE.lightChunk(chunkWrapper, nearbyChunkList, dhLevel.hasSkyLight() ? 15 : 0); } - DhLightingEngine.INSTANCE.lightChunk(chunkWrapper, nearbyChunkList, dhLevel.hasSkyLight() ? 15 : 0); + dhLevel.updateChunkAsync(chunkWrapper); } - - dhLevel.updateChunkAsync(chunkWrapper); - } - catch (Exception e) - { - LOGGER.error("Unexpected error when updating chunk at pos: ["+chunkWrapper.getChunkPos()+"]", e); - } - finally - { - // the LOD chunk has finished being updated - int updateTimeoutInSec = Config.Client.Advanced.LodBuilding.minTimeBetweenChunkUpdatesInSeconds.get(); - if (updateTimeoutInSec != 0) + catch (Exception e) { - // prevent updating this chunk again until the timeout finishes - CHUNK_UPDATE_TIMER.schedule(new TimerTask() + LOGGER.error("Unexpected error when updating chunk at pos: [" + chunkWrapper.getChunkPos() + "]", e); + } + finally + { + // the LOD chunk has finished being updated + int updateTimeoutInSec = Config.Client.Advanced.LodBuilding.minTimeBetweenChunkUpdatesInSeconds.get(); + if (updateTimeoutInSec != 0) { - @Override - public void run() { UPDATING_CHUNK_POS_SET.remove(chunkWrapper.getChunkPos()); } - }, updateTimeoutInSec * 1000L); + // prevent updating this chunk again until the timeout finishes + CHUNK_UPDATE_TIMER.schedule(new TimerTask() + { + @Override + public void run() { UPDATING_CHUNK_POS_SET.remove(chunkWrapper.getChunkPos()); } + }, updateTimeoutInSec * 1000L); + } + else + { + // instantly allow this chunk to be updated again + UPDATING_CHUNK_POS_SET.remove(chunkWrapper.getChunkPos()); + } } - else - { - // instantly allow this chunk to be updated again - UPDATING_CHUNK_POS_SET.remove(chunkWrapper.getChunkPos()); - } - } - }); + }); + } + catch (RejectedExecutionException ignore) { /* the executor was shut down, it should be back up shortly and able to accept new jobs */ } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 1cef462cb..7ff76dfdc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -38,6 +38,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; /** @@ -77,8 +78,9 @@ public class ColumnRenderBufferBuilder return future; } - //LOGGER.info("RenderRegion startBuild @ "+renderSource.sectionPos); - return CompletableFuture.supplyAsync(() -> + try + { + return CompletableFuture.supplyAsync(() -> { try { @@ -87,13 +89,13 @@ public class ColumnRenderBufferBuilder //EVENT_LOGGER.trace("RenderRegion start QuadBuild @ " + renderSource.sectionPos); boolean enableSkyLightCulling = ( - // dimensions with a ceiling will be all caves so we don't want cave culling - !clientLevel.getLevelWrapper().hasCeiling() - // the end has a lot of overhangs with 0 lighting above the void, which look broken with - // the current cave culling logic (this could probably be improved, but just skipping it works best for now) - && !clientLevel.getLevelWrapper().getDimensionType().isTheEnd() - ) - && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); + // dimensions with a ceiling will be all caves so we don't want cave culling + !clientLevel.getLevelWrapper().hasCeiling() + // the end has a lot of overhangs with 0 lighting above the void, which look broken with + // the current cave culling logic (this could probably be improved, but just skipping it works best for now) + && !clientLevel.getLevelWrapper().getDimensionType().isTheEnd() + ) + && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); int skyLightCullingBelow = Config.Client.Advanced.Graphics.AdvancedGraphics.caveCullingHeight.get(); // FIXME: Clamp also to the max world height. @@ -144,10 +146,19 @@ public class ColumnRenderBufferBuilder } catch (Throwable e3) { - LOGGER.error("LodNodeBufferBuilder was unable to upload buffer: "+e3.getMessage(), e3); + LOGGER.error("LodNodeBufferBuilder was unable to upload buffer: " + e3.getMessage(), e3); throw e3; } }, bufferUploaderExecutor); + } + catch (RejectedExecutionException ignore) + { + // the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back + + CompletableFuture future = new CompletableFuture<>(); + future.cancel(true); + return future; + } } private static void makeLodRenderData(LodQuadBuilder quadBuilder, ColumnRenderSource renderSource, ColumnRenderSource[] adjRegions) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java index 663a6f033..7336b42bd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/transformers/ChunkToLodBuilder.java @@ -112,17 +112,21 @@ public class ChunkToLodBuilder implements AutoCloseable for (int i = 0; i < threadCount; i++) { this.runningCount.incrementAndGet(); - CompletableFuture.runAsync(() -> + try { - try + CompletableFuture.runAsync(() -> { - this.tickThreadTask(); - } - finally - { - this.runningCount.decrementAndGet(); - } - }, lodBuilderExecutor); + try + { + this.tickThreadTask(); + } + finally + { + this.runningCount.decrementAndGet(); + } + }, lodBuilderExecutor); + } + catch (RejectedExecutionException ignore) { /* the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back */ } } } private void tickThreadTask() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index bf0ace1b9..0f5244621 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -122,7 +122,16 @@ public abstract class AbstractNewDataSourceHandler return CompletableFuture.completedFuture(null); } - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + + try + { + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + catch (RejectedExecutionException ignore) + { + // the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back + return CompletableFuture.completedFuture(null); + } } /** * Should only be used in internal file handler methods where we are already running on a file handler thread. diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index d0163d600..a994b60e4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.locks.ReentrantLock; @@ -98,7 +99,15 @@ public class FullDataSourceProviderV1 return CompletableFuture.completedFuture(null); } - return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + try + { + return CompletableFuture.supplyAsync(() -> this.get(pos), executor); + } + catch (RejectedExecutionException ignore) + { + // the thread pool was probably shut down because it's size is being changed, just wait a sec and it should be back + return CompletableFuture.completedFuture(null); + } } /** * Should only be used in internal file handler methods where we are already running on a file handler thread. diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 876cd3745..506cf5016 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -278,6 +278,7 @@ public class FullDataSourceProviderV2 } }); } + catch (RejectedExecutionException ignore) { /* the executor was shut down, it should be back up shortly and able to accept new jobs */ } catch (Exception e) { this.parentUpdatingPosSet.remove(parentUpdatePos); From 2d323cef393442f8bf99ab7313210e616f69913b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 6 Apr 2024 18:18:57 -0500 Subject: [PATCH 136/183] remove missed unlimited horizontal quality reference --- .../seibel/distanthorizons/core/render/LodQuadTree.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 203ceee3d..245919657 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -368,13 +368,6 @@ public class LodQuadTree extends QuadTree implements AutoClose public byte calculateExpectedDetailLevel(DhBlockPos2D playerPos, DhSectionPos sectionPos) { return this.getDetailLevelFromDistance(playerPos.dist(sectionPos.getCenterBlockPosX(), sectionPos.getCenterBlockPosZ())); } private byte getDetailLevelFromDistance(double distance) { - // special case, never drop the quality - if (Config.Client.Advanced.Graphics.Quality.horizontalQuality.get() == EDhApiHorizontalQuality.UNLIMITED) - { - return this.maxRenderDetailLevel; - } - - double maxDetailDistance = this.getDrawDistanceFromDetail(Byte.MAX_VALUE - 1); if (distance > maxDetailDistance) { From 46e3b98b624e5d49a6053473f7db74fc96186c67 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 7 Apr 2024 13:50:05 -0500 Subject: [PATCH 137/183] Remove near fog option --- .../enums/rendering/EDhApiFogDistance.java | 41 ------ .../config/client/IDhApiFogConfig.java | 4 - .../methods/config/client/DhApiFogConfig.java | 5 - .../distanthorizons/core/config/Config.java | 6 - .../core/render/fog/LodFogConfig.java | 120 ++++++------------ 5 files changed, 37 insertions(+), 139 deletions(-) delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java deleted file mode 100644 index d7b585802..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/rendering/EDhApiFogDistance.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.api.enums.rendering; - -/** - * NEAR,
- * FAR,
- * NEAR_AND_FAR
- * - * @author James Seibel - * @version 2022-6-2 - * @since API 1.1.0 - */ -@Deprecated -public enum EDhApiFogDistance -{ - // Reminder: - // when adding items up the API minor version - // when removing items up the API major version - - NEAR, - FAR, - NEAR_AND_FAR -} \ No newline at end of file diff --git a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java index e976ef8ad..3303615b1 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiFogConfig.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.api.interfaces.config.client; import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogColorMode; -import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDistance; import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigGroup; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; @@ -57,9 +56,6 @@ public interface IDhApiFogConfig extends IDhApiConfigGroup // basic fog settings // //====================// - /** Defines at what distance fog is rendered on fake chunks. */ - IDhApiConfigValue distance(); - /** Should be used to enable/disable fog rendering. */ IDhApiConfigValue drawMode(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java index 61a2d3d92..cc89c53de 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiFogConfig.java @@ -20,7 +20,6 @@ package com.seibel.distanthorizons.core.api.external.methods.config.client; import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogColorMode; -import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDistance; import com.seibel.distanthorizons.api.enums.rendering.EDhApiFogDrawMode; import com.seibel.distanthorizons.api.interfaces.config.IDhApiConfigValue; import com.seibel.distanthorizons.api.interfaces.config.client.IDhApiFarFogConfig; @@ -52,10 +51,6 @@ public class DhApiFogConfig implements IDhApiFogConfig // basic fog settings // //====================// - @Override - public IDhApiConfigValue distance() - { return new DhApiConfigValue<>(Config.Client.Advanced.Graphics.Fog.distance); } - @Override public IDhApiConfigValue drawMode() { return new DhApiConfigValue<>(Config.Client.Advanced.Graphics.Fog.drawMode); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 2cf5dd26f..ec19ad4f0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -250,12 +250,6 @@ public class Config .setPerformance(EConfigEntryPerformance.VERY_LOW) .build(); - public static ConfigEntry distance = new ConfigEntry.Builder() - .set(EDhApiFogDistance.FAR) - .comment("At what distance should Fog be drawn on the LODs?") - .setPerformance(EConfigEntryPerformance.NONE) - .build(); - public static ConfigEntry colorMode = new ConfigEntry.Builder() .set(EDhApiFogColorMode.USE_WORLD_FOG_COLOR) .comment("" diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java b/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java index 5bc86eb2e..be4adf73e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/fog/LodFogConfig.java @@ -52,8 +52,6 @@ public class LodFogConfig public final EDhApiHeightFogMode heightFogMode; public final float heightFogHeight; - final boolean drawNearFog; - // TODO: Move these out of here public final int earthCurveRatio; @@ -88,72 +86,55 @@ public class LodFogConfig if (fogDrawMode != EDhApiFogDrawMode.FOG_DISABLED) { - EDhApiFogDistance fogDistance = Config.Client.Advanced.Graphics.Fog.distance.get(); - drawNearFog = (fogDistance == EDhApiFogDistance.NEAR || fogDistance == EDhApiFogDistance.NEAR_AND_FAR); + // fog should be drawn - if (fogDistance == EDhApiFogDistance.FAR || fogDistance == EDhApiFogDistance.NEAR_AND_FAR) + farFogSetting = new FogSettings( + Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogStart.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogEnd.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogMin.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogMax.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogDensity.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogFalloff.get() + ); + + heightFogMixMode = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMixMode.get(); + if (heightFogMixMode == EDhApiHeightFogMixMode.IGNORE_HEIGHT || heightFogMixMode == EDhApiHeightFogMixMode.BASIC) { - // far fog should be drawn + // basic fog mixing - farFogSetting = new FogSettings( - Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogStart.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogEnd.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogMin.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogMax.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogDensity.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.farFogFalloff.get() + heightFogSetting = null; + heightFogMode = null; + heightFogHeight = 0.f; + } + else + { + // advanced fog mixing + + heightFogSetting = new FogSettings( + Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogDensity.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogEnd.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMin.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMax.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogDensity.get(), + Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogFalloff.get() ); - heightFogMixMode = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMixMode.get(); - if (heightFogMixMode == EDhApiHeightFogMixMode.IGNORE_HEIGHT || heightFogMixMode == EDhApiHeightFogMixMode.BASIC) + heightFogMode = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMode.get(); + + if (heightFogMode.basedOnCamera) { - // basic fog mixing - - heightFogSetting = null; - heightFogMode = null; heightFogHeight = 0.f; } else { - // advanced fog mixing - - heightFogSetting = new FogSettings( - Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogDensity.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogEnd.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMin.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMax.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogDensity.get(), - Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogFalloff.get() - ); - - heightFogMode = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogMode.get(); - - if (heightFogMode.basedOnCamera) - { - heightFogHeight = 0.f; - } - else - { - heightFogHeight = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogBaseHeight.get().floatValue(); - } + heightFogHeight = Config.Client.Advanced.Graphics.Fog.AdvancedFog.HeightFog.heightFogBaseHeight.get().floatValue(); } } - else - { - // far fog should not be drawn - - farFogSetting = null; - heightFogSetting = null; - heightFogMode = null; - heightFogMixMode = null; - heightFogHeight = 0.f; - } } else { // fog disabled - drawNearFog = false; farFogSetting = null; heightFogMixMode = null; heightFogMode = null; @@ -205,7 +186,7 @@ public class LodFogConfig "float calculateHeightFogDepth(float vertical, float realY) { return 0.0; } \n" + "float mixFogThickness(float near, float far, float height) \n" + "{ \n" + - (drawNearFog ? "return 1.0-near;" : "return 0.0;") + + " return 0.0; \n" + "} \n\n"); } else @@ -247,7 +228,7 @@ public class LodFogConfig str.append("" + "float mixFogThickness(float near, float far, float height) \n" + "{ \n" + - getMixFogLine(heightFogMixMode, drawNearFog) + "\n" + + getMixFogLine(heightFogMixMode) + "\n" + "} \n"); } } @@ -354,7 +335,7 @@ public class LodFogConfig * creates a line in the format
* " return max(1.0-near, far);" */ - private static String getMixFogLine(EDhApiHeightFogMixMode heightFogMode, boolean drawNearFog) + private static String getMixFogLine(EDhApiHeightFogMixMode heightFogMode) { String str = " return "; @@ -362,65 +343,38 @@ public class LodFogConfig { case BASIC: case IGNORE_HEIGHT: - if (drawNearFog) - str += "max(1.0-near, far);\n"; - else str += "near * far;\n"; break; case ADDITION: - if (drawNearFog) - str += "max(1.0-near, far + height);\n"; - else str += "near * (far + height);\n"; break; case MAX: - if (drawNearFog) - str += "max(1.0-near, max(far, height));\n"; - else str += "near * max(far, height);\n"; break; case INVERSE_MULTIPLY: - if (drawNearFog) - str += "max(1.0-near, 1.0 - (1.0-far)*(1.0-height));\n"; - else str += "near * (1.0 - (1.0-far)*(1.0-height));\n"; break; case MULTIPLY: - if (drawNearFog) - str += "max(1.0-near, far*height);\n"; - else str += "near * far * height;\n"; break; case LIMITED_ADDITION: - if (drawNearFog) - str += "max(1.0-near, far + max(far, height));\n"; - else str += "near * (far + max(far, height));\n"; break; case MULTIPLY_ADDITION: - if (drawNearFog) - str += "max(1.0-near, far + far*height);\n"; - else str += "near * (far + far*height);\n"; break; case INVERSE_MULTIPLY_ADDITION: - if (drawNearFog) - str += "max(1.0-near, far + 1.0 - (1.0-far)*(1.0-height));\n"; - else str += "near * (far + 1.0 - (1.0-far)*(1.0-height));\n"; break; case AVERAGE: - if (drawNearFog) - str += "max(1.0-near, far*0.5 + height*0.5);\n"; - else str += "near * (far*0.5 + height*0.5);\n"; break; @@ -449,7 +403,7 @@ public class LodFogConfig return false; LodFogConfig that = (LodFogConfig) o; return Float.compare(that.heightFogHeight, heightFogHeight) == 0 && - drawNearFog == that.drawNearFog && Objects.equals(farFogSetting, that.farFogSetting) && + Objects.equals(farFogSetting, that.farFogSetting) && Objects.equals(heightFogSetting, that.heightFogSetting) && heightFogMixMode == that.heightFogMixMode && heightFogMode == that.heightFogMode // TODO: Move these out of here @@ -460,7 +414,7 @@ public class LodFogConfig @Override public int hashCode() { - return Objects.hash(farFogSetting, heightFogSetting, heightFogMixMode, heightFogMode, heightFogHeight, drawNearFog, earthCurveRatio, noiseEnable, noiseSteps, noiseIntensity, noiseDropoff); + return Objects.hash(farFogSetting, heightFogSetting, heightFogMixMode, heightFogMode, heightFogHeight, earthCurveRatio, noiseEnable, noiseSteps, noiseIntensity, noiseDropoff); } } From 74d193d294fbf2bcd2d7f28a32e2183dd4d757db Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 7 Apr 2024 14:00:03 -0500 Subject: [PATCH 138/183] Add a couple full data source null checks --- .../dataObjects/fullData/sources/FullDataSourceV2.java | 5 +++-- .../core/file/AbstractNewDataSourceHandler.java | 10 ++++++---- .../file/fullDatafile/FullDataSourceProviderV2.java | 7 +++++-- .../distanthorizons/core/render/LodRenderSection.java | 6 ++++++ 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 8e20be290..bbac771da 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -37,6 +37,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; import it.unimi.dsi.fastutil.longs.LongArrayList; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import javax.annotation.Nullable; import java.util.Arrays; @@ -223,8 +224,8 @@ public class FullDataSourceV2 implements IDataSource public LongArrayList get(int relX, int relZ) throws IndexOutOfBoundsException { return this.dataPoints[relativePosToIndex(relX, relZ)]; } @Override - public boolean update(FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } - public boolean update(FullDataSourceV2 inputDataSource) + public boolean update(@NotNull FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } + public boolean update(@NotNull FullDataSourceV2 inputDataSource) { byte thisDetailLevel = this.pos.getDetailLevel(); byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 0f5244621..1495dc2a4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -11,6 +11,7 @@ import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.threading.PositionalLockProvider; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.File; @@ -110,7 +111,7 @@ public abstract class AbstractNewDataSourceHandler /** * Returns the {@link TDataSource} for the given section position.
- * The returned data source may be null if there was a problem.

+ * The returned data source may be null if repo is in the process of shutting down.

* * This call is concurrent. I.e. it supports being called by multiple threads at the same time. */ @@ -135,9 +136,10 @@ public abstract class AbstractNewDataSourceHandler } /** * Should only be used in internal file handler methods where we are already running on a file handler thread. - * Shouldn't return null. + * Can return null if the repo is in the process of being shut down * @see AbstractNewDataSourceHandler#getAsync(DhSectionPos) */ + @Nullable public TDataSource get(DhSectionPos pos) { TDataSource dataSource = null; @@ -170,7 +172,7 @@ public abstract class AbstractNewDataSourceHandler // data updating // //===============// - public CompletableFuture updateDataSourceAsync(FullDataSourceV2 inputDataSource) + public CompletableFuture updateDataSourceAsync(@NotNull FullDataSourceV2 inputDataSource) { ThreadPoolExecutor executor = ThreadPoolUtil.getUpdatePropagatorExecutor(); if (executor == null || executor.isTerminated()) @@ -211,7 +213,7 @@ public abstract class AbstractNewDataSourceHandler * After this method returns the inputData will be written to file. * @param updatePos the position to update */ - protected void updateDataSourceAtPos(DhSectionPos updatePos, FullDataSourceV2 inputData, boolean lockOnUpdatePos) + protected void updateDataSourceAtPos(DhSectionPos updatePos, @NotNull FullDataSourceV2 inputData, boolean lockOnUpdatePos) { boolean methodLocked = false; // a lock is necessary to prevent two threads from writing to the same position at once, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 506cf5016..9fe3c89c9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -250,8 +250,11 @@ public class FullDataSourceProviderV2 try (FullDataSourceV2 dataSource = this.get(childPos)) { - this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); - this.repo.setApplyToParent(childPos, false); + if (dataSource != null) // can return null when the file handler is being shut down + { + this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); + this.repo.setApplyToParent(childPos, false); + } } } catch (Exception e) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 0952559b2..e619aaafe 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -138,6 +138,12 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { // get this positions data source fullDataSource = this.fullDataSourceProvider.get(this.pos); + if (fullDataSource == null) + { + // the file handler is being shut down, we won't be rendering anything anyway + return; + } + renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); if (renderSource.isEmpty()) { From 03e11fbe3e3139fea8bae496aa205248cdd4c544 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 7 Apr 2024 14:19:55 -0500 Subject: [PATCH 139/183] Improve migration update timeout message and increase the time --- .../FullDataSourceProviderV2.java | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 9fe3c89c9..122c1c0f0 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -66,8 +66,12 @@ public class FullDataSourceProviderV2 /** how many data sources should be pulled down for migration at once */ private static final int MIGRATION_BATCH_COUNT = NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD; private static final String MIGRATION_THREAD_NAME_PREFIX = "Full Data Migration Thread: "; - /** 1 minute */ - private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 60 * 1_000; + /** + * 5 minutes
+ * This should be much longer than any update should take. This is just + * to make sure the thread doesn't get stuck. + */ + private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000; protected final ThreadPoolExecutor migrationThreadPool; @@ -323,7 +327,7 @@ public class FullDataSourceProviderV2 int progressCount = 0; while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) { - LOGGER.info("Migrating ["+dimensionName+"] - [" + progressCount + "/" + totalCount + "]..."); + LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalCount + "]..."); ArrayList> updateFutureList = new ArrayList<>(); for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) @@ -341,7 +345,7 @@ public class FullDataSourceProviderV2 // the render data along with the full data, so running it async on the update threads gains us a good bit of speed CompletableFuture future = this.updateDataSourceAsync(newDataSource); updateFutureList.add(future); - future.thenRun(() -> + future.thenRun(() -> { // after the update finishes the legacy data source can be safely deleted this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos()); @@ -350,7 +354,7 @@ public class FullDataSourceProviderV2 catch (Exception e) { DhSectionPos migrationPos = legacyDataSource.getPos(); - LOGGER.error("Unexpected issue migrating data source at pos "+migrationPos+". Error: "+e.getMessage(), e); + LOGGER.error("Unexpected issue migrating data source at pos " + migrationPos + ". Error: " + e.getMessage(), e); this.legacyFileHandler.markMigrationFailed(migrationPos); } } @@ -362,9 +366,13 @@ public class FullDataSourceProviderV2 CompletableFuture combinedFutures = CompletableFuture.allOf(updateFutureList.toArray(new CompletableFuture[0])); combinedFutures.get(MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS, TimeUnit.MILLISECONDS); } - catch (InterruptedException | ExecutionException | TimeoutException e) + catch (InterruptedException | TimeoutException e) { - LOGGER.warn("Migration update timed out. Migration will re-try the same positions again. Error:"+e.getMessage(), e); + LOGGER.warn("Migration update timed out after ["+MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS+"] milliseconds. Migration will re-try the same positions again in a moment..", e); + } + catch (ExecutionException e) + { + LOGGER.warn("Migration update failed. Migration will re-try the same positions again. Error:"+e.getMessage(), e); } legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); From 7bdafa28cce081bdd016df60abb6851d57e1e730 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 7 Apr 2024 20:38:59 -0500 Subject: [PATCH 140/183] Fix debug detail render mode --- .../distanthorizons/api/enums/config/EDhApiLodShading.java | 6 +++--- .../dataObjects/render/bufferBuilding/CubicLodTemplate.java | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java index bfab3f7dc..b3ca135b8 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiLodShading.java @@ -20,9 +20,9 @@ package com.seibel.distanthorizons.api.enums.config; /** - * MINECRAFT
- * OLD_LIGHTING
- * NONE
+ * AUTO
+ * ENABLED
+ * DISABLED
* * @since API 1.1.0 * @version 2024-4-6 diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java index 9bb9c2076..45d107c99 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java @@ -88,6 +88,11 @@ public class CubicLodTemplate break; } case SHOW_DETAIL: + { + color = LodUtil.DEBUG_DETAIL_LEVEL_COLORS[detailLevel]; + fullBright = true; + break; + } case SHOW_BLOCK_MATERIAL: { switch (blockMaterialId) From 1ff4a56e2d8848a364276c1a035c824d3ee526e8 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sun, 7 Apr 2024 21:42:31 -0500 Subject: [PATCH 141/183] Improve grass side rendering and add a config --- .../config/EDhApiGrassSideRendering.java | 40 +++++++++++++++++ .../distanthorizons/core/config/Config.java | 11 +++++ .../RenderCacheConfigEventHandler.java | 12 +++-- .../render/bufferBuilding/ColumnBox.java | 19 +++----- .../bufferBuilding/CubicLodTemplate.java | 4 +- .../render/bufferBuilding/LodQuadBuilder.java | 45 ++++++++++++++----- .../distanthorizons/core/util/ColorUtil.java | 2 +- .../assets/distanthorizons/lang/en_us.json | 14 +++++- 8 files changed, 116 insertions(+), 31 deletions(-) create mode 100644 api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGrassSideRendering.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGrassSideRendering.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGrassSideRendering.java new file mode 100644 index 000000000..2f2f1a8fa --- /dev/null +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiGrassSideRendering.java @@ -0,0 +1,40 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.api.enums.config; + +/** + * AS_GRASS
+ * FADE_TO_DIRT
+ * AS_DIRT
+ * + * @since API 1.1.0 + * @version 2024-4-7 + */ +public enum EDhApiGrassSideRendering +{ + // Reminder: + // when adding items up the API minor version + // when removing items up the API major version + + AS_GRASS, + FADE_TO_DIRT, + AS_DIRT; + +} diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index ec19ad4f0..391aefa2d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -645,6 +645,17 @@ public class Config + "Disable this if shadows render incorrectly.") .build(); + public static ConfigEntry grassSideRendering = new ConfigEntry.Builder() + .set(EDhApiGrassSideRendering.AS_DIRT) + .comment("" + + "How should the sides and bottom of grass block LODs render? \n" + + "\n" + + EDhApiGrassSideRendering.AS_GRASS + ": all sides of dirt LOD's render using the top (green) color. \n" + + EDhApiGrassSideRendering.FADE_TO_DIRT + ": sides fade from grass to dirt. \n" + + EDhApiGrassSideRendering.AS_DIRT + ": sides render entirely as dirt. \n" + + "") + .setPerformance(EConfigEntryPerformance.NONE) + .build(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java index 6e3901462..24b613e9b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/RenderCacheConfigEventHandler.java @@ -20,10 +20,8 @@ package com.seibel.distanthorizons.core.config.eventHandlers; import com.seibel.distanthorizons.api.DhApi; -import com.seibel.distanthorizons.api.enums.config.EDhApiBlocksToAvoid; -import com.seibel.distanthorizons.api.enums.config.EDhApiLodShading; -import com.seibel.distanthorizons.api.enums.config.EDhApiMaxHorizontalResolution; -import com.seibel.distanthorizons.api.enums.config.EDhApiVerticalQuality; +import com.seibel.distanthorizons.api.enums.config.*; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; import com.seibel.distanthorizons.api.enums.rendering.EDhApiTransparency; import com.seibel.distanthorizons.core.config.listeners.ConfigChangeListener; import com.seibel.distanthorizons.core.config.Config; @@ -53,6 +51,9 @@ public class RenderCacheConfigEventHandler private final ConfigChangeListener brightnessMultiplierChangeListener; private final ConfigChangeListener saturationMultiplierChangeListener; private final ConfigChangeListener lodShadingChangeListener; + private final ConfigChangeListener grassSideChangeListener; + + private final ConfigChangeListener debugRenderingChangeListener; /** how long to wait in milliseconds before applying the config changes */ private static final long TIMEOUT_IN_MS = 4_000L; @@ -82,6 +83,9 @@ public class RenderCacheConfigEventHandler this.brightnessMultiplierChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.AdvancedGraphics.brightnessMultiplier, (newValue) -> this.refreshRenderDataAfterTimeout()); this.saturationMultiplierChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.AdvancedGraphics.saturationMultiplier, (newValue) -> this.refreshRenderDataAfterTimeout()); this.lodShadingChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.AdvancedGraphics.lodShading, (newValue) -> this.refreshRenderDataAfterTimeout()); + this.grassSideChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Graphics.AdvancedGraphics.grassSideRendering, (newValue) -> this.refreshRenderDataAfterTimeout()); + + this.debugRenderingChangeListener = new ConfigChangeListener<>(Config.Client.Advanced.Debugging.debugRendering, (newValue) -> this.refreshRenderDataAfterTimeout()); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java index 8d52addd6..edf2a4c67 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnBox.java @@ -61,23 +61,16 @@ public class ColumnBox color = ColorUtil.setAlpha(color, 255); } - // try to prevent displaying grass gradients for dirt blocks underneath solid blocks - if (!isTopTransparent && irisBlockMaterialId == IBlockStateWrapper.IrisBlockMaterial.GRASS) - { - irisBlockMaterialId = IBlockStateWrapper.IrisBlockMaterial.DIRT; - } - // cave culling prevention // prevents certain faces from being culled underground that should be allowed if (builder.skipQuadsWithZeroSkylight - && 0 == skyLight - && builder.skyLightCullingBelow > maxY - && - ( - (RenderDataPointUtil.getAlpha(topData) < 255 && RenderDataPointUtil.getYMax(topData) >= builder.skyLightCullingBelow) - || (RenderDataPointUtil.getYMin(topData) >= builder.skyLightCullingBelow) - || !RenderDataPointUtil.doesDataPointExist(topData) + && 0 == skyLight + && builder.skyLightCullingBelow > maxY + && ( + (RenderDataPointUtil.getAlpha(topData) < 255 && RenderDataPointUtil.getYMax(topData) >= builder.skyLightCullingBelow) + || (RenderDataPointUtil.getYMin(topData) >= builder.skyLightCullingBelow) + || !RenderDataPointUtil.doesDataPointExist(topData) ) ) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java index 45d107c99..87b2e7720 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/CubicLodTemplate.java @@ -103,7 +103,7 @@ public class CubicLodTemplate break; case IBlockStateWrapper.IrisBlockMaterial.LEAVES: - color = ColorUtil.GREEN; + color = ColorUtil.DARK_GREEN; break; case IBlockStateWrapper.IrisBlockMaterial.STONE: color = ColorUtil.GRAY; @@ -139,7 +139,7 @@ public class CubicLodTemplate color = ColorUtil.BLUE; break; case IBlockStateWrapper.IrisBlockMaterial.GRASS: - color = ColorUtil.LIGHT_GREEN; + color = ColorUtil.GREEN; break; case IBlockStateWrapper.IrisBlockMaterial.ILLUMINATED: color = ColorUtil.YELLOW; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java index dabcea970..07744dcb9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/LodQuadBuilder.java @@ -23,6 +23,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.*; +import com.seibel.distanthorizons.api.enums.config.EDhApiGrassSideRendering; +import com.seibel.distanthorizons.api.enums.rendering.EDhApiDebugRendering; +import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.enums.EDhDirection; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; @@ -58,6 +61,8 @@ public class LodQuadBuilder private final boolean doTransparency; private final IClientLevelWrapper clientLevelWrapper; + private final EDhApiDebugRendering debugRenderingMode; + private final EDhApiGrassSideRendering grassSideRenderingMode; public static final int[][][] DIRECTION_VERTEX_IBO_QUAD = new int[][][] @@ -130,6 +135,9 @@ public class LodQuadBuilder this.skyLightCullingBelow = skyLightCullingBelow; this.clientLevelWrapper = clientLevelWrapper; + this.debugRenderingMode = Config.Client.Advanced.Debugging.debugRendering.get(); + this.grassSideRenderingMode = Config.Client.Advanced.Graphics.AdvancedGraphics.grassSideRendering.get(); + } @@ -263,19 +271,36 @@ public class LodQuadBuilder int color = quad.color; - if (quad.irisBlockMaterialId == IBlockStateWrapper.IrisBlockMaterial.GRASS - && - ( - (quad.direction.getAxis().isHorizontal() && quadBase[i][1] == 0) // TODO what does "quadBase[i][1] == 0" mean? - || quad.direction == EDhDirection.DOWN) - ) + + // use custom side color logic for grass blocks + if (quad.irisBlockMaterialId == IBlockStateWrapper.IrisBlockMaterial.GRASS) { - // for horizontal and bottom faces of grass blocks, use the dirt color to - // prevent green cliff walls - color = this.clientLevelWrapper.getDirtBlockColor(); - color = ColorUtil.applyShade(color, MC.getShade(quad.direction)); + // only use dirt colors if debug rendering is disabled + if (this.debugRenderingMode == EDhApiDebugRendering.OFF) + { + // determine if any custom coloring logic should be used + if (this.grassSideRenderingMode != EDhApiGrassSideRendering.AS_GRASS) + { + // only change the vertex color if it's on the side or bottom + if (quad.direction.getAxis().isHorizontal() || quad.direction == EDhDirection.DOWN) + { + if (this.grassSideRenderingMode == EDhApiGrassSideRendering.AS_DIRT + // if we want the color to fade, only apply the dirt color to the bottom vertices + || (this.grassSideRenderingMode == EDhApiGrassSideRendering.FADE_TO_DIRT && quadBase[i][1] == 0) + // always render the bottom as dirt + || quad.direction == EDhDirection.DOWN) + { + // for horizontal and bottom faces of grass blocks, use the dirt color to + // prevent green cliff walls + color = this.clientLevelWrapper.getDirtBlockColor(); + color = ColorUtil.applyShade(color, MC.getShade(quad.direction)); + } + } + } + } } + this.putVertex(bb, (short) (quad.x + dx), (short) (quad.y + dy), (short) (quad.z + dz), quad.hasError ? ColorUtil.RED : color, quad.hasError ? 0 : normalIndex, diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java index c0ffb7554..222fcbb9b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/ColorUtil.java @@ -41,7 +41,7 @@ public class ColorUtil public static final int RED = rgbToInt(255, 0, 0); public static final int DARK_RED = rgbToInt(100, 0, 0); public static final int GREEN = rgbToInt(0, 255, 0); - public static final int LIGHT_GREEN = rgbToInt(80, 255, 80); + public static final int DARK_GREEN = rgbToInt(80, 140, 80); public static final int BLUE = rgbToInt(0, 0, 255); public static final int YELLOW = rgbToInt(255, 255, 0); public static final int CYAN = rgbToInt(0, 255, 255); diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index d8f6bdc19..d88fb35c3 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -305,6 +305,10 @@ "Disable Shadow Pass Frustum Culling", "distanthorizons.config.client.advanced.graphics.advancedGraphics.disableShadowPassFrustumCulling.@tooltip": "Identical to the other frustum culling option except that it is \nonly used when a shader mod is present using the DH API \nand the shadow pass is being rendered. \n\nDisable this if shadows render incorrectly.", + "distanthorizons.config.client.advanced.graphics.advancedGraphics.grassSideRendering": + "Grass Side Rendering", + "distanthorizons.config.client.advanced.graphics.advancedGraphics.grassSideRendering.@tooltip": + "How should the sides and bottom of grass block LODs render?", "distanthorizons.config.client.advanced.worldGenerator": @@ -913,5 +917,13 @@ "distanthorizons.config.enum.EDhApiUpdateBranch.STABLE": "Stable", "distanthorizons.config.enum.EDhApiUpdateBranch.NIGHTLY": - "Nightly" + "Nightly", + + "distanthorizons.config.enum.EDhApiGrassSideRendering.AS_GRASS": + "As Grass", + "distanthorizons.config.enum.EDhApiGrassSideRendering.FADE_TO_DIRT": + "Fade To Dirt", + "distanthorizons.config.enum.EDhApiGrassSideRendering.AS_DIRT": + "As Dirt" + } From d8bac9df8cc834dec5315ca8e0e5ac9ed3fd3af4 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 8 Apr 2024 07:17:17 -0500 Subject: [PATCH 142/183] Remove deprecated DhApiScreenResizeEvent --- .../DhApiScreenResizeEvent.java | 77 ------------------- .../core/render/renderer/LodRenderer.java | 8 -- 2 files changed, 85 deletions(-) delete mode 100644 api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java diff --git a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java b/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java deleted file mode 100644 index 2fdad3b66..000000000 --- a/api/src/main/java/com/seibel/distanthorizons/api/methods/events/abstractEvents/DhApiScreenResizeEvent.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * This file is part of the Distant Horizons mod - * licensed under the GNU LGPL v3 License. - * - * Copyright (C) 2020-2023 James Seibel - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, version 3. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this program. If not, see . - */ - -package com.seibel.distanthorizons.api.methods.events.abstractEvents; - -import com.seibel.distanthorizons.api.methods.events.interfaces.IDhApiEvent; -import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiEventParam; - -/** - * TODO remove - * - * @author James Seibel - * @version 2023-1-23 - * @since API 1.1.0 - */ -@Deprecated // TODO remove before stable release and write a merge request for Iris -public abstract class DhApiScreenResizeEvent implements IDhApiEvent -{ - /** Fired immediately before Distant Horizons handles the screen resize. */ - public abstract void onResize(DhApiEventParam event); - - - //=========================// - // internal DH API methods // - //=========================// - - @Override - public final void fireEvent(DhApiEventParam event) { this.onResize(event); } - - - //==================// - // parameter object // - //==================// - - public static class EventParam - { - /** Measured in pixels */ - public final int previousWidth; - /** Measured in pixels */ - public final int previousHeight; - - /** Measured in pixels */ - public final int newWidth; - /** Measured in pixels */ - public final int newHeight; - - - public EventParam( - int previousWidth, int previousHeight, - int newWidth, int newHeight) - { - this.previousWidth = previousWidth; - this.previousHeight = previousHeight; - - this.newWidth = newWidth; - this.newHeight = newHeight; - - } - } - -} \ No newline at end of file diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index 78de044ef..2d66a10fd 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -744,19 +744,11 @@ public class LodRenderer this.cachedHeight = MC_RENDER.getTargetFrameBufferViewportHeight(); - // TODO add on texture (re)created event - // https://discord.com/channels/881614130614767666/1211290858134052894/1211431000580554752 ApiEventInjector.INSTANCE.fireAllEvents(DhApiColorDepthTextureCreatedEvent.class, new DhApiColorDepthTextureCreatedEvent.EventParam( oldWidth, oldHeight, this.cachedWidth, this.cachedHeight )); - - ApiEventInjector.INSTANCE.fireAllEvents(DhApiScreenResizeEvent.class, - new DhApiScreenResizeEvent.EventParam( - oldWidth, oldHeight, - this.cachedWidth, this.cachedHeight - )); // also update the override if present From 336e8b99ec6d86a69c24c59991f81c0b4d3a5500 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 8 Apr 2024 07:43:21 -0500 Subject: [PATCH 143/183] Reduce update thread pool default counts --- .../eventHandlers/presets/ThreadPresetConfigEventHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 0a25d0695..b1acc2e79 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -85,13 +85,13 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan }}); - public static int getUpdatePropagatorDefaultThreadCount() { return getThreadCountByPercent(0.5); } + public static int getUpdatePropagatorDefaultThreadCount() { return getThreadCountByPercent(0.25); } private final ConfigEntryWithPresetOptions UpdatePropagatorThreadCount = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads, new HashMap() {{ this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 1); this.put(EDhApiThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultThreadCount()); - this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.75)); + this.put(EDhApiThreadPreset.BALANCED, getThreadCountByPercent(0.5)); this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.75)); this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); From cd2b5fd668629b9ec37f1519f028051c7d773ded Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 9 Apr 2024 07:11:59 -0500 Subject: [PATCH 144/183] Speed up world gen queue initial loading --- .../GeneratedFullDataSourceProvider.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index e2bc45e77..8c637ece3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -236,7 +236,36 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im } - // TODO based on the column generation step, only check children that are un-generated + // don't check any child positions if this position is already fully generated + if (this.repo.existsWithKey(pos)) + { + byte[] columnGenerationSteps = this.repo.getColumnGenerationStepForPos(pos); + // shouldn't happen, but just in case + if (columnGenerationSteps != null) + { + boolean positionFullyGenerated = true; + + // check if any positions are ungenerated + for (int i = 0; i < columnGenerationSteps.length; i++) + { + if (columnGenerationSteps[i] == EDhApiWorldGenerationStep.EMPTY.value) + { + positionFullyGenerated = false; + break; + } + } + + if (positionFullyGenerated) + { + return new ArrayList<>(); + } + } + } + + + + // this section is missing one or more columns, queue the missing ones for generation. + // TODO speed up this logic by only checking ungenerated columns ArrayList generationList = new ArrayList<>(); byte minGeneratorSectionDetailLevel = (byte) (worldGenQueue.highestDataDetail() + DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL); pos.forEachChildAtDetailLevel(minGeneratorSectionDetailLevel, (genPos) -> From e10617ac1958c140871c25d1a8f8755617a7f1aa Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 9 Apr 2024 07:13:01 -0500 Subject: [PATCH 145/183] increase update populator thread default run time ratio --- .../eventHandlers/presets/ThreadPresetConfigEventHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index b1acc2e79..134465c08 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -95,13 +95,13 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.75)); this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); - public static double getUpdatePropagatorDefaultRunTimeRatio() { return 0.25; } + public static double getUpdatePropagatorDefaultRunTimeRatio() { return 0.5; } private final ConfigEntryWithPresetOptions UpdatePropagatorRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads, new HashMap() {{ this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.25); this.put(EDhApiThreadPreset.LOW_IMPACT, getUpdatePropagatorDefaultRunTimeRatio()); - this.put(EDhApiThreadPreset.BALANCED, 0.5); + this.put(EDhApiThreadPreset.BALANCED, 0.75); this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0); this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); }}); From a0e7bb94c63935f446b1be727ae98d72571f0fef Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 9 Apr 2024 07:48:31 -0500 Subject: [PATCH 146/183] Add a ApplyToParent DB index to improve file handling speed --- .../resources/sqlScripts/0050-sqlite-addApplyToParentIndex.sql | 3 +++ core/src/main/resources/sqlScripts/scriptList.txt | 1 + 2 files changed, 4 insertions(+) create mode 100644 core/src/main/resources/sqlScripts/0050-sqlite-addApplyToParentIndex.sql diff --git a/core/src/main/resources/sqlScripts/0050-sqlite-addApplyToParentIndex.sql b/core/src/main/resources/sqlScripts/0050-sqlite-addApplyToParentIndex.sql new file mode 100644 index 000000000..3f8fb5e93 --- /dev/null +++ b/core/src/main/resources/sqlScripts/0050-sqlite-addApplyToParentIndex.sql @@ -0,0 +1,3 @@ + +-- significantly speeds up parent update handling +create index FullDataUpdatedIndex on FullData (ApplyToParent) where ApplyToParent = 1 diff --git a/core/src/main/resources/sqlScripts/scriptList.txt b/core/src/main/resources/sqlScripts/scriptList.txt index 62901d865..d2104f42c 100644 --- a/core/src/main/resources/sqlScripts/scriptList.txt +++ b/core/src/main/resources/sqlScripts/scriptList.txt @@ -4,3 +4,4 @@ 0030-sqlite-changeTableJournaling.sql 0031-sqlite-useSqliteWalJournaling.sql 0040-sqlite-removeRenderCache.sql +0050-sqlite-addApplyToParentIndex.sql From cf74d17e1b6cc834c561b2991274fef3c74341f9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 9 Apr 2024 19:27:57 -0500 Subject: [PATCH 147/183] Fix world gen tasks being incorrectly removed from the queue at long distances --- .../FullDataSourceProviderV2.java | 6 ++ .../GeneratedFullDataSourceProvider.java | 62 +++++++++---------- .../IFullDataSourceRetrievalQueue.java | 9 +-- .../core/generation/WorldGenerationQueue.java | 12 +--- .../core/level/DhClientServerLevel.java | 7 --- .../core/level/WorldGenModule.java | 2 +- .../core/render/LodRenderSection.java | 17 +++++ .../core/util/threading/ThreadPoolUtil.java | 7 +++ 8 files changed, 65 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 122c1c0f0..5f97d5587 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -45,6 +45,7 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; /** * Handles reading/writing {@link FullDataSourceV2} @@ -452,6 +453,11 @@ public class FullDataSourceProviderV2 /** @return true if the position was queued, false if not */ public boolean queuePositionForRetrieval(DhSectionPos genPos) { return false; } + /** does nothing if the given position isn't present in the queue */ + public void removeRetrievalRequestIf(Function removeIf) { } + + public void clearRetrievalQueue() { } + /** Can be used to display how many total retrieval requests might be available. */ public void setTotalRetrievalPositionCount(int newCount) { } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java index 8c637ece3..dc2abe9cf 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/GeneratedFullDataSourceProvider.java @@ -65,40 +65,6 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im - //==================// - // generation queue // - //==================// - - /** - * Assigns the queue for handling world gen and does first time setup as well.
- * Assumes there isn't a pre-existing queue. - */ - public void setWorldGenerationQueue(IFullDataSourceRetrievalQueue newWorldGenQueue) - { - boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); - LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); - LOGGER.info("Set world gen queue for level ["+this.level+"]."); - } - - public void clearGenerationQueue() { this.worldGenQueueRef.set(null); } - - /** Can be used to remove positions that are outside the player's render distance. */ - public void removeGenRequestIf(Function removeIf) - { - // TODO there has to be a better way to do this - - IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); - if (worldGenQueue != null) - { - worldGenQueue.removeGenRequestIf(removeIf); - } - } - - @Override - public int getUnsavedDataSourceCount() { return this.delayedFullDataSourceSaveCache.getUnsavedCount(); } - - - //=================// // event listeners // //=================// @@ -157,6 +123,17 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im // world gen (data source retrieval) // //===================================// + /** + * Assigns the queue for handling world gen and does first time setup as well.
+ * Assumes there isn't a pre-existing queue. + */ + public void setWorldGenerationQueue(IFullDataSourceRetrievalQueue newWorldGenQueue) + { + boolean oldQueueExists = this.worldGenQueueRef.compareAndSet(null, newWorldGenQueue); + LodUtil.assertTrue(oldQueueExists, "previous world gen queue is still here!"); + LOGGER.info("Set world gen queue for level ["+this.level+"]."); + } + @Override public boolean canRetrieveMissingDataSources() { return true; } @@ -226,6 +203,23 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im return true; } + @Override + public void removeRetrievalRequestIf(Function removeIf) + { + IFullDataSourceRetrievalQueue worldGenQueue = this.worldGenQueueRef.get(); + if (worldGenQueue != null) + { + worldGenQueue.removeRetrievalRequestIf(removeIf); + } + } + + @Override + public void clearRetrievalQueue() { this.worldGenQueueRef.set(null); } + + @Override + public int getUnsavedDataSourceCount() { return this.delayedFullDataSourceSaveCache.getUnsavedCount(); } + + @Override public ArrayList getPositionsToRetrieve(DhSectionPos pos) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java index 5bf47abd8..ade0d524e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/IFullDataSourceRetrievalQueue.java @@ -76,10 +76,11 @@ public interface IFullDataSourceRetrievalQueue extends Closeable // task handling // //===============// - /** @deprecated replace with {@link IFullDataSourceRetrievalQueue#removeGenTask(DhSectionPos)} */ - @Deprecated - void removeGenRequestIf(Function removeIf); - void removeGenTask(DhSectionPos pos); + /** + * Generally the retrieval queue should be fairly small, so its faster to iterate over the existing list + * and check if each one is valid vs dumbly attempting to remove every position that just went out of range. + */ + void removeRetrievalRequestIf(Function removeIf); CompletableFuture submitGenTask(DhSectionPos pos, byte requiredDataDetail, IWorldGenTaskTracker tracker); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java index 05d75ec8c..04c485bf2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/generation/WorldGenerationQueue.java @@ -159,7 +159,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb } @Override - public void removeGenRequestIf(Function removeIf) + public void removeRetrievalRequestIf(Function removeIf) { this.waitingTasks.forEachKey(100, (genPos) -> { @@ -170,16 +170,6 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb }); } - @Override - public void removeGenTask(DhSectionPos pos) - { - WorldGenTask task = this.waitingTasks.remove(pos); - if (task != null) - { - task.future.cancel(true); - } - } - diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 1728d7825..7da152d82 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -124,13 +124,6 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev if (isWorldGenRunning) { - ClientLevelModule.ClientRenderState renderState = this.clientside.ClientRenderStateRef.get(); - if (renderState != null && renderState.quadtree != null) - { - // remove any generator sections that are out of bounds - this.serverside.dataFileHandler.removeGenRequestIf(pos -> !renderState.quadtree.isSectionPosInBounds(pos)); - } - this.serverside.worldGenModule.worldGenTick(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos())); } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index ce0040165..baab12724 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -101,7 +101,7 @@ public class WorldGenModule implements Closeable return; } } - dataFileHandler.clearGenerationQueue(); + dataFileHandler.clearRetrievalQueue(); worldGenState.closeAsync(true).join(); //TODO: Make it async. dataFileHandler.removeWorldGenCompleteListener(this.onWorldGenCompleteListener); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index e619aaafe..81346e1e2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -41,6 +41,7 @@ import java.awt.*; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; /** @@ -345,6 +346,22 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { this.renderSourceLoadingFuture.cancel(true); } + + + // remove any active world gen requests that may be for this position + ThreadPoolExecutor executor = ThreadPoolUtil.getCleanupExecutor(); + if (executor != null && !executor.isTerminated()) + { + // while this should generally be a fast operation + // this is run on a separate thread to prevent lag on the render thread + + try + { + executor.execute(() -> this.fullDataSourceProvider.removeRetrievalRequestIf((genPos) -> this.pos.contains(genPos))); + } + catch (RejectedExecutionException ignore) + { /* If this happens that means everything is already shut down and no additional cleanup will be necessary */ } + } } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java index 3adacc81c..e4f786a84 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/threading/ThreadPoolUtil.java @@ -61,6 +61,11 @@ public class ThreadPoolUtil @Nullable public static ThreadPoolExecutor getBufferUploaderExecutor() { return bufferUploaderThreadPool; } + public static final String CLEANUP_THREAD_NAME = "Cleanup"; + private static ThreadPoolExecutor cleanupThreadPool; + @Nullable + public static ThreadPoolExecutor getCleanupExecutor() { return cleanupThreadPool; } + //======================// @@ -108,6 +113,7 @@ public class ThreadPoolUtil updatePropagatorThreadPool = new ConfigThreadPool(UPDATE_PROPAGATOR_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfUpdatePropagatorThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads, null); worldGenThreadPool = new ConfigThreadPool(WORLD_GEN_THREAD_FACTORY, Config.Client.Advanced.MultiThreading.numberOfWorldGenerationThreads, Config.Client.Advanced.MultiThreading.runTimeRatioForWorldGenerationThreads, null); bufferUploaderThreadPool = ThreadUtil.makeSingleThreadPool(BUFFER_UPLOADER_THREAD_NAME); + cleanupThreadPool = ThreadUtil.makeSingleThreadPool(CLEANUP_THREAD_NAME); @@ -148,6 +154,7 @@ public class ThreadPoolUtil updatePropagatorThreadPool.shutdownExecutorService(); worldGenThreadPool.shutdownExecutorService(); bufferUploaderThreadPool.shutdown(); + cleanupThreadPool.shutdown(); // worker threads From 77fffb1a9b67feffaeed60296cd72a0cba044d7a Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 9 Apr 2024 19:31:20 -0500 Subject: [PATCH 148/183] add a TODO to FullDataSourceProviderV2 --- .../core/file/fullDatafile/FullDataSourceProviderV2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 5f97d5587..ac30b3249 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -193,7 +193,7 @@ public class FullDataSourceProviderV2 continue; } - + // TODO it might be worth skipping this logic if no parent updates happened // queue parent updates if (executor.getQueue().size() < MAX_UPDATE_TASK_COUNT From e560ddf3dce36edcd3a93ecf7285e2feed85abf6 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Tue, 9 Apr 2024 21:29:12 -0500 Subject: [PATCH 149/183] Fix sqlite database files not being unlocked --- .../FullDataSourceProviderV2.java | 6 +++-- .../core/level/ClientLevelModule.java | 10 +++---- .../core/level/DhClientLevel.java | 26 ++++++------------- .../core/level/DhClientServerLevel.java | 9 +++---- .../core/level/DhServerLevel.java | 6 ++--- .../core/level/ServerLevelModule.java | 11 ++++---- .../core/level/WorldGenModule.java | 5 +--- .../core/render/LodQuadTree.java | 4 ++- .../core/render/LodRenderSection.java | 12 +++++---- .../core/sql/repo/AbstractDhRepo.java | 15 ++++++++--- 10 files changed, 52 insertions(+), 52 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index ac30b3249..7a8a7cf57 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -81,7 +81,7 @@ public class FullDataSourceProviderV2 * vs gracefully shutting down the thread ourselves. */ protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); - protected final FullDataSourceProviderV1 legacyFileHandler; + protected final FullDataSourceProviderV1 legacyFileHandler; /** * Tracks which positions are currently being updated @@ -106,7 +106,7 @@ public class FullDataSourceProviderV2 public FullDataSourceProviderV2(IDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { super(level, saveStructure, saveDirOverride); - this.legacyFileHandler = new FullDataSourceProviderV1(level, saveStructure, saveDirOverride); + this.legacyFileHandler = new FullDataSourceProviderV1<>(level, saveStructure, saveDirOverride); DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showFullDataUpdateStatus); @@ -492,6 +492,8 @@ public class FullDataSourceProviderV2 super.close(); this.updateQueueProcessor.shutdownNow(); + this.legacyFileHandler.close(); + this.migrationThreadRunning.set(false); this.migrationThreadPool.shutdown(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 31fcd1158..801a2a256 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -41,6 +41,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrap import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import org.apache.logging.log4j.Logger; +import javax.annotation.WillNotClose; import java.io.Closeable; import java.util.ArrayList; import java.util.concurrent.CompletableFuture; @@ -54,6 +55,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle private final IDhClientLevel clientLevel; + @WillNotClose public final FullDataSourceProviderV2 fullDataSourceProvider; public final AtomicReference ClientRenderStateRef = new AtomicReference<>(); @@ -111,7 +113,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle } clientRenderState.close(); - clientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure()); + clientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider()); if (!this.ClientRenderStateRef.compareAndSet(null, clientRenderState)) { //FIXME: How to handle this? @@ -144,7 +146,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle public boolean startRenderer(IClientLevelWrapper clientLevelWrapper) { // TODO why are we passing in a level wrapper? Our client level is already defined. - ClientRenderState ClientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider(), this.clientLevel.getSaveStructure()); + ClientRenderState ClientRenderState = new ClientRenderState(this.clientLevel, clientLevelWrapper, this.clientLevel.getFullDataProvider()); if (!this.ClientRenderStateRef.compareAndSet(null, ClientRenderState)) { LOGGER.warn("Failed to start renderer due to concurrency"); @@ -324,9 +326,7 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle public final LodQuadTree quadtree; public final LodRenderer renderer; - public ClientRenderState( - IDhClientLevel dhClientLevel, IClientLevelWrapper clientLevelWrapper, FullDataSourceProviderV2 fullDataSourceProvider, - AbstractSaveStructure saveStructure) + public ClientRenderState(IDhClientLevel dhClientLevel, IClientLevelWrapper clientLevelWrapper, FullDataSourceProviderV2 fullDataSourceProvider) { this.clientLevelWrapper = clientLevelWrapper; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java index f3af49231..184bf20af 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientLevel.java @@ -103,47 +103,37 @@ public class DhClientLevel extends AbstractDhLevel implements IDhClientLevel //================// @Override - public int computeBaseColor(DhBlockPos pos, IBiomeWrapper biome, IBlockStateWrapper block) { return levelWrapper.computeBaseColor(pos, biome, block); } + public int computeBaseColor(DhBlockPos pos, IBiomeWrapper biome, IBlockStateWrapper block) { return this.levelWrapper.computeBaseColor(pos, biome, block); } @Override - public IClientLevelWrapper getClientLevelWrapper() { return levelWrapper; } + public IClientLevelWrapper getClientLevelWrapper() { return this.levelWrapper; } @Override - public void clearRenderCache() - { - clientside.clearRenderCache(); - } + public void clearRenderCache() { this.clientside.clearRenderCache(); } @Override - public ILevelWrapper getLevelWrapper() { return levelWrapper; } + public ILevelWrapper getLevelWrapper() { return this.levelWrapper; } @Override public CompletableFuture updateDataSourcesAsync(FullDataSourceV2 data) { return this.clientside.updateDataSourcesAsync(data); } @Override - public int getMinY() { return levelWrapper.getMinHeight(); } + public int getMinY() { return this.levelWrapper.getMinHeight(); } @Override public void close() { - clientside.close(); + this.clientside.close(); super.close(); - dataFileHandler.close(); + this.dataFileHandler.close(); LOGGER.info("Closed " + DhClientLevel.class.getSimpleName() + " for " + levelWrapper); } - //=======================// - // misc helper functions // - //=======================// - @Override public FullDataSourceProviderV2 getFullDataProvider() { return this.dataFileHandler; } @Override - public AbstractSaveStructure getSaveStructure() - { - return saveStructure; - } + public AbstractSaveStructure getSaveStructure() { return this.saveStructure; } @Override public boolean hasSkyLight() { return this.levelWrapper.hasSkyLight(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java index 7da152d82..0527f61c9 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhClientServerLevel.java @@ -23,14 +23,12 @@ import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhAp import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.render.LodRenderSection; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; -import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; @@ -41,7 +39,6 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.IServerLevelWrapp import org.apache.logging.log4j.Logger; import java.awt.*; -import java.util.Iterator; import java.util.concurrent.CompletableFuture; /** The level used on a singleplayer world */ @@ -103,7 +100,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev // start world gen // create a new queue - this.serverside.worldGenModule.startWorldGen(this.serverside.dataFileHandler, new ServerLevelModule.WorldGenState(this)); + this.serverside.worldGenModule.startWorldGen(this.serverside.fullDataFileHandler, new ServerLevelModule.WorldGenState(this)); // TODO I think this used to queue the world gen // is it still needed? @@ -119,7 +116,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev else if (!shouldDoWorldGen && isWorldGenRunning) { // stop world gen - this.serverside.worldGenModule.stopWorldGen(this.serverside.dataFileHandler); + this.serverside.worldGenModule.stopWorldGen(this.serverside.fullDataFileHandler); } if (isWorldGenRunning) @@ -169,7 +166,7 @@ public class DhClientServerLevel extends AbstractDhLevel implements IDhClientLev public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public FullDataSourceProviderV2 getFullDataProvider() { return this.serverside.dataFileHandler; } + public FullDataSourceProviderV2 getFullDataProvider() { return this.serverside.fullDataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java index 9af6fdc32..8b591258a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/DhServerLevel.java @@ -72,12 +72,12 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel if (shouldDoWorldGen && !isWorldGenRunning) { // start world gen - serverside.worldGenModule.startWorldGen(serverside.dataFileHandler, new ServerLevelModule.WorldGenState(this)); + serverside.worldGenModule.startWorldGen(serverside.fullDataFileHandler, new ServerLevelModule.WorldGenState(this)); } else if (!shouldDoWorldGen && isWorldGenRunning) { // stop world gen - serverside.worldGenModule.stopWorldGen(serverside.dataFileHandler); + serverside.worldGenModule.stopWorldGen(serverside.fullDataFileHandler); } if (serverside.worldGenModule.isWorldGenRunning()) @@ -93,7 +93,7 @@ public class DhServerLevel extends AbstractDhLevel implements IDhServerLevel public ILevelWrapper getLevelWrapper() { return getServerLevelWrapper(); } @Override - public FullDataSourceProviderV2 getFullDataProvider() { return this.serverside.dataFileHandler; } + public FullDataSourceProviderV2 getFullDataProvider() { return this.serverside.fullDataFileHandler; } @Override public AbstractSaveStructure getSaveStructure() diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java index 1deb4a4e4..57a7fedf4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ServerLevelModule.java @@ -30,13 +30,13 @@ import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.coreapi.DependencyInjection.WorldGeneratorInjector; import org.apache.logging.log4j.Logger; -public class ServerLevelModule +public class ServerLevelModule implements AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); public final IDhServerLevel parentServerLevel; public final AbstractSaveStructure saveStructure; - public final GeneratedFullDataSourceProvider dataFileHandler; + public final GeneratedFullDataSourceProvider fullDataFileHandler; public final AppliedConfigState worldGeneratorEnabledConfig; public final WorldGenModule worldGenModule; @@ -47,18 +47,19 @@ public class ServerLevelModule { this.parentServerLevel = parentServerLevel; this.saveStructure = saveStructure; - this.dataFileHandler = new GeneratedFullDataSourceProvider(parentServerLevel, saveStructure); + this.fullDataFileHandler = new GeneratedFullDataSourceProvider(parentServerLevel, saveStructure); this.worldGeneratorEnabledConfig = new AppliedConfigState<>(Config.Client.Advanced.WorldGenerator.enableDistantGeneration); - this.worldGenModule = new WorldGenModule(this.dataFileHandler, this.parentServerLevel); + this.worldGenModule = new WorldGenModule(this.parentServerLevel); } + @Override public void close() { // shutdown the world-gen this.worldGenModule.close(); - this.dataFileHandler.close(); + this.fullDataFileHandler.close(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java index baab12724..07c9dd3b7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/WorldGenModule.java @@ -34,7 +34,6 @@ public class WorldGenModule implements Closeable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - private final GeneratedFullDataSourceProvider dataFileHandler; private final GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener; private final AtomicReference worldGenStateRef = new AtomicReference<>(); @@ -42,9 +41,8 @@ public class WorldGenModule implements Closeable - public WorldGenModule(GeneratedFullDataSourceProvider dataFileHandler, GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener) + public WorldGenModule(GeneratedFullDataSourceProvider.IOnWorldGenCompleteListener onWorldGenCompleteListener) { - this.dataFileHandler = dataFileHandler; this.onWorldGenCompleteListener = onWorldGenCompleteListener; this.worldGenF3Message = new F3Screen.DynamicMessage(() -> { @@ -138,7 +136,6 @@ public class WorldGenModule implements Closeable } } - this.dataFileHandler.close(); this.worldGenF3Message.close(); } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 245919657..6ade073f2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -37,6 +37,7 @@ import com.seibel.distanthorizons.coreapi.util.MathUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import javax.annotation.WillNotClose; import java.util.ArrayList; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; @@ -59,6 +60,7 @@ public class LodQuadTree extends QuadTree implements AutoClose public final int blockRenderDistanceDiameter; + @WillNotClose private final FullDataSourceProviderV2 fullDataSourceProvider; /** @@ -508,7 +510,7 @@ public class LodQuadTree extends QuadTree implements AutoClose break; } - renderSection.tryQueuingMissingLodRetrieval(this.fullDataSourceProvider); + renderSection.tryQueuingMissingLodRetrieval(); } // calculate an estimate for the max number of tasks for the queue diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 81346e1e2..2720fe020 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -37,6 +37,7 @@ import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; +import javax.annotation.WillNotClose; import java.awt.*; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -68,6 +69,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public final DhSectionPos pos; private final IDhClientLevel level; + @WillNotClose private final FullDataSourceProviderV2 fullDataSourceProvider; @@ -279,14 +281,14 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public boolean missingPositionsCalculated() { return this.missingPositionsCalculated; } public int ungeneratedPositionCount() { return (this.missingGenerationPos != null) ? this.missingGenerationPos.size() : 0; } - public void tryQueuingMissingLodRetrieval(FullDataSourceProviderV2 fullDataSourceProvider) + public void tryQueuingMissingLodRetrieval() { - if (fullDataSourceProvider.canRetrieveMissingDataSources() && fullDataSourceProvider.canQueueRetrieval()) + if (this.fullDataSourceProvider.canRetrieveMissingDataSources() && this.fullDataSourceProvider.canQueueRetrieval()) { // calculate the missing positions if not already done if (!this.missingPositionsCalculated) { - this.missingGenerationPos = fullDataSourceProvider.getPositionsToRetrieve(this.pos); + this.missingGenerationPos = this.fullDataSourceProvider.getPositionsToRetrieve(this.pos); if (this.missingGenerationPos != null) { this.missingPositionsCalculated = true; @@ -299,14 +301,14 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // queue from last to first to prevent shifting the array unnecessarily for (int i = this.missingGenerationPos.size() - 1; i >= 0; i--) { - if (!fullDataSourceProvider.canQueueRetrieval()) + if (!this.fullDataSourceProvider.canQueueRetrieval()) { // the data source provider isn't accepting any more jobs break; } DhSectionPos pos = this.missingGenerationPos.remove(i); - boolean positionQueued = fullDataSourceProvider.queuePositionForRetrieval(pos); + boolean positionQueued = this.fullDataSourceProvider.queuePositionForRetrieval(pos); if (!positionQueued) { // shouldn't normally happen, but just in case diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java index aa45199f9..cdd059b59 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/AbstractDhRepo.java @@ -37,7 +37,7 @@ import java.util.concurrent.locks.ReentrantLock; * * @param DTO stands for "Data Transfer Object" */ -public abstract class AbstractDhRepo> +public abstract class AbstractDhRepo> implements AutoCloseable { public static final int TIMEOUT_SECONDS = 30; @@ -379,6 +379,7 @@ public abstract class AbstractDhRepo> } } + @Override public void close() { try @@ -391,9 +392,17 @@ public abstract class AbstractDhRepo> { if(this.connection != null) { - LOGGER.info("Closing database connection ["+this.connectionString+"]"); CONNECTIONS_BY_CONNECTION_STRING.remove(this.connectionString); - this.connection.close(); + + if (!this.connection.isClosed()) + { + LOGGER.info("Closing database connection: [" + this.connectionString + "]"); + this.connection.close(); + } + else + { + LOGGER.warn("Attempting to close already closed database connection: [" + this.connectionString + "]"); + } } ACTIVE_CONNECTION_STRINGS_BY_REPO.remove(this); } From 35681fe9a54f750f11086cf587a41a3292c8164d Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 10 Apr 2024 07:11:23 -0500 Subject: [PATCH 150/183] Fix Z-fighting at very high heights --- .../render/renderer/LodRenderProgram.java | 8 +++- .../distanthorizons/core/util/RenderUtil.java | 37 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderProgram.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderProgram.java index d044b149d..96d106c38 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderProgram.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderProgram.java @@ -191,7 +191,13 @@ public class LodRenderProgram extends ShaderProgram implements IDhApiShaderProgr // this is to try and allow the fragment culling to go farther than the near clip plane. // Currently this only works for certain FOV/screen ratio combos. dhNearClipDistance *= 2.0f; - setUniform(clipDistanceUniform, dhNearClipDistance); + // if the player is very high up and the near clip plane has been modified, disable the distance clipping + // we're high enough that nothing will render on top of the player and this can cause issues otherwise + if (RenderUtil.getHeightBasedNearClipOverride() != -1) + { + dhNearClipDistance = 1.0f; + } + this.setUniform(this.clipDistanceUniform, dhNearClipDistance); } @Override diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderUtil.java index 0ea8414ca..05fb69b54 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/RenderUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/RenderUtil.java @@ -28,6 +28,7 @@ import com.seibel.distanthorizons.core.pos.DhChunkPos; import com.seibel.distanthorizons.core.world.IDhClientWorld; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper; +import com.seibel.distanthorizons.core.wrapperInterfaces.world.IClientLevelWrapper; import com.seibel.distanthorizons.coreapi.util.MathUtil; import com.seibel.distanthorizons.coreapi.util.math.Mat4f; import com.seibel.distanthorizons.coreapi.util.math.Vec3f; @@ -148,11 +149,6 @@ public class RenderUtil // in James' testing a near clip plane distance of 2 blocks is enough to allow the fragment // culling to take effect instead of seeing the near clip plane. float nearClipDist = RenderUtil.getNearClipPlaneDistanceInBlocks(partialTicks); - if (Config.Client.Advanced.Debugging.lodOnlyMode.get()) - { - nearClipDist = 0.1f; - } - float farClipDist = (float) RenderUtil.getFarClipPlaneDistanceInBlocks(); // Create a copy of the current matrix, so it won't be modified. @@ -210,6 +206,15 @@ public class RenderUtil } } + + // TODO move into method and use to override discard value in shader program + float heightOverride = getHeightBasedNearClipOverride(); + if (heightOverride != -1.0f) + { + nearClipPlane = heightOverride; + } + + // modify based on the player's FOV double fov = MC_RENDER.getFov(partialTicks); double aspectRatio = (double) MC_RENDER.getScreenWidth() / MC_RENDER.getScreenHeight(); @@ -227,6 +232,28 @@ public class RenderUtil return (int)((lodBlockDist + LodUtil.REGION_WIDTH) * Math.sqrt(2)); } + /** @return -1 if no override is necessary */ + public static float getHeightBasedNearClipOverride() + { + // TODO always using the client level like this might cause issues with immersive portals and the like, + // but for now it should work well enough + IClientLevelWrapper level = MC.getWrappedClientLevel(); + // a level should always be loaded, but just in case + if (level != null) + { + // if the player is a significant distance above the work, increase the + // near clip plane to fix Z imprecision issues + int playerHeight = MC.getPlayerBlockPos().y; + int levelMaxHeight = level.getHeight(); + if (playerHeight > levelMaxHeight + 1_000) + { + return playerHeight - (levelMaxHeight + 1000); + } + } + + return -1.0f; + } + /** @return false if LODs shouldn't be rendered for any reason */ public static boolean shouldLodsRender(ILevelWrapper levelWrapper) { From 6d0ec3331660c1fd0c8e2e28c876298092321a61 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Wed, 10 Apr 2024 18:51:14 -0500 Subject: [PATCH 151/183] fix minor line spacing --- .../bufferBuilding/ColumnRenderBufferBuilder.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index 7ff76dfdc..fd7b7b486 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -89,13 +89,13 @@ public class ColumnRenderBufferBuilder //EVENT_LOGGER.trace("RenderRegion start QuadBuild @ " + renderSource.sectionPos); boolean enableSkyLightCulling = ( - // dimensions with a ceiling will be all caves so we don't want cave culling - !clientLevel.getLevelWrapper().hasCeiling() - // the end has a lot of overhangs with 0 lighting above the void, which look broken with - // the current cave culling logic (this could probably be improved, but just skipping it works best for now) - && !clientLevel.getLevelWrapper().getDimensionType().isTheEnd() + // dimensions with a ceiling will be all caves so we don't want cave culling + !clientLevel.getLevelWrapper().hasCeiling() + // the end has a lot of overhangs with 0 lighting above the void, which look broken with + // the current cave culling logic (this could probably be improved, but just skipping it works best for now) + && !clientLevel.getLevelWrapper().getDimensionType().isTheEnd() ) - && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); + && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); int skyLightCullingBelow = Config.Client.Advanced.Graphics.AdvancedGraphics.caveCullingHeight.get(); // FIXME: Clamp also to the max world height. From a216ba364db3ab3a7932bcb001bf9a46ad353056 Mon Sep 17 00:00:00 2001 From: cola98765 Date: Thu, 11 Apr 2024 09:43:05 +0000 Subject: [PATCH 152/183] hopefully make old file conversion a bit faster. --- .../fullData/sources/FullDataSourceV2.java | 16 ++++++---------- .../core/util/FullDataPointUtil.java | 4 ++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index bbac771da..c849c24c4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -174,25 +174,21 @@ public class FullDataSourceV2 implements IDataSource { long dataPoint = legacyDataColumn[i]; - int id = FullDataPointUtil.getId(dataPoint); - int height = FullDataPointUtil.getHeight(dataPoint); - int bottomY = FullDataPointUtil.getBottomY(dataPoint); + boolean isAir = legacyData.mapping.getBlockStateWrapper(FullDataPointUtil.getId(dataPoint)).isAir; byte blockLight = (byte) FullDataPointUtil.getBlockLight(dataPoint); - byte skyLight = (byte) FullDataPointUtil.getSkyLight(dataPoint); - - IBlockStateWrapper blockState = legacyData.mapping.getBlockStateWrapper(id); - if (blockState.isAir()) + + if (isAir) { // air shouldn't have any light, otherwise down sampling will look weird blockLight = 0; } - long newDataPoint = FullDataPointUtil.encode(id, height, bottomY, blockLight, skyLight); - newDataColumn.set(i, newDataPoint); + long dataPoint = FullDataPointUtil.setBlockLight(dataPoint, blockLight); + newDataColumn.set(i, dataPoint); // check if this datapoint is air - if (!columnHasNonAirBlock && !blockState.isAir()) + if (!columnHasNonAirBlock && !isAir) { columnHasNonAirBlock = true; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index a1eb81042..d0daabab5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -127,6 +127,10 @@ public class FullDataPointUtil */ public static int getBottomY(long data) { return (int) ((data >> MIN_Y_OFFSET) & MIN_Y_MASK); } public static int getBlockLight(long data) { return (int) ((data >> BLOCK_LIGHT_OFFSET) & BLOCK_LIGHT_MASK); } + public static long setBlockLight(long data, byte blockLight) + { + return (data & ~((long) BLOCK_LIGHT_MASK << BLOCK_LIGHT_OFFSET) | (long) blockLight << BLOCK_LIGHT_OFFSET); + } public static int getSkyLight(long data) { return (int) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } From f8b127a85dfc1d2351ad2e4a69b4b060f34c3803 Mon Sep 17 00:00:00 2001 From: cola98765 Date: Thu, 11 Apr 2024 09:45:30 +0000 Subject: [PATCH 153/183] I may not be smart --- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index c849c24c4..0094ac3e3 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -183,7 +183,7 @@ public class FullDataSourceV2 implements IDataSource blockLight = 0; } - long dataPoint = FullDataPointUtil.setBlockLight(dataPoint, blockLight); + dataPoint = FullDataPointUtil.setBlockLight(dataPoint, blockLight); newDataColumn.set(i, dataPoint); From b5d938475ae5c690af969547d5c5661dda20f662 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Fri, 12 Apr 2024 22:19:28 -0500 Subject: [PATCH 154/183] Optimize LodRenderSection loading and data caching --- .../core/render/LodRenderSection.java | 353 +++++++++++------- 1 file changed, 221 insertions(+), 132 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 2720fe020..7d2d8fbd6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -33,7 +33,6 @@ import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable; import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer; import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; -import com.seibel.distanthorizons.core.util.TimerUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; @@ -41,9 +40,10 @@ import javax.annotation.WillNotClose; import java.awt.*; import java.util.*; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; /** * A render section represents an area that could be rendered. @@ -53,17 +53,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); - /** - * Only the adjacent render sources should be cached to prevent accidentally using cached data when the LOD data was changed. - * This cache should really only be used when initially loading LODs or generating new terrain. - */ - private static final ConcurrentHashMap ADJACENT_RENDER_SOURCE_BY_POS = new ConcurrentHashMap<>(); - - private static final ConcurrentHashMap RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS = new ConcurrentHashMap<>(); - private static final Timer RENDER_SOURCE_CACHE_REMOVAL_TIMER = TimerUtil.CreateTimer("LodRenderSection Render Source Cache Removal Timer"); - - public static final long RENDER_CACHE_EXPIRATION_TIME_IN_MS = 4000L; - public final DhSectionPos pos; @@ -71,16 +60,25 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable private final IDhClientLevel level; @WillNotClose private final FullDataSourceProviderV2 fullDataSourceProvider; + private final LodQuadTree quadTree; public boolean renderingEnabled = false; + private boolean canRender = false; /** this reference is necessary so we can determine what VBO to render */ public ColumnRenderBuffer renderBuffer; - private CompletableFuture renderSourceLoadingFuture = null; - private boolean canRender = false; + /** + * Encapsulates everything between pulling data from the database (including neighbors) + * up to the point when geometry data is uploaded to the GPU. + */ + private CompletableFuture uploadRenderDataToGpuFuture = null; + + private final ReentrantLock getRenderSourceLock = new ReentrantLock(); + /** Used to track this position's render data loading */ + private ReferenceCountingFutureWrapper renderSourceLoadingFuture = null; private boolean missingPositionsCalculated = false; /** should be an empty array if no positions need to be generated */ @@ -92,9 +90,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // constructor // //=============// - public LodRenderSection(DhSectionPos pos, IDhClientLevel level, FullDataSourceProviderV2 fullDataSourceProvider) + public LodRenderSection(DhSectionPos pos, LodQuadTree quadTree, IDhClientLevel level, FullDataSourceProviderV2 fullDataSourceProvider) { this.pos = pos; + this.quadTree = quadTree; this.level = level; this.fullDataSourceProvider = fullDataSourceProvider; @@ -103,11 +102,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable - //=============// - // render data // - //=============// + //===============================// + // render data loading/uploading // + //===============================// - public void loadRenderSourceAsync() + public void uploadRenderDataToGpuAsync() { if (!GLProxy.hasInstance()) { @@ -116,148 +115,183 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable return; } - if (this.renderSourceLoadingFuture != null) + if (this.uploadRenderDataToGpuFuture != null) { + // don't accidentally queue multiple uploads at the same time return; } - // run on the file handler pool since a number of operations - // require a number of database hits ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { return; } - - this.renderSourceLoadingFuture = CompletableFuture.runAsync(() -> + this.uploadRenderDataToGpuFuture = CompletableFuture.runAsync(() -> { - FullDataSourceV2 fullDataSource = null; - ColumnRenderSource renderSource = null; - - try + ReferenceCountingFutureWrapper thisRenderSourceFutureWrapper = this.getRenderSourceAsync(); + thisRenderSourceFutureWrapper.future.thenAccept((renderSource) -> { - // get this positions data source - fullDataSource = this.fullDataSourceProvider.get(this.pos); - if (fullDataSource == null) - { - // the file handler is being shut down, we won't be rendering anything anyway - return; - } - - renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - if (renderSource.isEmpty()) - { - // nothing needs to be rendered - this.canRender = false; - return; - } - - - ColumnRenderSource[] adjacentRenderSections = this.getAndCreateNeighborRenderSources(); - - ColumnRenderBuffer previousBuffer = this.renderBuffer; - - CompletableFuture uploadFuture = ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections); - this.renderBuffer = uploadFuture.join(); - - if (previousBuffer != null) - { - previousBuffer.close(); - } - - this.canRender = true; - } - catch (Exception e) - { - LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); - } - finally - { - // clean up pooled data sources try { - if (fullDataSource != null) + if (renderSource == null || renderSource.isEmpty()) { - fullDataSource.close(); + // nothing needs to be rendered + this.canRender = false; + thisRenderSourceFutureWrapper.decrementRefCount(); + return; } - if (renderSource != null) + + + //=================================// + // get the neighbor render sources // + //=================================// + + ReferenceCountingFutureWrapper[] adjacentLoadFutureWrappers = this.getNeighborRenderSourcesAsync(); + CompletableFuture[] adjacentLoadFutures = new CompletableFuture[adjacentLoadFutureWrappers.length]; + for (int i = 0; i < adjacentLoadFutureWrappers.length; i++) { - renderSource.close(); + adjacentLoadFutures[i] = adjacentLoadFutureWrappers[i].future; } + + + + //==============================// + // build/upload new render data // + //==============================// + + CompletableFuture.allOf(adjacentLoadFutures).thenRun(() -> + { + try + { + ColumnRenderBuffer previousBuffer = this.renderBuffer; + + ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; + for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) + { + adjacentRenderSections[i] = adjacentLoadFutureWrappers[i].future.getNow(null); + } + ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections).thenAccept((buffer) -> + { + // upload complete, clean up the old data if + this.renderBuffer = buffer; + this.canRender = true; + this.uploadRenderDataToGpuFuture = null; + + + + //=========// + // cleanup // + //=========// + + // the old buffer isn't needed anymore + if (previousBuffer != null) + { + previousBuffer.close(); + } + + // if these render sources aren't needed anymore they can be put back in the pool + try + { + thisRenderSourceFutureWrapper.decrementRefCount(); + + for (ReferenceCountingFutureWrapper adjLoadFutureWrapper : adjacentLoadFutureWrappers) + { + adjLoadFutureWrapper.decrementRefCount(); + } + } + catch (Exception ignore) { } + }); + } + catch (Exception e) + { + LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); + this.uploadRenderDataToGpuFuture = null; + } + }); } - catch (Exception ignore){ } - - this.renderSourceLoadingFuture = null; - } + catch (Exception e) + { + LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); + this.uploadRenderDataToGpuFuture = null; + } + }); }, executor); } /** Should be called on the {@link ThreadPoolUtil#getFileHandlerExecutor()} */ - private ColumnRenderSource[] getAndCreateNeighborRenderSources() + private ReferenceCountingFutureWrapper[] getNeighborRenderSourcesAsync() { - ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; - //for (EDhDirection direction : EDhDirection.ADJ_DIRECTIONS) - //{ - // DhSectionPos adjPos = this.pos.getAdjacentPos(direction); - // - // ColumnRenderSource renderSource = ADJACENT_RENDER_SOURCE_BY_POS.compute(adjPos, this::computeCachedRenderSource); - // adjacentRenderSections[direction.ordinal() - 2] = renderSource; - //} - - return adjacentRenderSections; - } - private ColumnRenderSource computeCachedRenderSource(DhSectionPos pos, ColumnRenderSource oldRenderSource) - { - try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(pos)) + ReferenceCountingFutureWrapper[] futureArray = new ReferenceCountingFutureWrapper[EDhDirection.ADJ_DIRECTIONS.length]; + for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { - // use the old render source if it isn't null - if (oldRenderSource == null) - { - oldRenderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - } - - // create a new timer task to reset the cache timeout - TimerTask timerTask = RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS.compute(pos, (timerPos, oldTimerTask) -> - { - if (oldTimerTask != null) - { - oldTimerTask.cancel(); - } - - return new TimerTask() - { - @Override - public void run() - { - // remove the finished task - RENDER_SOURCE_CLOSING_TIMER_TASK_BY_POS.remove(pos); - - // return the pooled data source if present - ColumnRenderSource expiredRenderSource = ADJACENT_RENDER_SOURCE_BY_POS.remove(pos); - if (expiredRenderSource != null) - { - try { expiredRenderSource.close(); } catch (Exception ignored) { } - } - - //LOGGER.info("cache size " +cachedNeighborSections.size()+" pool size:"+ColumnRenderSource.DATA_SOURCE_POOL.size()); - } - }; - }); + EDhDirection direction = EDhDirection.ADJ_DIRECTIONS[i]; + DhSectionPos adjPos = this.pos.getAdjacentPos(direction); try { - RENDER_SOURCE_CACHE_REMOVAL_TIMER.schedule(timerTask, RENDER_CACHE_EXPIRATION_TIME_IN_MS); + LodRenderSection adjRenderSection = this.quadTree.getValue(adjPos); + if (adjRenderSection != null) + { + futureArray[i] = adjRenderSection.getRenderSourceAsync(); + } } - catch (IllegalStateException ignore) { /* can rarely happen due to some minor concurrency bug with how Timer works. It isn't an issue and can be ignored. */ } + catch (IndexOutOfBoundsException ignore) {} - return oldRenderSource; + if (futureArray[i] == null) + { + futureArray[i] = new ReferenceCountingFutureWrapper(CompletableFuture.completedFuture(null)); + } } - catch (Exception e) + + return futureArray; + } + /** Will try to return the same {@link CompletableFuture} if multiple requests are made for the same position */ + private ReferenceCountingFutureWrapper getRenderSourceAsync() + { + try { - LOGGER.warn("Unable to get neighbor render source " + this.pos + " - " + pos + ", error: " + e.getMessage(), e); - return null; + this.getRenderSourceLock.lock(); + + + // if a load is already in progress, use that existing one + // (this reduces the number of duplicate loads that may happen when initially loading the world) + if (this.renderSourceLoadingFuture != null) + { + this.renderSourceLoadingFuture.incrementRefCount(); + return this.renderSourceLoadingFuture; + } + + + + ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); + if (executor == null || executor.isTerminated()) + { + return new ReferenceCountingFutureWrapper(CompletableFuture.completedFuture(null)); + } + + CompletableFuture future = CompletableFuture.supplyAsync(() -> + { + try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(this.pos)) + { + ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); + this.renderSourceLoadingFuture = null; + return renderSource; + } + catch (Exception e) + { + LOGGER.warn("Unable to get render source " + this.pos + ", error: " + e.getMessage(), e); + this.renderSourceLoadingFuture = null; + return null; + } + }, executor); + this.renderSourceLoadingFuture = new ReferenceCountingFutureWrapper(future); + return this.renderSourceLoadingFuture; + } + finally + { + this.getRenderSourceLock.unlock(); } } @@ -269,7 +303,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable public boolean canRender() { return this.canRender; } - public boolean loadingRenderSource() { return this.renderSourceLoadingFuture != null; } + public boolean gpuUploadInProgress() { return this.uploadRenderDataToGpuFuture != null; } @@ -344,9 +378,14 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable this.renderBuffer.close(); } + if (this.uploadRenderDataToGpuFuture != null) + { + this.uploadRenderDataToGpuFuture.cancel(true); + } + if (this.renderSourceLoadingFuture != null) { - this.renderSourceLoadingFuture.cancel(true); + this.renderSourceLoadingFuture.future.cancel(true); } @@ -374,7 +413,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { color = Color.green; } - else if (this.renderSourceLoadingFuture != null) + else if (this.uploadRenderDataToGpuFuture != null) { color = Color.yellow; } @@ -386,4 +425,54 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 400, 8f, Objects.hashCode(this), 0.1f, color)); } + + + //================// + // helper classes // + //================// + + /** + * Used to keep track of how many references a given {@link ColumnRenderSource} + * has, so it can be closed when it's no longer needed.

+ * + * If the reference counting isn't perfect that's ok. + * This just optimizes loading by putting finished {@link ColumnRenderSource} + * back into the pool to reduce GC overhead, no data will be leaked if they aren't closed. + */ + private static class ReferenceCountingFutureWrapper + { + public final CompletableFuture future; + private final AtomicInteger referenceCount = new AtomicInteger(1); + + public ReferenceCountingFutureWrapper(CompletableFuture future) + { + this.future = future; + } + + + public void incrementRefCount() { this.referenceCount.incrementAndGet(); } + public void decrementRefCount() + { + if (this.referenceCount.decrementAndGet() <= 0) + { + try + { + // this logic assumes that the data source has finished loading + // if it hasn't finished loading it will just be garbage collected + ColumnRenderSource renderSource = this.future.getNow(null); + if (renderSource != null) + { + renderSource.close(); + } + } + catch (Exception e) + { + LOGGER.error(e.getMessage(), e); + } + } + } + + } + + } From 07b495d48c4319964db905b71d974e1e6ad5cf12 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Fri, 12 Apr 2024 22:19:44 -0500 Subject: [PATCH 155/183] Increase file handler thread pool default runtime ratio --- .../presets/ThreadPresetConfigEventHandler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java index 134465c08..c6bbf1120 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/eventHandlers/presets/ThreadPresetConfigEventHandler.java @@ -73,13 +73,13 @@ public class ThreadPresetConfigEventHandler extends AbstractPresetConfigEventHan this.put(EDhApiThreadPreset.AGGRESSIVE, getThreadCountByPercent(0.2)); this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, getThreadCountByPercent(1.0)); }}); - public static double getFileHandlerDefaultRunTimeRatio() { return 0.5; } + public static double getFileHandlerDefaultRunTimeRatio() { return 0.75; } private final ConfigEntryWithPresetOptions fileHandlerRunTime = new ConfigEntryWithPresetOptions<>(Config.Client.Advanced.MultiThreading.runTimeRatioForFileHandlerThreads, new HashMap() {{ - this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.25); + this.put(EDhApiThreadPreset.MINIMAL_IMPACT, 0.50); this.put(EDhApiThreadPreset.LOW_IMPACT, getFileHandlerDefaultRunTimeRatio()); - this.put(EDhApiThreadPreset.BALANCED, 0.75); + this.put(EDhApiThreadPreset.BALANCED, 1.0); this.put(EDhApiThreadPreset.AGGRESSIVE, 1.0); this.put(EDhApiThreadPreset.I_PAID_FOR_THE_WHOLE_CPU, 1.0); }}); From f7c5b5725f4c7254e330e312d664d77088199454 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Fri, 12 Apr 2024 22:39:32 -0500 Subject: [PATCH 156/183] Fix LodRenderSection adjacent source indicies --- .../distanthorizons/core/render/LodRenderSection.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 7d2d8fbd6..1c3e7a00c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -228,20 +228,22 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { EDhDirection direction = EDhDirection.ADJ_DIRECTIONS[i]; + int arrayIndex = direction.ordinal() - 2; + DhSectionPos adjPos = this.pos.getAdjacentPos(direction); try { LodRenderSection adjRenderSection = this.quadTree.getValue(adjPos); if (adjRenderSection != null) { - futureArray[i] = adjRenderSection.getRenderSourceAsync(); + futureArray[arrayIndex] = adjRenderSection.getRenderSourceAsync(); } } catch (IndexOutOfBoundsException ignore) {} - if (futureArray[i] == null) + if (futureArray[arrayIndex] == null) { - futureArray[i] = new ReferenceCountingFutureWrapper(CompletableFuture.completedFuture(null)); + futureArray[arrayIndex] = new ReferenceCountingFutureWrapper(CompletableFuture.completedFuture(null)); } } From cba55aa643b35e464aea1ea654e3e16a2d6d56b7 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Fri, 12 Apr 2024 22:40:08 -0500 Subject: [PATCH 157/183] Improve LOD load order --- .../core/render/LodQuadTree.java | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 6ade073f2..4ab22746a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -163,7 +163,7 @@ public class LodQuadTree extends QuadTree implements AutoClose LodRenderSection renderSection = this.getValue(pos); if (renderSection != null && renderSection.renderingEnabled) { - renderSection.loadRenderSourceAsync(); + renderSection.uploadRenderDataToGpuAsync(); } } catch (IndexOutOfBoundsException e) @@ -176,6 +176,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // walk through each root node ArrayList nodesNeedingRetrieval = new ArrayList<>(); + ArrayList nodesNeedingLoading = new ArrayList<>(); Iterator rootPosIterator = this.rootNodePosIterator(); while (rootPosIterator.hasNext()) { @@ -183,11 +184,11 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos rootPos = rootPosIterator.next(); if (this.getNode(rootPos) == null) { - this.setValue(rootPos, new LodRenderSection(rootPos, this.level, this.fullDataSourceProvider)); + this.setValue(rootPos, new LodRenderSection(rootPos, this, this.level, this.fullDataSourceProvider)); } QuadNode rootNode = this.getNode(rootPos); - this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingRetrieval); + this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, rootNode, rootNode.sectionPos, false, nodesNeedingRetrieval, nodesNeedingLoading); } @@ -197,13 +198,32 @@ public class LodQuadTree extends QuadTree implements AutoClose this.fullDataRetrievalQueueRunning.set(true); FULL_DATA_RETRIEVAL_QUEUE_THREAD.execute(() -> this.queueFullDataRetrievalTasks(playerPos, nodesNeedingRetrieval)); } + + + nodesNeedingLoading.sort((a, b) -> + { + int aDist = a.pos.getManhattanBlockDistance(playerPos); + int bDist = b.pos.getManhattanBlockDistance(playerPos); + return Integer.compare(aDist, bDist); + }); + + for (int i = 0; i < nodesNeedingLoading.size(); i++) + { + LodRenderSection renderSection = nodesNeedingLoading.get(i); + if (!renderSection.gpuUploadInProgress() && renderSection.renderBuffer == null) + { + renderSection.uploadRenderDataToGpuAsync(); + } + } + } /** @return whether the current position is able to render (note: not if it IS rendering, just if it is ABLE to.) */ private boolean recursivelyUpdateRenderSectionNode( DhBlockPos2D playerPos, QuadNode rootNode, QuadNode quadNode, DhSectionPos sectionPos, boolean parentRenderSectionIsEnabled, - ArrayList nodesNeedingRetrieval) + ArrayList nodesNeedingRetrieval, + ArrayList nodesNeedingLoading) { //===============================// // node and render section setup // @@ -212,7 +232,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // make sure the node is created if (quadNode == null && this.isSectionPosInBounds(sectionPos)) // the position bounds should only fail when at the edge of the user's render distance { - rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this.level, this.fullDataSourceProvider)); + rootNode.setValue(sectionPos, new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider)); quadNode = rootNode.getNode(sectionPos); } if (quadNode == null) @@ -226,7 +246,7 @@ public class LodQuadTree extends QuadTree implements AutoClose // create a new render section if missing if (renderSection == null) { - LodRenderSection newRenderSection = new LodRenderSection(sectionPos, this.level, this.fullDataSourceProvider); + LodRenderSection newRenderSection = new LodRenderSection(sectionPos, this, this.level, this.fullDataSourceProvider); rootNode.setValue(sectionPos, newRenderSection); renderSection = newRenderSection; // TODO this never seemed to be called, is it necessary? @@ -258,7 +278,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled, nodesNeedingRetrieval); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled, nodesNeedingRetrieval, nodesNeedingLoading); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } @@ -279,7 +299,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled, nodesNeedingRetrieval); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled, nodesNeedingRetrieval, nodesNeedingLoading); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } if (!allChildrenSectionsAreLoaded) @@ -310,9 +330,9 @@ public class LodQuadTree extends QuadTree implements AutoClose { // prepare this section for rendering // TODO this should fire for the lowest detail level first to improve loading speed - if (!renderSection.loadingRenderSource() && renderSection.renderBuffer == null) + if (!renderSection.gpuUploadInProgress() && renderSection.renderBuffer == null) { - renderSection.loadRenderSourceAsync(); + nodesNeedingLoading.add(renderSection); } // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections From 8cecdebc056830a02a363017df619e87f2cfb4fb Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 07:40:15 -0500 Subject: [PATCH 158/183] Disable cave culling for any detail level above 0 --- .../bufferBuilding/ColumnRenderBufferBuilder.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java index fd7b7b486..aad029d2b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java @@ -27,6 +27,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.ConfigBasedLogger; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos; +import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.render.glObject.GLProxy; import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer; import com.seibel.distanthorizons.core.util.LodUtil; @@ -88,14 +89,17 @@ public class ColumnRenderBufferBuilder //EVENT_LOGGER.trace("RenderRegion start QuadBuild @ " + renderSource.sectionPos); boolean enableSkyLightCulling = - ( + Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get() + && ( // dimensions with a ceiling will be all caves so we don't want cave culling !clientLevel.getLevelWrapper().hasCeiling() // the end has a lot of overhangs with 0 lighting above the void, which look broken with // the current cave culling logic (this could probably be improved, but just skipping it works best for now) && !clientLevel.getLevelWrapper().getDimensionType().isTheEnd() - ) - && Config.Client.Advanced.Graphics.AdvancedGraphics.enableCaveCulling.get(); + // FIXME temporary fix + // Cave culling is currently broken for any detail level above 0 + && renderSource.pos.getDetailLevel() == DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + ); int skyLightCullingBelow = Config.Client.Advanced.Graphics.AdvancedGraphics.caveCullingHeight.get(); // FIXME: Clamp also to the max world height. From eafaf2b4cd884217eb435bf126682292670a756e Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 07:42:32 -0500 Subject: [PATCH 159/183] Fix compiler error in FullDataSourceV2 and minor reformat --- .../core/dataObjects/fullData/sources/FullDataSourceV2.java | 2 +- .../seibel/distanthorizons/core/util/FullDataPointUtil.java | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 0094ac3e3..1fd66bb9d 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -174,7 +174,7 @@ public class FullDataSourceV2 implements IDataSource { long dataPoint = legacyDataColumn[i]; - boolean isAir = legacyData.mapping.getBlockStateWrapper(FullDataPointUtil.getId(dataPoint)).isAir; + boolean isAir = legacyData.mapping.getBlockStateWrapper(FullDataPointUtil.getId(dataPoint)).isAir(); byte blockLight = (byte) FullDataPointUtil.getBlockLight(dataPoint); if (isAir) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java index d0daabab5..b63c6f0e7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/FullDataPointUtil.java @@ -127,12 +127,9 @@ public class FullDataPointUtil */ public static int getBottomY(long data) { return (int) ((data >> MIN_Y_OFFSET) & MIN_Y_MASK); } public static int getBlockLight(long data) { return (int) ((data >> BLOCK_LIGHT_OFFSET) & BLOCK_LIGHT_MASK); } - public static long setBlockLight(long data, byte blockLight) - { - return (data & ~((long) BLOCK_LIGHT_MASK << BLOCK_LIGHT_OFFSET) | (long) blockLight << BLOCK_LIGHT_OFFSET); - } public static int getSkyLight(long data) { return (int) ((data >> SKY_LIGHT_OFFSET) & SKY_LIGHT_MASK); } + public static long setBlockLight(long data, byte blockLight) { return (data & ~((long) BLOCK_LIGHT_MASK << BLOCK_LIGHT_OFFSET) | (long) blockLight << BLOCK_LIGHT_OFFSET); } public static String toString(long data) { return "[ID:" + getId(data) + ",Y:" + getBottomY(data) + ",Height:" + getHeight(data) + ",BlockLight:" + getBlockLight(data) + ",SkyLight:" + getSkyLight(data) + "]"; } From 22422321abd404578d46446c94baeeb05d782c71 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 08:22:36 -0500 Subject: [PATCH 160/183] Remove unhelpful data source pooling in LodRenderSection --- .../core/render/LodRenderSection.java | 104 ++---------------- 1 file changed, 11 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 1c3e7a00c..76b0282f5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -78,7 +78,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable private final ReentrantLock getRenderSourceLock = new ReentrantLock(); /** Used to track this position's render data loading */ - private ReferenceCountingFutureWrapper renderSourceLoadingFuture = null; + private CompletableFuture renderSourceLoadingFuture = null; private boolean missingPositionsCalculated = false; /** should be an empty array if no positions need to be generated */ @@ -131,8 +131,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable this.uploadRenderDataToGpuFuture = CompletableFuture.runAsync(() -> { - ReferenceCountingFutureWrapper thisRenderSourceFutureWrapper = this.getRenderSourceAsync(); - thisRenderSourceFutureWrapper.future.thenAccept((renderSource) -> + this.getRenderSourceAsync().thenAccept((renderSource) -> { try { @@ -140,7 +139,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { // nothing needs to be rendered this.canRender = false; - thisRenderSourceFutureWrapper.decrementRefCount(); return; } @@ -150,12 +148,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // get the neighbor render sources // //=================================// - ReferenceCountingFutureWrapper[] adjacentLoadFutureWrappers = this.getNeighborRenderSourcesAsync(); - CompletableFuture[] adjacentLoadFutures = new CompletableFuture[adjacentLoadFutureWrappers.length]; - for (int i = 0; i < adjacentLoadFutureWrappers.length; i++) - { - adjacentLoadFutures[i] = adjacentLoadFutureWrappers[i].future; - } + CompletableFuture[] adjacentLoadFutures = this.getNeighborRenderSourcesAsync(); @@ -172,7 +165,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { - adjacentRenderSections[i] = adjacentLoadFutureWrappers[i].future.getNow(null); + adjacentRenderSections[i] = adjacentLoadFutures[i].getNow(null); } ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections).thenAccept((buffer) -> { @@ -181,29 +174,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable this.canRender = true; this.uploadRenderDataToGpuFuture = null; - - - //=========// - // cleanup // - //=========// - - // the old buffer isn't needed anymore - if (previousBuffer != null) - { - previousBuffer.close(); - } - - // if these render sources aren't needed anymore they can be put back in the pool - try - { - thisRenderSourceFutureWrapper.decrementRefCount(); - - for (ReferenceCountingFutureWrapper adjLoadFutureWrapper : adjacentLoadFutureWrappers) - { - adjLoadFutureWrapper.decrementRefCount(); - } - } - catch (Exception ignore) { } }); } catch (Exception e) @@ -222,9 +192,9 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable }, executor); } /** Should be called on the {@link ThreadPoolUtil#getFileHandlerExecutor()} */ - private ReferenceCountingFutureWrapper[] getNeighborRenderSourcesAsync() + private CompletableFuture[] getNeighborRenderSourcesAsync() { - ReferenceCountingFutureWrapper[] futureArray = new ReferenceCountingFutureWrapper[EDhDirection.ADJ_DIRECTIONS.length]; + CompletableFuture[] futureArray = new CompletableFuture[EDhDirection.ADJ_DIRECTIONS.length]; for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { EDhDirection direction = EDhDirection.ADJ_DIRECTIONS[i]; @@ -243,14 +213,14 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable if (futureArray[arrayIndex] == null) { - futureArray[arrayIndex] = new ReferenceCountingFutureWrapper(CompletableFuture.completedFuture(null)); + futureArray[arrayIndex] = CompletableFuture.completedFuture(null); } } return futureArray; } /** Will try to return the same {@link CompletableFuture} if multiple requests are made for the same position */ - private ReferenceCountingFutureWrapper getRenderSourceAsync() + private CompletableFuture getRenderSourceAsync() { try { @@ -261,7 +231,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // (this reduces the number of duplicate loads that may happen when initially loading the world) if (this.renderSourceLoadingFuture != null) { - this.renderSourceLoadingFuture.incrementRefCount(); return this.renderSourceLoadingFuture; } @@ -270,10 +239,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { - return new ReferenceCountingFutureWrapper(CompletableFuture.completedFuture(null)); + return CompletableFuture.completedFuture(null); } - CompletableFuture future = CompletableFuture.supplyAsync(() -> + this.renderSourceLoadingFuture = CompletableFuture.supplyAsync(() -> { try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(this.pos)) { @@ -288,7 +257,6 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable return null; } }, executor); - this.renderSourceLoadingFuture = new ReferenceCountingFutureWrapper(future); return this.renderSourceLoadingFuture; } finally @@ -387,7 +355,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable if (this.renderSourceLoadingFuture != null) { - this.renderSourceLoadingFuture.future.cancel(true); + this.renderSourceLoadingFuture.cancel(true); } @@ -427,54 +395,4 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 400, 8f, Objects.hashCode(this), 0.1f, color)); } - - - //================// - // helper classes // - //================// - - /** - * Used to keep track of how many references a given {@link ColumnRenderSource} - * has, so it can be closed when it's no longer needed.

- * - * If the reference counting isn't perfect that's ok. - * This just optimizes loading by putting finished {@link ColumnRenderSource} - * back into the pool to reduce GC overhead, no data will be leaked if they aren't closed. - */ - private static class ReferenceCountingFutureWrapper - { - public final CompletableFuture future; - private final AtomicInteger referenceCount = new AtomicInteger(1); - - public ReferenceCountingFutureWrapper(CompletableFuture future) - { - this.future = future; - } - - - public void incrementRefCount() { this.referenceCount.incrementAndGet(); } - public void decrementRefCount() - { - if (this.referenceCount.decrementAndGet() <= 0) - { - try - { - // this logic assumes that the data source has finished loading - // if it hasn't finished loading it will just be garbage collected - ColumnRenderSource renderSource = this.future.getNow(null); - if (renderSource != null) - { - renderSource.close(); - } - } - catch (Exception e) - { - LOGGER.error(e.getMessage(), e); - } - } - } - - } - - } From 4a40e19b3464b9b8eced110e9f0f706d1b78e30b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 09:45:49 -0500 Subject: [PATCH 161/183] suppress unchecked warning in LodRenderSection --- .../com/seibel/distanthorizons/core/render/LodRenderSection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 76b0282f5..71c9a79a1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -192,6 +192,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable }, executor); } /** Should be called on the {@link ThreadPoolUtil#getFileHandlerExecutor()} */ + @SuppressWarnings("unchecked") // creating an array of CompletableFuture's is unchecked, unfortunately I don't currently see a better fix private CompletableFuture[] getNeighborRenderSourcesAsync() { CompletableFuture[] futureArray = new CompletableFuture[EDhDirection.ADJ_DIRECTIONS.length]; From 94304eb055969e1262feba280fcceea96e3bcdd5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 12:12:50 -0500 Subject: [PATCH 162/183] speed up initial loading when DB migration is necessary --- .../FullDataSourceProviderV2.java | 34 ++++++++++++- .../core/sql/repo/FullDataSourceV1Repo.java | 48 +++++++++++++++++++ ...20-sqlite-createFullDataSourceV2Tables.sql | 6 --- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 7a8a7cf57..7844eb84f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -317,10 +317,42 @@ public class FullDataSourceProviderV2 String dimensionName = this.level.getLevelWrapper().getDimensionType().getDimensionName(); LOGGER.info("Attempting to migrate data sources for: ["+dimensionName+"]-["+this.saveDir+"]..."); + + + //============================// + // delete unused data sources // + //============================// + + // this could be done all at once via SQL, + // but doing it in chunks prevents locking the database for long periods of time + long unusedCount = 0; + long totalUnusedCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); + if (totalUnusedCount != 0) + { + LOGGER.info("deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources..."); + + ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); + while (unusedDataPosList.size() != 0) + { + this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); + unusedCount += unusedDataPosList.size(); + unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); + + LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalUnusedCount + "]..."); + } + + LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources."); + } + + + + //=========// + // migrate // + //=========// + int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); LOGGER.info("Found ["+totalCount+"] data sources that need migration."); - ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index 521163695..30dfa21c2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; +import com.seibel.distanthorizons.coreapi.util.StringUtil; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -234,4 +235,51 @@ public class FullDataSourceV1Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( + "select Count(*) as unusedCount from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource'"); + + if (resultMap != null) + { + // Number cast is necessary because the returned number can be an int or long + Number resultNumber = (Number) resultMap.get("unusedCount"); + long count = resultNumber.longValue(); + return count; + } + else + { + return 0; + } + } + + /** Returns single quote surrounded {@link DhSectionPos} serailzed values */ + public ArrayList getUnusedDataSourcePositionStringList(int deleteCount) + { + List> deletePosResultMapList = this.queryDictionary( + "select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource' limit "+deleteCount); + + ArrayList deletePosList = new ArrayList<>(); + for (Map deletePosMap : deletePosResultMapList) + { + String posString = (String) deletePosMap.get("DhSectionPos"); + deletePosList.add("'"+posString+"'"); + } + + return deletePosList; + } + + /** Expects positions to already be surrounded in single quotes */ + public void deleteUnusedLegacyData(ArrayList deletePosList) + { + String sectionPosCsv = StringUtil.join(",", deletePosList); + this.queryDictionaryFirst("delete from " + this.getTableName() + " where DhSectionPos in (" + sectionPosCsv + ")"); + } + } diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql index 007fa7b5b..150b788cf 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql @@ -7,12 +7,6 @@ ALTER TABLE Legacy_FullData_V1 ADD COLUMN MigrationFailed BIT NOT NULL DEFAULT 0 --batch-- --- we only want to convert the level 0 LOD data, the rest can be generated later -delete from Legacy_FullData_V1 -where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0; - ---batch-- - CREATE TABLE FullData ( -- compound primary key DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8 From 169429fe487c3dae44c387af43c4449c9815db04 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 13:35:04 -0500 Subject: [PATCH 163/183] improve client level load logging --- .../seibel/distanthorizons/core/api/internal/ClientApi.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index 48b1bf012..6707f5f90 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -176,7 +176,7 @@ public class ClientApi // can happen on certain multiverse servers return; } - LOGGER.info("Unloading client level [" + level + "]."); + LOGGER.info("Unloading client level [" + level + "]-["+level.getDimensionType().getDimensionName()+"]."); AbstractDhWorld world = SharedApi.getAbstractDhWorld(); if (world != null) @@ -215,7 +215,7 @@ public class ClientApi - LOGGER.info("Loading " + (isServerCommunication ? "Multiverse" : "") + " client level [" + level + "]."); + LOGGER.info("Loading " + (isServerCommunication ? "Multiverse" : "") + " client level [" + level + "]-["+level.getDimensionType().getDimensionName()+"]."); AbstractDhWorld world = SharedApi.getAbstractDhWorld(); if (world != null) From 7d4d899226d5586f70e55db8ae2b7255678e3bdc Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 14:25:28 -0500 Subject: [PATCH 164/183] log migration status in F3 menu and chat --- .../core/api/internal/ClientApi.java | 17 +++++++++++++++++ .../distanthorizons/core/config/Config.java | 7 +++++++ .../fullDatafile/FullDataSourceProviderV1.java | 2 +- .../fullDatafile/FullDataSourceProviderV2.java | 11 +++++++++++ .../core/level/ClientLevelModule.java | 6 ++++++ .../assets/distanthorizons/lang/en_us.json | 2 ++ 6 files changed, 44 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index 6707f5f90..88a365e72 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -78,6 +78,8 @@ public class ClientApi public static final long SPAM_LOGGER_FLUSH_NS = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS); private boolean configOverrideReminderPrinted = false; + private boolean showMigrationMessage = false; + public boolean rendererDisabledBecauseOfExceptions = false; private long lastFlushNanoTime = 0; @@ -470,6 +472,7 @@ public class ClientApi { // logging // + // dev build if (ModInfo.IS_DEV_BUILD && !this.configOverrideReminderPrinted && MC.playerExists()) { this.configOverrideReminderPrinted = true; @@ -478,6 +481,18 @@ public class ClientApi MC.sendChatMessage("Distant Horizons nightly experimental build version [" + ModInfo.VERSION+"]."); MC.sendChatMessage("You are running an unsupported version of Distant Horizons!"); MC.sendChatMessage("Here be dragons!"); + MC.sendChatMessage(""); + } + + // data migration + if (this.showMigrationMessage + && Config.Client.Advanced.LodBuilding.showMigrationChatWarning.get()) + { + this.showMigrationMessage = false; + + MC.sendChatMessage("Old Distant Horizons data is being migrated."); + MC.sendChatMessage("While running LODs may load slowly and DH world gen is disabled."); + MC.sendChatMessage(""); } IProfilerWrapper profiler = MC.getProfiler(); @@ -629,5 +644,7 @@ public class ClientApi } } + // TODO there's probably a better way of handling chat messages + public void showMigrationMessageOnNextFrame() { this.showMigrationMessage = true; } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 391aefa2d..136a6e66b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -805,6 +805,13 @@ public class Config + "") .build(); + public static ConfigEntry showMigrationChatWarning = new ConfigEntry.Builder() + .set(true) + .comment("" + + "Determines if a message should be displayed in the chat when LOD migration starts. \n" + + "") + .build(); + } public static class Multiplayer diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index a994b60e4..9fb3bd9fc 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -148,7 +148,7 @@ public class FullDataSourceProviderV1 { ArrayList dataSourceList = new ArrayList<>(); - ArrayList migrationPosList = ((FullDataSourceV1Repo) this.repo).getPositionsToMigrate(limit); + ArrayList migrationPosList = this.repo.getPositionsToMigrate(limit); for (int i = 0; i < migrationPosList.size(); i++) { DhSectionPos pos = migrationPosList.get(i); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 7844eb84f..da63e4857 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -20,9 +20,11 @@ package com.seibel.distanthorizons.core.file.fullDatafile; import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode; +import com.seibel.distanthorizons.core.api.internal.ClientApi; import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV1; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; +import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -34,6 +36,7 @@ import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO; import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; +import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; @@ -56,6 +59,7 @@ public class FullDataSourceProviderV2 implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + private static final IMinecraftClientWrapper MC = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); protected static final int NUMBER_OF_PARENT_UPDATE_TASKS_PER_THREAD = 50; /** how many parent update tasks can be in the queue at once */ @@ -75,6 +79,9 @@ public class FullDataSourceProviderV2 private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000; + private static boolean migrationMessageShown = false; + + protected final ThreadPoolExecutor migrationThreadPool; /** * Interrupting the migration thread pool doesn't work well and may corrupt the database @@ -356,6 +363,8 @@ public class FullDataSourceProviderV2 ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { + ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + // keep going until every data source has been migrated int progressCount = 0; while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) @@ -430,6 +439,8 @@ public class FullDataSourceProviderV2 this.migrationThreadRunning.set(false); } + public int getMigrationCount() { return this.legacyFileHandler.getDataSourceMigrationCount(); } + //=======================// diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 801a2a256..90accbcde 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -272,6 +272,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); + int migrationCount = this.fullDataSourceProvider.getMigrationCount(); + ArrayList lines = new ArrayList<>(); @@ -282,6 +284,10 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle { lines.add("File Handler [" + dimName + "]"); lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); + if (migrationCount > 0) + { + lines.add(" Legacy Migration #: " + migrationCount); + } lines.add(" Update thread pool tasks: " + updateQueueSize + " (completed: " + updateCompletedTaskSize + ")"); lines.add(" Level Unsaved #: " + this.clientLevel.getUnsavedDataSourceCount()); if (unsavedDataSourceCount != -1) diff --git a/core/src/main/resources/assets/distanthorizons/lang/en_us.json b/core/src/main/resources/assets/distanthorizons/lang/en_us.json index d88fb35c3..d3694ab96 100644 --- a/core/src/main/resources/assets/distanthorizons/lang/en_us.json +++ b/core/src/main/resources/assets/distanthorizons/lang/en_us.json @@ -356,6 +356,8 @@ "Lossy World Compression", "distanthorizons.config.client.advanced.lodBuilding.worldCompression.@tooltip": "How should block data be compressed when creating LOD data? \nThis setting will only affect new or updated LOD data, \nany data already generated when this setting is changed will be \nunaffected until it is modified or re-loaded. \n\nMost Accurate: Merge Same Blocks \nHighest Compression: Visually Equal", + "distanthorizons.config.client.advanced.lodBuilding.showMigrationChatWarning": + "Log Migration In Chat", "distanthorizons.config.client.advanced.multiplayer": From c9a7527bcbc1201981f872a9ada8c87e203bcc6b Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 16:52:09 -0500 Subject: [PATCH 165/183] Improve migration message --- .../com/seibel/distanthorizons/core/api/internal/ClientApi.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index 88a365e72..8f9c48df4 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -491,7 +491,7 @@ public class ClientApi this.showMigrationMessage = false; MC.sendChatMessage("Old Distant Horizons data is being migrated."); - MC.sendChatMessage("While running LODs may load slowly and DH world gen is disabled."); + MC.sendChatMessage("During migration LODs may load slowly and DH world gen is disabled."); MC.sendChatMessage(""); } From 383ab1121d5df38cbd05a8abf90201071e88ce6f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 16:58:33 -0500 Subject: [PATCH 166/183] Fix migration getter SQL --- .../distanthorizons/core/sql/repo/FullDataSourceV1Repo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index 30dfa21c2..d40f2889c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -244,7 +244,7 @@ public class FullDataSourceV1Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( - "select Count(*) as unusedCount from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource'"); + "select Count(*) as unusedCount from "+this.getTableName()+" where DataDetailLevel <> 0 or DataType <> 'CompleteFullDataSource'"); if (resultMap != null) { @@ -263,7 +263,7 @@ public class FullDataSourceV1Repo extends AbstractDhRepo getUnusedDataSourcePositionStringList(int deleteCount) { List> deletePosResultMapList = this.queryDictionary( - "select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource' limit "+deleteCount); + "select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 or DataType <> 'CompleteFullDataSource' limit "+deleteCount); ArrayList deletePosList = new ArrayList<>(); for (Map deletePosMap : deletePosResultMapList) From 63371e8111a7426d1c78f057c285e0f802d28349 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 16:58:52 -0500 Subject: [PATCH 167/183] improve migration deletion logic and add F3 logging --- .../FullDataSourceProviderV1.java | 2 +- .../FullDataSourceProviderV2.java | 77 ++++++++++++++----- .../core/level/ClientLevelModule.java | 8 +- .../core/sql/repo/FullDataSourceV1Repo.java | 5 +- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index 9fb3bd9fc..fde42a6a7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -142,7 +142,7 @@ public class FullDataSourceProviderV1 // migration // //===========// - public int getDataSourceMigrationCount() { return this.repo.getMigrationCount(); } + public long getDataSourceMigrationCount() { return this.repo.getMigrationCount(); } public ArrayList getDataSourcesToMigrate(int limit) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index da63e4857..d1658064b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -79,9 +79,6 @@ public class FullDataSourceProviderV2 private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000; - private static boolean migrationMessageShown = false; - - protected final ThreadPoolExecutor migrationThreadPool; /** * Interrupting the migration thread pool doesn't work well and may corrupt the database @@ -90,6 +87,9 @@ public class FullDataSourceProviderV2 protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); protected final FullDataSourceProviderV1 legacyFileHandler; + protected long legacyDeletionCount = -1; + protected long migrationCount = -1; + /** * Tracks which positions are currently being updated * to prevent duplicate concurrent updates. @@ -120,8 +120,6 @@ public class FullDataSourceProviderV2 String dimensionName = level.getLevelWrapper().getDimensionType().getDimensionName(); // start migrating any legacy data sources present in the background - int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); - LOGGER.info("Found ["+totalCount+"] data sources that need migration."); this.migrationThreadPool = ThreadUtil.makeRateLimitedThreadPool(1, MIGRATION_THREAD_NAME_PREFIX +"["+dimensionName+"]", Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads.get(), Thread.MIN_PRIORITY, (Semaphore)null); this.migrationThreadPool.execute(() -> this.convertLegacyDataSources()); @@ -333,43 +331,70 @@ public class FullDataSourceProviderV2 // this could be done all at once via SQL, // but doing it in chunks prevents locking the database for long periods of time long unusedCount = 0; - long totalUnusedCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); - if (totalUnusedCount != 0) + long totalDeleteCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); + if (totalDeleteCount != 0) { - LOGGER.info("deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources..."); + // this should only be shown once per session but should be shown during + // either when the deletion or migration phases start + ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + + + LOGGER.info("deleting [" + dimensionName + "] - ["+totalDeleteCount+"] unused data sources..."); + this.legacyDeletionCount = totalDeleteCount; - ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); + ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50); while (unusedDataPosList.size() != 0) { - this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); unusedCount += unusedDataPosList.size(); - unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); - - LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalUnusedCount + "]..."); + this.legacyDeletionCount -= unusedDataPosList.size(); + + + long startTime = System.currentTimeMillis(); + + // delete batch and get next batch + this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); + unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50); + + long endStart = System.currentTimeMillis(); + long deleteTime = endStart - startTime; + LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalDeleteCount + "] in ["+deleteTime+"]ms ..."); + + + // a slight delay is added to prevent accidentally locking the database when deleting a lot of rows + // (that shouldn't be the case since we're using WAL journaling, but just in case) + try + { + // use the delete time so we don't make powerful computers wait super long + // and weak computers wait no time at all + Thread.sleep(deleteTime / 2); + } + catch (InterruptedException ignore){} } - LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources."); + LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalDeleteCount+"] unused data sources."); } - //=========// - // migrate // - //=========// + //===========// + // migration // + //===========// - int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); - LOGGER.info("Found ["+totalCount+"] data sources that need migration."); + long totalMigrationCount = this.legacyFileHandler.getDataSourceMigrationCount(); + this.migrationCount = totalMigrationCount; + LOGGER.info("Found ["+totalMigrationCount+"] data sources that need migration."); ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + // keep going until every data source has been migrated int progressCount = 0; while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) { - LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalCount + "]..."); + LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalMigrationCount + "]..."); ArrayList> updateFutureList = new ArrayList<>(); for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) @@ -391,6 +416,12 @@ public class FullDataSourceProviderV2 { // after the update finishes the legacy data source can be safely deleted this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos()); + + try + { + newDataSource.close(); + } + catch (Exception ignore){ } }); } catch (Exception e) @@ -418,13 +449,16 @@ public class FullDataSourceProviderV2 } legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + progressCount += legacyDataSourceList.size(); + this.migrationCount -= legacyDataSourceList.size(); } if (this.migrationThreadRunning.get()) { LOGGER.info("migration complete for: ["+dimensionName+"]-["+this.saveDir+"]."); + this.migrationCount = 0; } else { @@ -439,7 +473,8 @@ public class FullDataSourceProviderV2 this.migrationThreadRunning.set(false); } - public int getMigrationCount() { return this.legacyFileHandler.getDataSourceMigrationCount(); } + public long getLegacyDeletionCount() { return this.legacyDeletionCount; } + public long getTotalMigrationCount() { return this.migrationCount; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 90accbcde..b10692021 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -26,7 +26,6 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; @@ -272,7 +271,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); - int migrationCount = this.fullDataSourceProvider.getMigrationCount(); + long legacyDeletionCount = this.fullDataSourceProvider.getLegacyDeletionCount(); + long migrationCount = this.fullDataSourceProvider.getTotalMigrationCount(); @@ -284,6 +284,10 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle { lines.add("File Handler [" + dimName + "]"); lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); + if (legacyDeletionCount > 0) + { + lines.add(" Legacy Deletion #: " + legacyDeletionCount); + } if (migrationCount > 0) { lines.add(" Legacy Migration #: " + migrationCount); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index d40f2889c..03a9f232c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -187,7 +187,7 @@ public class FullDataSourceV1Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); @@ -198,7 +198,8 @@ public class FullDataSourceV1Repo extends AbstractDhRepo Date: Sat, 13 Apr 2024 17:02:31 -0500 Subject: [PATCH 168/183] Fix double sending migration messages --- .../distanthorizons/core/api/internal/ClientApi.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index 8f9c48df4..9b15b1333 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -78,7 +78,9 @@ public class ClientApi public static final long SPAM_LOGGER_FLUSH_NS = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS); private boolean configOverrideReminderPrinted = false; - private boolean showMigrationMessage = false; + + private boolean migrationMessageShown = false; + private boolean showMigrationMessageNextFrame = false; public boolean rendererDisabledBecauseOfExceptions = false; @@ -485,10 +487,12 @@ public class ClientApi } // data migration - if (this.showMigrationMessage + if (this.showMigrationMessageNextFrame + && this.migrationMessageShown && Config.Client.Advanced.LodBuilding.showMigrationChatWarning.get()) { - this.showMigrationMessage = false; + this.showMigrationMessageNextFrame = false; + this.migrationMessageShown = false; MC.sendChatMessage("Old Distant Horizons data is being migrated."); MC.sendChatMessage("During migration LODs may load slowly and DH world gen is disabled."); @@ -645,6 +649,6 @@ public class ClientApi } // TODO there's probably a better way of handling chat messages - public void showMigrationMessageOnNextFrame() { this.showMigrationMessage = true; } + public void showMigrationMessageOnNextFrame() { this.showMigrationMessageNextFrame = true; } } From 6e0071a04695f6131b333e95ae86b3b4045187e5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 17:30:39 -0500 Subject: [PATCH 169/183] Fix migration chat message not appearing --- .../seibel/distanthorizons/core/api/internal/ClientApi.java | 4 ++-- .../core/file/fullDatafile/FullDataSourceProviderV2.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java index 9b15b1333..c29fcfeef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/api/internal/ClientApi.java @@ -488,11 +488,11 @@ public class ClientApi // data migration if (this.showMigrationMessageNextFrame - && this.migrationMessageShown + && !this.migrationMessageShown && Config.Client.Advanced.LodBuilding.showMigrationChatWarning.get()) { this.showMigrationMessageNextFrame = false; - this.migrationMessageShown = false; + this.migrationMessageShown = true; MC.sendChatMessage("Old Distant Horizons data is being migrated."); MC.sendChatMessage("During migration LODs may load slowly and DH world gen is disabled."); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index d1658064b..ced9c204e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -427,7 +427,7 @@ public class FullDataSourceProviderV2 catch (Exception e) { DhSectionPos migrationPos = legacyDataSource.getPos(); - LOGGER.error("Unexpected issue migrating data source at pos " + migrationPos + ". Error: " + e.getMessage(), e); + LOGGER.warn("Unexpected issue migrating data source at pos " + migrationPos + ". Error: " + e.getMessage(), e); this.legacyFileHandler.markMigrationFailed(migrationPos); } } From a2e2559b2630e9da21c805aeda4a1174ebf81ab9 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 18 Apr 2024 07:47:42 -0500 Subject: [PATCH 170/183] Fix transparent blocks glowing (thanks IMS) --- .../distanthorizons/core/render/renderer/LodRenderer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java index 2d66a10fd..584e275f1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java @@ -514,7 +514,7 @@ public class LodRenderer GL32.glEnable(GL32.GL_BLEND); GL32.glBlendEquation(GL32.GL_FUNC_ADD); - GL32.glBlendFunc(GL32.GL_ONE, GL32.GL_ONE_MINUS_SRC_ALPHA); + GL32.glBlendFuncSeparate(GL32.GL_SRC_ALPHA, GL32.GL_ONE_MINUS_SRC_ALPHA, GL32.GL_ONE, GL32.GL_ONE_MINUS_SRC_ALPHA); ApiEventInjector.INSTANCE.fireAllEvents(DhApiBeforeRenderPassEvent.class, renderEventParam); this.bufferHandler.renderTransparent(this, renderEventParam); GL32.glDepthMask(true); // Apparently the depth mask state is stored in the FBO, so glState fails to restore it... From 031742a951229696808d335b2042f53daade9182 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 18 Apr 2024 21:06:25 -0500 Subject: [PATCH 171/183] speed up initial LOD loading --- .../core/render/LodRenderSection.java | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index 71c9a79a1..f0b3dd12a 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -131,10 +131,23 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable this.uploadRenderDataToGpuFuture = CompletableFuture.runAsync(() -> { - this.getRenderSourceAsync().thenAccept((renderSource) -> + //==================// + // load render data // + //==================// + + CompletableFuture thisLoadFuture = this.getRenderSourceAsync(); + CompletableFuture[] adjacentLoadFutures = this.getNeighborRenderSourcesAsync(); + + // wait for all futures to complete together, + // merging the futures makes loading significantly faster than loading this position then loading the neighbors + ArrayList> futureList = new ArrayList<>(); + futureList.add(thisLoadFuture); + futureList.addAll(Arrays.asList(adjacentLoadFutures)); + CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).thenAccept((voidObj) -> { try { + ColumnRenderSource renderSource = thisLoadFuture.get(); if (renderSource == null || renderSource.isEmpty()) { // nothing needs to be rendered @@ -144,44 +157,38 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable - //=================================// - // get the neighbor render sources // - //=================================// - - CompletableFuture[] adjacentLoadFutures = this.getNeighborRenderSourcesAsync(); - - - //==============================// // build/upload new render data // //==============================// - CompletableFuture.allOf(adjacentLoadFutures).thenRun(() -> + try { - try + ColumnRenderBuffer previousBuffer = this.renderBuffer; + + ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; + for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { - ColumnRenderBuffer previousBuffer = this.renderBuffer; - - ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; - for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) - { - adjacentRenderSections[i] = adjacentLoadFutures[i].getNow(null); - } - ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections).thenAccept((buffer) -> - { - // upload complete, clean up the old data if - this.renderBuffer = buffer; - this.canRender = true; - this.uploadRenderDataToGpuFuture = null; - - }); + adjacentRenderSections[i] = adjacentLoadFutures[i].getNow(null); } - catch (Exception e) + ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections).thenAccept((buffer) -> { - LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); + // upload complete, clean up the old data if + this.renderBuffer = buffer; + this.canRender = true; this.uploadRenderDataToGpuFuture = null; - } - }); + + + if (previousBuffer != null) + { + previousBuffer.close(); + } + }); + } + catch (Exception e) + { + LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); + this.uploadRenderDataToGpuFuture = null; + } } catch (Exception e) { From 57cd143009d4d51703cc9aba7aa16ba0f22e92d5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 08:49:44 -0500 Subject: [PATCH 172/183] Fix LodRenderSections not stopping load futures when closed --- .../core/render/LodRenderSection.java | 162 +++++++++++++++--- .../core/util/objects/quadTree/QuadTree.java | 2 + 2 files changed, 139 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java index f0b3dd12a..d4c25cc74 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodRenderSection.java @@ -35,6 +35,7 @@ import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnR import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; import javax.annotation.WillNotClose; import java.awt.*; @@ -77,8 +78,10 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable private CompletableFuture uploadRenderDataToGpuFuture = null; private final ReentrantLock getRenderSourceLock = new ReentrantLock(); - /** Used to track this position's render data loading */ - private CompletableFuture renderSourceLoadingFuture = null; + /** Stored as a class variable so we can reuse it's result across multiple LOD loads if necessary */ + private ReferencedFutureWrapper renderSourceLoadingRefFuture = null; + /** Stored as a class variable so we can decrement reference counts as each {@link LodRenderSection} finishes using them. */ + private ReferencedFutureWrapper[] adjacentLoadRefFutures; private boolean missingPositionsCalculated = false; /** should be an empty array if no positions need to be generated */ @@ -106,7 +109,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // render data loading/uploading // //===============================// - public void uploadRenderDataToGpuAsync() + public synchronized void uploadRenderDataToGpuAsync() { if (!GLProxy.hasInstance()) { @@ -135,21 +138,34 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // load render data // //==================// - CompletableFuture thisLoadFuture = this.getRenderSourceAsync(); - CompletableFuture[] adjacentLoadFutures = this.getNeighborRenderSourcesAsync(); + this.tryDecrementingLoadFutureArray(this.adjacentLoadRefFutures); + + ReferencedFutureWrapper thisLoadFuture = this.getRenderSourceAsync(); + ReferencedFutureWrapper[] adjLoadRefFutures = this.getNeighborRenderSourcesAsync(); + // wait for all futures to complete together, - // merging the futures makes loading significantly faster than loading this position then loading the neighbors + // merging the futures makes loading significantly faster than loading this position then loading its neighbors ArrayList> futureList = new ArrayList<>(); - futureList.add(thisLoadFuture); - futureList.addAll(Arrays.asList(adjacentLoadFutures)); + futureList.add(thisLoadFuture.future); + for (ReferencedFutureWrapper refFuture : adjLoadRefFutures) + { + futureList.add(refFuture.future); + } + CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).thenAccept((voidObj) -> { try { - ColumnRenderSource renderSource = thisLoadFuture.get(); + ColumnRenderSource renderSource = thisLoadFuture.future.get(); if (renderSource == null || renderSource.isEmpty()) { + thisLoadFuture.decrementRefCount(); + for (ReferencedFutureWrapper futureWrapper : adjLoadRefFutures) + { + futureWrapper.decrementRefCount(); + } + // nothing needs to be rendered this.canRender = false; return; @@ -168,7 +184,7 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ColumnRenderSource[] adjacentRenderSections = new ColumnRenderSource[EDhDirection.ADJ_DIRECTIONS.length]; for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { - adjacentRenderSections[i] = adjacentLoadFutures[i].getNow(null); + adjacentRenderSections[i] = adjLoadRefFutures[i].future.getNow(null); } ColumnRenderBufferBuilder.buildAndUploadBuffersAsync(this.level, renderSource, adjacentRenderSections).thenAccept((buffer) -> { @@ -182,16 +198,28 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { previousBuffer.close(); } + + thisLoadFuture.decrementRefCount(); + this.tryDecrementingLoadFutureArray(adjLoadRefFutures); + this.adjacentLoadRefFutures = null; }); } catch (Exception e) { + thisLoadFuture.decrementRefCount(); + this.tryDecrementingLoadFutureArray(adjLoadRefFutures); + this.adjacentLoadRefFutures = null; + LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); this.uploadRenderDataToGpuFuture = null; } } catch (Exception e) { + thisLoadFuture.decrementRefCount(); + this.tryDecrementingLoadFutureArray(adjLoadRefFutures); + this.adjacentLoadRefFutures = null; + LOGGER.error("Unexpected error in LodRenderSection loading, Error: "+e.getMessage(), e); this.uploadRenderDataToGpuFuture = null; } @@ -199,10 +227,9 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable }, executor); } /** Should be called on the {@link ThreadPoolUtil#getFileHandlerExecutor()} */ - @SuppressWarnings("unchecked") // creating an array of CompletableFuture's is unchecked, unfortunately I don't currently see a better fix - private CompletableFuture[] getNeighborRenderSourcesAsync() + private ReferencedFutureWrapper[] getNeighborRenderSourcesAsync() { - CompletableFuture[] futureArray = new CompletableFuture[EDhDirection.ADJ_DIRECTIONS.length]; + ReferencedFutureWrapper[] futureArray = new ReferencedFutureWrapper[EDhDirection.ADJ_DIRECTIONS.length]; for (int i = 0; i < EDhDirection.ADJ_DIRECTIONS.length; i++) { EDhDirection direction = EDhDirection.ADJ_DIRECTIONS[i]; @@ -221,14 +248,15 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable if (futureArray[arrayIndex] == null) { - futureArray[arrayIndex] = CompletableFuture.completedFuture(null); + futureArray[arrayIndex] = new ReferencedFutureWrapper(CompletableFuture.completedFuture(null)); } } + this.adjacentLoadRefFutures = futureArray; return futureArray; } /** Will try to return the same {@link CompletableFuture} if multiple requests are made for the same position */ - private CompletableFuture getRenderSourceAsync() + private ReferencedFutureWrapper getRenderSourceAsync() { try { @@ -237,9 +265,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable // if a load is already in progress, use that existing one // (this reduces the number of duplicate loads that may happen when initially loading the world) - if (this.renderSourceLoadingFuture != null) + if (this.renderSourceLoadingRefFuture != null) { - return this.renderSourceLoadingFuture; + // increment the number of objects needing this future + this.renderSourceLoadingRefFuture.incrementRefCount(); + return this.renderSourceLoadingRefFuture; } @@ -247,25 +277,25 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable ThreadPoolExecutor executor = ThreadPoolUtil.getFileHandlerExecutor(); if (executor == null || executor.isTerminated()) { - return CompletableFuture.completedFuture(null); + return new ReferencedFutureWrapper(CompletableFuture.completedFuture(null)); } - this.renderSourceLoadingFuture = CompletableFuture.supplyAsync(() -> + this.renderSourceLoadingRefFuture = new ReferencedFutureWrapper(CompletableFuture.supplyAsync(() -> { try (FullDataSourceV2 fullDataSource = this.fullDataSourceProvider.get(this.pos)) { ColumnRenderSource renderSource = FullDataToRenderDataTransformer.transformFullDataToRenderSource(fullDataSource, this.level); - this.renderSourceLoadingFuture = null; + this.renderSourceLoadingRefFuture = null; return renderSource; } catch (Exception e) { LOGGER.warn("Unable to get render source " + this.pos + ", error: " + e.getMessage(), e); - this.renderSourceLoadingFuture = null; + this.renderSourceLoadingRefFuture = null; return null; } - }, executor); - return this.renderSourceLoadingFuture; + }, executor)); + return this.renderSourceLoadingRefFuture; } finally { @@ -334,6 +364,27 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable + //=========// + // cleanup // + //=========// + + /** does nothing if the passed in value is null. */ + private void tryDecrementingLoadFutureArray(@Nullable ReferencedFutureWrapper[] refFutures) + { + if (refFutures != null) + { + for (ReferencedFutureWrapper futureWrapper : refFutures) + { + if (futureWrapper != null) + { + futureWrapper.decrementRefCount(); + } + } + } + } + + + //==============// // base methods // //==============// @@ -351,6 +402,19 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable { DebugRenderer.unregister(this, Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus); + if (Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get()) + { + // show a particle for the closed section + DebugRenderer.makeParticle( + new DebugRenderer.BoxParticle( + new DebugRenderer.Box(this.pos, 128f, 156f, 0.09f, Color.RED.darker()), + 0.5, 32f + ) + ); + } + + + if (this.renderBuffer != null) { this.renderBuffer.close(); @@ -361,9 +425,11 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable this.uploadRenderDataToGpuFuture.cancel(true); } - if (this.renderSourceLoadingFuture != null) + // this render section won't be rendering, we don't need to load any data for it + this.tryDecrementingLoadFutureArray(this.adjacentLoadRefFutures); + if (this.renderSourceLoadingRefFuture != null) { - this.renderSourceLoadingFuture.cancel(true); + this.renderSourceLoadingRefFuture.decrementRefCount(); } @@ -403,4 +469,50 @@ public class LodRenderSection implements IDebugRenderable, AutoCloseable debugRenderer.renderBox(new DebugRenderer.Box(this.pos, 400, 8f, Objects.hashCode(this), 0.1f, color)); } + + + //================// + // helper classes // + //================// + + /** + * Used to keep track of whether a {@link ColumnRenderSource} {@link CompletableFuture} + * is in use or not, and if not in use cancels the future.

+ * + * This helps speed up LOD loading by canceling loads that are no longer needed, + * IE out of range or in an unloaded dimension. + */ + private static class ReferencedFutureWrapper + { + public final CompletableFuture future; + // starts at 1 since the constructing method is referencing this future + private final AtomicInteger refCount = new AtomicInteger(1); + + + + public ReferencedFutureWrapper(CompletableFuture future) { this.future = future; } + + public void incrementRefCount() { this.refCount.incrementAndGet(); } + public void decrementRefCount() + { + // automatically clean up this future if no one else is referencing it + if (this.refCount.decrementAndGet() <= 0) + { + if (this.future != null) + { + if (!this.future.isDone()) + { + this.future.cancel(true); + } + } + } + } + + + + @Override + public String toString() { return this.future.toString() + " - " + this.refCount.get(); } + + } + } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java index 49b67e736..1679bea9e 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/util/objects/quadTree/QuadTree.java @@ -264,6 +264,8 @@ public class QuadTree { if (quadNode != null && removedItemConsumer != null) { + quadNode.deleteAllChildren(removedItemConsumer); + removedItemConsumer.accept(quadNode.value); } }); From 22f4757aae28aab5a29ec5a8c212681ad7510403 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 11:02:52 -0500 Subject: [PATCH 173/183] Fix rendering holes --- .../core/render/LodQuadTree.java | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java index 4ab22746a..391d834ee 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java @@ -29,6 +29,7 @@ import com.seibel.distanthorizons.core.level.IDhClientLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; import com.seibel.distanthorizons.core.pos.DhSectionPos; +import com.seibel.distanthorizons.core.render.renderer.DebugRenderer; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.ThreadUtil; import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode; @@ -38,6 +39,7 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import javax.annotation.WillNotClose; +import java.awt.*; import java.util.ArrayList; import java.util.Iterator; import java.util.concurrent.ConcurrentLinkedQueue; @@ -221,7 +223,7 @@ public class LodQuadTree extends QuadTree implements AutoClose private boolean recursivelyUpdateRenderSectionNode( DhBlockPos2D playerPos, QuadNode rootNode, QuadNode quadNode, DhSectionPos sectionPos, - boolean parentRenderSectionIsEnabled, + boolean parentSectionIsRendering, ArrayList nodesNeedingRetrieval, ArrayList nodesNeedingLoading) { @@ -268,7 +270,7 @@ public class LodQuadTree extends QuadTree implements AutoClose if (sectionPos.getDetailLevel() > expectedDetailLevel) { // section detail level too high // - boolean canThisPosRender = renderSection.canRender(); + boolean thisPosIsRendering = renderSection.renderingEnabled; boolean allChildrenSectionsAreLoaded = true; // recursively update all child render sections @@ -278,17 +280,29 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, canThisPosRender || parentRenderSectionIsEnabled, nodesNeedingRetrieval, nodesNeedingLoading); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, thisPosIsRendering || parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } if (!allChildrenSectionsAreLoaded) { // not all child positions are loaded yet, or this section is out of render range - return canThisPosRender; + return thisPosIsRendering; } else { + if (renderSection.renderingEnabled + && Config.Client.Advanced.Debugging.DebugWireframe.showRenderSectionStatus.get()) + { + // show that this position has just been disabled + DebugRenderer.makeParticle( + new DebugRenderer.BoxParticle( + new DebugRenderer.Box(renderSection.pos, 128f, 156f, 0.09f, Color.CYAN.darker()), + 0.2, 32f + ) + ); + } + // all child positions are loaded, disable this section and enable its children. renderSection.renderingEnabled = false; @@ -299,7 +313,7 @@ public class LodQuadTree extends QuadTree implements AutoClose DhSectionPos childPos = childPosIterator.next(); QuadNode childNode = rootNode.getNode(childPos); - boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentRenderSectionIsEnabled, nodesNeedingRetrieval, nodesNeedingLoading); + boolean childSectionLoaded = this.recursivelyUpdateRenderSectionNode(playerPos, rootNode, childNode, childPos, parentSectionIsRendering, nodesNeedingRetrieval, nodesNeedingLoading); allChildrenSectionsAreLoaded = childSectionLoaded && allChildrenSectionsAreLoaded; } if (!allChildrenSectionsAreLoaded) @@ -310,7 +324,7 @@ public class LodQuadTree extends QuadTree implements AutoClose } // this section is now being rendered via its children - return true; + return allChildrenSectionsAreLoaded; } } // TODO this should only equal the expected detail level, the (expectedDetailLevel-1) is a temporary fix to prevent corners from being cut out @@ -336,7 +350,7 @@ public class LodQuadTree extends QuadTree implements AutoClose } // wait for the parent to disable before enabling this section, so we don't overdraw/overlap render sections - if (!parentRenderSectionIsEnabled && renderSection.canRender()) + if (!parentSectionIsRendering && renderSection.canRender()) { // if rendering is already enabled we don't have to re-enable it if (!renderSection.renderingEnabled) @@ -348,6 +362,17 @@ public class LodQuadTree extends QuadTree implements AutoClose { if (childRenderSection != null) { + if (childRenderSection.renderingEnabled) + { + // show that this position's rendering has been disabled due to a parent rendering + DebugRenderer.makeParticle( + new DebugRenderer.BoxParticle( + new DebugRenderer.Box(childRenderSection.pos, 128f, 156f, 0.09f, Color.MAGENTA.darker()), + 0.2, 32f + ) + ); + } + childRenderSection.renderingEnabled = false; childRenderSection.close(); } From 37381896bc5b79bd4c8cf2b1980d87305e66abfd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 11:38:31 -0500 Subject: [PATCH 174/183] Fix warnings about updating empty maps --- .../core/dataObjects/fullData/FullDataPointIdMap.java | 2 +- .../dataObjects/fullData/sources/FullDataSourceV2.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java index 4dd9ef5b9..ce362a4e6 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/FullDataPointIdMap.java @@ -61,7 +61,6 @@ public class FullDataPointIdMap private static final String BLOCK_STATE_SEPARATOR_STRING = "_DH-BSW_"; - // FIXME: Improve performance maybe? /** used when the data point map is running normally */ private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); @@ -118,6 +117,7 @@ public class FullDataPointIdMap /** @return -1 if the list is empty */ public int getMaxValidId() { return this.entryList.size() - 1; } + public boolean isEmpty() { return this.entryList.isEmpty(); } public DhSectionPos getPos() { return this.pos; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 1fd66bb9d..2282ac0df 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -223,6 +223,13 @@ public class FullDataSourceV2 implements IDataSource public boolean update(@NotNull FullDataSourceV2 inputDataSource, @Nullable IDhLevel level) { return this.update(inputDataSource); } public boolean update(@NotNull FullDataSourceV2 inputDataSource) { + // don't try updating if the input is empty + if (inputDataSource.mapping.isEmpty()) + { + return false; + } + + byte thisDetailLevel = this.pos.getDetailLevel(); byte inputDetailLevel = inputDataSource.pos.getDetailLevel(); From 382917033a3323fca452e893e3764c3486149d44 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 11:38:52 -0500 Subject: [PATCH 175/183] Fix a few warnings --- .../file/AbstractNewDataSourceHandler.java | 37 +++++++++---------- .../FullDataSourceProviderV2.java | 3 +- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java index 1495dc2a4..499e812d1 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java @@ -98,9 +98,6 @@ public abstract class AbstractNewDataSourceHandler protected abstract TDataSource createDataSourceFromDto(TDTO dto) throws InterruptedException, IOException; protected abstract TDTO createDtoFromDataSource(TDataSource dataSource); - /** Creates a new data source using any DTOs already present in the database. */ - protected abstract TDataSource createNewDataSourceFromExistingDtos(DhSectionPos pos); - protected abstract TDataSource makeEmptyDataSource(DhSectionPos pos); @@ -153,8 +150,10 @@ public abstract class AbstractNewDataSourceHandler } else { - // attempt to create from any existing files - dataSource = this.createNewDataSourceFromExistingDtos(pos); + // TODO does this need any special logic to be populated from the existing DTOs? + // and/or is that necessary? + // Everything already appears to be populating correctly. + dataSource = this.makeEmptyDataSource(pos); } } catch (InterruptedException ignore) { } @@ -228,29 +227,27 @@ public abstract class AbstractNewDataSourceHandler updateLock.lock(); this.lockedPosSet.add(updatePos); } - else - { - methodLocked = false; - } // get or create the data source try (TDataSource recipientDataSource = this.get(updatePos)) { - boolean dataModified = recipientDataSource.update(inputData, this.level); - - if (dataModified) + if (recipientDataSource != null) { - // save the updated data to the database - TDTO dto = this.createDtoFromDataSource(recipientDataSource); - this.repo.save(dto); - - - for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) + boolean dataModified = recipientDataSource.update(inputData, this.level); + if (dataModified) { - if (listener != null) + // save the updated data to the database + TDTO dto = this.createDtoFromDataSource(recipientDataSource); + this.repo.save(dto); + + + for (IDataSourceUpdateFunc listener : this.dateSourceUpdateListeners) { - listener.OnDataSourceUpdated(recipientDataSource); + if (listener != null) + { + listener.OnDataSourceUpdated(recipientDataSource); + } } } } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index ced9c204e..769ff34d5 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -260,7 +260,8 @@ public class FullDataSourceProviderV2 try (FullDataSourceV2 dataSource = this.get(childPos)) { - if (dataSource != null) // can return null when the file handler is being shut down + // can return null when the file handler is being shut down + if (dataSource != null) { this.updateDataSourceAtPos(parentUpdatePos, dataSource, false); this.repo.setApplyToParent(childPos, false); From 3826c83d898e6cdb8bdaf88385838a5edd254a3c Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 11:41:02 -0500 Subject: [PATCH 176/183] Remove "new" from AbstractDataSourceHandler's name --- .../fullData/sources/FullDataSourceV2.java | 5 ++--- ...ceHandler.java => AbstractDataSourceHandler.java} | 12 ++++++------ .../file/fullDatafile/FullDataSourceProviderV2.java | 6 +++--- .../core/level/ClientLevelModule.java | 4 ++-- 4 files changed, 13 insertions(+), 14 deletions(-) rename core/src/main/java/com/seibel/distanthorizons/core/file/{AbstractNewDataSourceHandler.java => AbstractDataSourceHandler.java} (95%) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java index 2282ac0df..8846ba085 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/dataObjects/fullData/sources/FullDataSourceV2.java @@ -23,7 +23,7 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap; import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder; -import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; +import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; import com.seibel.distanthorizons.core.file.DataSourcePool; import com.seibel.distanthorizons.core.file.IDataSource; import com.seibel.distanthorizons.core.level.IDhLevel; @@ -32,7 +32,6 @@ import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.util.FullDataPointUtil; import com.seibel.distanthorizons.core.util.LodUtil; import com.seibel.distanthorizons.core.util.RenderDataPointUtil; -import com.seibel.distanthorizons.core.wrapperInterfaces.block.IBlockStateWrapper; import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper; import com.seibel.distanthorizons.coreapi.ModInfo; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -256,7 +255,7 @@ public class FullDataSourceV2 implements IDataSource } // determine if this data source should be applied to its parent - this.applyToParent = (dataChanged && this.pos.getDetailLevel() < AbstractNewDataSourceHandler.TOP_SECTION_DETAIL_LEVEL); + this.applyToParent = (dataChanged && this.pos.getDetailLevel() < AbstractDataSourceHandler.TOP_SECTION_DETAIL_LEVEL); if (dataChanged) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java similarity index 95% rename from core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java rename to core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java index 499e812d1..b913e16c7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractNewDataSourceHandler.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/AbstractDataSourceHandler.java @@ -23,7 +23,7 @@ import java.util.concurrent.locks.ReentrantLock; // TODO is there a reason this is separate from FullDataSourceProviderV2? // We shouldn't need multiple data source handlers -public abstract class AbstractNewDataSourceHandler +public abstract class AbstractDataSourceHandler , TDTO extends IBaseDTO, TRepo extends AbstractDhRepo, @@ -36,13 +36,13 @@ public abstract class AbstractNewDataSourceHandler * The highest numerical detail level possible. * Used when determining which positions to update. * - * @see AbstractNewDataSourceHandler#MIN_SECTION_DETAIL_LEVEL + * @see AbstractDataSourceHandler#MIN_SECTION_DETAIL_LEVEL */ public static final byte TOP_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL + LodUtil.REGION_DETAIL_LEVEL; /** * The lowest numerical detail level possible. * - * @see AbstractNewDataSourceHandler#TOP_SECTION_DETAIL_LEVEL + * @see AbstractDataSourceHandler#TOP_SECTION_DETAIL_LEVEL * */ public static final byte MIN_SECTION_DETAIL_LEVEL = DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL; @@ -72,8 +72,8 @@ public abstract class AbstractNewDataSourceHandler // constructor // //=============// - public AbstractNewDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } - public AbstractNewDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) + public AbstractDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure) { this(level, saveStructure, null); } + public AbstractDataSourceHandler(TDhLevel level, AbstractSaveStructure saveStructure, @Nullable File saveDirOverride) { this.level = level; this.saveDir = (saveDirOverride == null) ? saveStructure.getFullDataFolder(level.getLevelWrapper()) : saveDirOverride; @@ -134,7 +134,7 @@ public abstract class AbstractNewDataSourceHandler /** * Should only be used in internal file handler methods where we are already running on a file handler thread. * Can return null if the repo is in the process of being shut down - * @see AbstractNewDataSourceHandler#getAsync(DhSectionPos) + * @see AbstractDataSourceHandler#getAsync(DhSectionPos) */ @Nullable public TDataSource get(DhSectionPos pos) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 769ff34d5..080d3ab77 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -26,7 +26,7 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; -import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; +import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; import com.seibel.distanthorizons.core.level.IDhLevel; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.pos.DhSectionPos; @@ -55,7 +55,7 @@ import java.util.function.Function; * to and from the database. */ public class FullDataSourceProviderV2 - extends AbstractNewDataSourceHandler + extends AbstractDataSourceHandler implements IDebugRenderable { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); @@ -98,7 +98,7 @@ public class FullDataSourceProviderV2 // TODO only run thread if modifications happened recently /** - * This isn't in {@link AbstractNewDataSourceHandler} since we don't need parent updating logic + * This isn't in {@link AbstractDataSourceHandler} since we don't need parent updating logic * for render data, only full data. */ private final ThreadPoolExecutor updateQueueProcessor; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index b10692021..c24341d62 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -24,7 +24,7 @@ import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhAp import com.seibel.distanthorizons.core.config.Config; import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2; import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; -import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; +import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; @@ -47,7 +47,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; -public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandler.IDataSourceUpdateFunc +public class ClientLevelModule implements Closeable, AbstractDataSourceHandler.IDataSourceUpdateFunc { private static final Logger LOGGER = DhLoggerBuilder.getLogger(); private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class); From 459bc5fc0c1f6855e67c283d63b9efea9e9e2435 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 12:35:48 -0500 Subject: [PATCH 177/183] Ops, forgot to remove this unused override in DataSourceProvider --- .../core/file/fullDatafile/FullDataSourceProviderV2.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 080d3ab77..795ffb15c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -167,13 +167,6 @@ public class FullDataSourceProviderV2 @Override protected FullDataSourceV2 createDataSourceFromDto(FullDataSourceV2DTO dto) throws InterruptedException, IOException { return dto.createPooledDataSource(this.level.getLevelWrapper()); } - @Override - protected FullDataSourceV2 createNewDataSourceFromExistingDtos(DhSectionPos pos) - { - // TODO maybe just set children update flags to true? - // TODO is any special logic necessary? All DTOs should be generated using their children via the update system anyway - return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); - } @Override protected FullDataSourceV2 makeEmptyDataSource(DhSectionPos pos) { return FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos, true); } From 88ff9e7cde382deb511563b3af330ab45deaceee Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 12:36:19 -0500 Subject: [PATCH 178/183] Drop high and extreme horizontal quality enums --- .../api/enums/config/EDhApiHorizontalQuality.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java index 9bb431172..bdf7967cf 100644 --- a/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java +++ b/api/src/main/java/com/seibel/distanthorizons/api/enums/config/EDhApiHorizontalQuality.java @@ -43,8 +43,8 @@ public enum EDhApiHorizontalQuality LOWEST(2.0f, 4), LOW(2.0f, 8), MEDIUM(2.0f, 12), - HIGH(2.2f, 24), - EXTREME(2.4f, 64), + HIGH(2.2f, 16), + EXTREME(2.4f, 32), ; From 10014d3729226696ede806aa32be0fadbb9c04fd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 20 Apr 2024 19:00:33 -0500 Subject: [PATCH 179/183] Change default grass side rendering to fade_to_dirt --- .../java/com/seibel/distanthorizons/core/config/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java index 136a6e66b..ac2f0bb33 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/config/Config.java @@ -646,7 +646,7 @@ public class Config .build(); public static ConfigEntry grassSideRendering = new ConfigEntry.Builder() - .set(EDhApiGrassSideRendering.AS_DIRT) + .set(EDhApiGrassSideRendering.FADE_TO_DIRT) .comment("" + "How should the sides and bottom of grass block LODs render? \n" + "\n" From c89fcb094aafc829aa1e7e2ce2c3109e4cc0da9f Mon Sep 17 00:00:00 2001 From: Cutiepie <43445785+Ran-Mewo@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:53:31 +1000 Subject: [PATCH 180/183] Add core to gradle --- core/build.gradle | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 core/build.gradle diff --git a/core/build.gradle b/core/build.gradle new file mode 100644 index 000000000..425b64b2b --- /dev/null +++ b/core/build.gradle @@ -0,0 +1,56 @@ +plugins { + id "java" + id "com.github.johnrengelman.shadow" version '7.1.2' apply false // Set this to true if you're using the standalone Core project +} + +configurations { + shadowedArtifact // Used by DH to specify that we want to implement the shadowed core JAR file instead of the regular JAR file + shade + implementation.extendsFrom shade +} + +OperatingSystem os = org.gradle.nativeplatform.platform.internal.DefaultNativePlatform.currentOperatingSystem; + +// Set the OS lwjgl is using to the current os +project.ext.lwjglNatives = "natives-" + os.toFamilyName() + +dependencies { // All of these dependencies are in Vanilla Minecraft, but we need to depend on it as we arent importing Minecraft in the core + // Imports most of lwjgl's libraries (well, only the ones that we need) + implementation platform("org.lwjgl:lwjgl-bom:${rootProject.lwjgl_version}") // TODO: Use Minecraft's version for lwjgl_version (which changes in nearly every version) instead of a hard defined version for all versions + + // REMEMBER: Dont shadow stuff here, these are just the libs that are included in Minecraft so that the core can use + implementation "org.lwjgl:lwjgl" + implementation "org.lwjgl:lwjgl-assimp" + implementation "org.lwjgl:lwjgl-glfw" + implementation "org.lwjgl:lwjgl-openal" + implementation "org.lwjgl:lwjgl-opengl" + implementation "org.lwjgl:lwjgl-stb" + implementation "org.lwjgl:lwjgl-tinyfd" + runtimeOnly "org.lwjgl:lwjgl::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-assimp::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-glfw::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-openal::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-opengl::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-stb::$lwjglNatives" + runtimeOnly "org.lwjgl:lwjgl-tinyfd::$lwjglNatives" + + shade "it.unimi.dsi:fastutil:${rootProject.fastutil_version}" // Add our own fastutil version + + + // Some other dependencies + implementation("org.jetbrains:annotations:16.0.2") + implementation("com.google.code.findbugs:jsr305:3.0.2") + implementation("com.google.common:google-collect:0.5") + implementation("com.google.guava:guava:31.1-jre") + +} + +artifacts { + shadowedArtifact shadowJar // Setup the configuration shadowedArtifact to be the shadowJar +} + +shadowJar { + def librariesLocation = "distanthorizons.libraries" + relocate "it.unimi.dsi.fastutil", "${librariesLocation}.unimi.dsi.fastutil" + mergeServiceFiles() +} \ No newline at end of file From a613540b6a9a287054569430a92118c6d74f981f Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 25 Apr 2024 21:51:48 -0500 Subject: [PATCH 181/183] Move most libraries from the main script to core --- core/build.gradle | 70 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index 425b64b2b..bb7aae297 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -3,6 +3,12 @@ plugins { id "com.github.johnrengelman.shadow" version '7.1.2' apply false // Set this to true if you're using the standalone Core project } +apply plugin: "application" + +application { + mainClass.set("com.seibel.distanthorizons.core.jar.JarMain") +} + configurations { shadowedArtifact // Used by DH to specify that we want to implement the shadowed core JAR file instead of the regular JAR file shade @@ -33,8 +39,36 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee runtimeOnly "org.lwjgl:lwjgl-opengl::$lwjglNatives" runtimeOnly "org.lwjgl:lwjgl-stb::$lwjglNatives" runtimeOnly "org.lwjgl:lwjgl-tinyfd::$lwjglNatives" + + + // fast util + shade("it.unimi.dsi:fastutil:${rootProject.fastutil_version}") + + // Compression + shade("org.lz4:lz4-java:${rootProject.lz4_version}") // LZ4 + shade("com.github.luben:zstd-jni:${rootProject.zstd_version}") // Zstd + shade("org.tukaani:xz:${rootProject.xz_version}") // LZMA + + // Sqlite Database + shade("org.xerial:sqlite-jdbc:${rootProject.sqlite_jdbc_version}") + + // Netty + shade("io.netty:netty-all:${rootProject.netty_version}") + + // NightConfig (includes Toml & Json) + // needed in both common and core + shade("com.electronwill.night-config:toml:${rootProject.nightconfig_version}") + shade("com.electronwill.night-config:json:${rootProject.nightconfig_version}") + + + // needed for the standalone jar +// shade 'org.apache.logging.log4j:log4j-core:2.23.1' +// shade 'org.apache.logging.log4j:log4j-api:2.23.1' + + // SVG (not needed atm) + //shade("com.formdev:svgSalamander:${rootProject.svgSalamander_version}") + - shade "it.unimi.dsi:fastutil:${rootProject.fastutil_version}" // Add our own fastutil version // Some other dependencies @@ -42,15 +76,45 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee implementation("com.google.code.findbugs:jsr305:3.0.2") implementation("com.google.common:google-collect:0.5") implementation("com.google.guava:guava:31.1-jre") - + } artifacts { + shade shadowJar shadowedArtifact shadowJar // Setup the configuration shadowedArtifact to be the shadowJar } shadowJar { + configurations = [project.configurations.shade] + def librariesLocation = "distanthorizons.libraries" + + relocate "it.unimi.dsi.fastutil", "${librariesLocation}.unimi.dsi.fastutil" + + // LWJGL + // Only ever shadow the dependencies we use otherwise some stuff would break when running on an external client + relocate "org.lwjgl.system.jawt", "${librariesLocation}.lwjgl.system.jawt" + + // Compression + relocate "net.jpountz", "${librariesLocation}.jpountz" + relocate "com.github.luben", "${librariesLocation}.github.luben" + relocate "org.tukaani", "${librariesLocation}.tukaani" + + // Sqlite Database + //At the moment, there is a bug in this library which doesnt allow it to be relocated +// relocate "org.sqlite", "${librariesLocation}.sqlite" + + // JOML + if (project.hasProperty("embed_joml") && embed_joml == "true") + relocate "org.joml", "${librariesLocation}.joml" + + // NightConfig (includes Toml & Json) + relocate "com.electronwill.nightconfig", "${librariesLocation}.electronwill.nightconfig" + + // Netty + relocate "io.netty", "${librariesLocation}.netty" + + mergeServiceFiles() -} \ No newline at end of file +} From 2cc6c8d8401f422f7650a5479ae5f895c952d0dd Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 25 Apr 2024 21:52:44 -0500 Subject: [PATCH 182/183] Change Initializer compressor test class --- .../com/seibel/distanthorizons/core/Initializer.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java index 4aa4fe286..b84c721ef 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/Initializer.java @@ -28,13 +28,12 @@ import com.seibel.distanthorizons.core.api.external.methods.config.DhApiConfig; import com.seibel.distanthorizons.core.api.external.methods.data.DhApiTerrainDataRepo; import com.seibel.distanthorizons.api.DhApi; import com.seibel.distanthorizons.core.render.DhApiRenderProxy; -//import io.netty.buffer.ByteBuf; -import net.jpountz.lz4.LZ4Compressor; +import io.netty.buffer.ByteBuf; +import net.jpountz.lz4.LZ4FrameOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.awt.*; -import java.io.InputStream; /** Handles first time Core setup. */ public class Initializer @@ -48,11 +47,11 @@ public class Initializer { // if any library isn't present in the jar its class // will throw an error (not an exception) - Class compressor = LZ4Compressor.class; - //Class networking = ByteBuf.class; + Class compressor = LZ4FrameOutputStream.class; + Class networking = ByteBuf.class; Class toml = com.electronwill.nightconfig.core.Config.class; } - catch (NoClassDefFoundError e) + catch (Throwable e) { LOGGER.fatal("Critical programmer error: One or more libraries aren't present. Error: [" + e.getMessage() + "]."); throw e; From b41e54e6a6e05bf5b4bc76f302848f4cf65cd8d5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Thu, 25 Apr 2024 22:09:51 -0500 Subject: [PATCH 183/183] shade in apache.logging for the standalone jar --- core/build.gradle | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/build.gradle b/core/build.gradle index bb7aae297..172f997d8 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -62,8 +62,8 @@ dependencies { // All of these dependencies are in Vanilla Minecraft, but we nee // needed for the standalone jar -// shade 'org.apache.logging.log4j:log4j-core:2.23.1' -// shade 'org.apache.logging.log4j:log4j-api:2.23.1' + shade("org.apache.logging.log4j:log4j-core:2.23.1") + shade("org.apache.logging.log4j:log4j-api:2.23.1") // SVG (not needed atm) //shade("com.formdev:svgSalamander:${rootProject.svgSalamander_version}") @@ -114,6 +114,8 @@ shadowJar { // Netty relocate "io.netty", "${librariesLocation}.netty" + + relocate "org.apache.logging", "${librariesLocation}.apache.logging" mergeServiceFiles()