Update the API to allow for N-sized world generation requests

This breaks old world generators
This commit is contained in:
James Seibel
2024-10-07 19:45:28 -05:00
parent 28ec1e2960
commit 1b59a269e6
11 changed files with 325 additions and 227 deletions
@@ -19,8 +19,8 @@
package com.seibel.distanthorizons.api.enums.worldGeneration;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGeneratorMode;
import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator;
import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
import java.util.concurrent.ExecutorService;
import java.util.function.Consumer;
@@ -28,16 +28,17 @@ import java.util.function.Consumer;
/**
* VANILLA_CHUNKS, <br>
* API_CHUNKS <br>
* API_DATA_SOURCES <br>
*
* @author Builderb0y, James Seibel
* @version 2023-12-21
* @version 2024-10-5
* @since API 2.0.0
*/
public enum EDhApiWorldGeneratorReturnType
{
/**
* when this constant is returned by {@link IDhApiWorldGenerator#getReturnType()},
* {@link IDhApiWorldGenerator#generateChunks(int, int, byte, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* {@link IDhApiWorldGenerator#generateChunks(int, int, int, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* will be used when generating terrain.
*
* @since API 2.0.0
@@ -46,11 +47,20 @@ public enum EDhApiWorldGeneratorReturnType
/**
* when this constant is returned by {@link IDhApiWorldGenerator#getReturnType()},
* {@link IDhApiWorldGenerator#generateApiChunks(int, int, byte, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* {@link IDhApiWorldGenerator#generateApiChunks(int, int, int, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* will be used when generating terrain.
*
* @since API 2.0.0
*/
API_CHUNKS;
API_CHUNKS,
/**
* when this constant is returned by {@link IDhApiWorldGenerator#getReturnType()},
* {@link IDhApiWorldGenerator#generateLod(int, int, byte, IDhApiFullDataSource, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* will be used when generating terrain.
*
* @since API 4.0.0
*/
API_DATA_SOURCES;
}
@@ -46,10 +46,6 @@ public abstract class AbstractDhApiChunkWorldGenerator implements Closeable, IDh
public final byte getSmallestDataDetailLevel() { return EDhApiDetailLevel.BLOCK.detailLevel; }
@Override
public final byte getLargestDataDetailLevel() { return EDhApiDetailLevel.BLOCK.detailLevel; }
@Override
public final byte getMinGenerationGranularity() { return EDhApiDetailLevel.CHUNK.detailLevel; }
@Override
public final byte getMaxGenerationGranularity() { return (byte) (EDhApiDetailLevel.CHUNK.detailLevel + 2); }
@@ -60,17 +56,14 @@ public abstract class AbstractDhApiChunkWorldGenerator implements Closeable, IDh
@Override
public final CompletableFuture<Void> generateChunks(
int chunkPosMinX, int chunkPosMinZ,
byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode,
int generationRequestChunkWidthCount, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool, Consumer<Object[]> resultConsumer) throws ClassCastException
{
return CompletableFuture.runAsync(() ->
{
// TODO what does this mean?
int genChunkWidth = BitShiftUtil.powerOfTwo(granularity - 4);
for (int chunkX = chunkPosMinX; chunkX < chunkPosMinX + genChunkWidth; chunkX++)
for (int chunkX = chunkPosMinX; chunkX < chunkPosMinX + generationRequestChunkWidthCount; chunkX++)
{
for (int chunkZ = chunkPosMinZ; chunkZ < chunkPosMinZ + genChunkWidth; chunkZ++)
for (int chunkZ = chunkPosMinZ; chunkZ < chunkPosMinZ + generationRequestChunkWidthCount; chunkZ++)
{
Object[] rawMcObjectArray = this.generateChunk(chunkX, chunkZ, generatorMode);
resultConsumer.accept(rawMcObjectArray);
@@ -83,7 +76,7 @@ public abstract class AbstractDhApiChunkWorldGenerator implements Closeable, IDh
public final CompletableFuture<Void> generateApiChunks(
int chunkPosMinX,
int chunkPosMinZ,
byte granularity,
int generationRequestChunkWidthCount,
byte targetDataDetail,
EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool,
@@ -92,12 +85,9 @@ public abstract class AbstractDhApiChunkWorldGenerator implements Closeable, IDh
{
return CompletableFuture.runAsync(() ->
{
// TODO what does this mean?
int genChunkWidth = BitShiftUtil.powerOfTwo(granularity - 4);
for (int chunkX = chunkPosMinX; chunkX < chunkPosMinX + genChunkWidth; chunkX++)
for (int chunkX = chunkPosMinX; chunkX < chunkPosMinX + generationRequestChunkWidthCount; chunkX++)
{
for (int chunkZ = chunkPosMinZ; chunkZ < chunkPosMinZ + genChunkWidth; chunkZ++)
for (int chunkZ = chunkPosMinZ; chunkZ < chunkPosMinZ + generationRequestChunkWidthCount; chunkZ++)
{
DhApiChunk apiChunk = this.generateApiChunk(chunkX, chunkZ, generatorMode);
resultConsumer.accept(apiChunk);
@@ -115,10 +105,10 @@ public abstract class AbstractDhApiChunkWorldGenerator implements Closeable, IDh
* @param chunkPosZ the chunk Z position in the level (not to be confused with the chunk's BlockPos in the level)
* @param generatorMode how far into the world gen pipeline this method should run. See {@link EDhApiDistantGeneratorMode} for additional documentation.
*
* @return See {@link IDhApiWorldGenerator#generateChunks(int, int, byte, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer) IDhApiWorldGenerator.generateChunks}
* @return See {@link IDhApiWorldGenerator#generateChunks(int, int, int, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer) IDhApiWorldGenerator.generateChunks}
* for the list of Object's this method should return along with additional documentation.
*
* @see IDhApiWorldGenerator#generateChunks(int, int, byte, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer) IDhApiWorldGenerator#generateChunks
* @see IDhApiWorldGenerator#generateChunks(int, int, int, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer) IDhApiWorldGenerator#generateChunks
*/
public abstract Object[] generateChunk(int chunkPosX, int chunkPosZ, EDhApiDistantGeneratorMode generatorMode);
@@ -133,7 +123,7 @@ public abstract class AbstractDhApiChunkWorldGenerator implements Closeable, IDh
* @return A {@link DhApiChunk} with the generated {@link DhApiTerrainDataPoint} including air blocks.
* Note: if air blocks aren't included with the proper lighting, lower detail levels will appear as black/unlit.
*
* @see IDhApiWorldGenerator#generateApiChunks(int, int, byte, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)
* @see IDhApiWorldGenerator#generateApiChunks(int, int, int, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)
*
* @since API 3.0.0
*/
@@ -24,6 +24,7 @@ import com.seibel.distanthorizons.api.interfaces.override.IDhApiOverrideable;
import com.seibel.distanthorizons.api.enums.EDhApiDetailLevel;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGeneratorMode;
import com.seibel.distanthorizons.api.objects.data.DhApiChunk;
import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
import java.io.Closeable;
import java.util.concurrent.CompletableFuture;
@@ -32,7 +33,7 @@ import java.util.function.Consumer;
/**
* @author James Seibel
* @version 2023-6-22
* @version 2024-10-07
* @since API 1.0.0
*/
public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
@@ -43,19 +44,17 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
/**
* Defines the smallest datapoint size that can be generated at a time. <br>
* Minimum detail level is 0 (1 block) <br>
* Maximum detail level (smallest numerical value) is 0 (1 block) <br>
* Default detail level is 0 <br>
* For more information on what detail levels represent see: {@link EDhApiDetailLevel}. <br><br>
*
* TODO: System currently only supports 1x1 block per data.
*
*
* @see EDhApiDetailLevel
* @since API 1.0.0
*/
default byte getSmallestDataDetailLevel() { return EDhApiDetailLevel.BLOCK.detailLevel; }
/**
* Defines the largest datapoint size that can be generated at a time. <br>
* Minimum detail level is 0 (1 block) <br>
* Maximum detail level (smallest numerical value) is 0 (1 block) <br>
* Default detail level is 0 <br>
* For more information on what detail levels represent see: {@link EDhApiDetailLevel}.
*
@@ -64,56 +63,18 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
*/
default byte getLargestDataDetailLevel() { return EDhApiDetailLevel.BLOCK.detailLevel; }
/**
* When creating generation requests the system will attempt to group nearby tasks together. <br><br>
* What is the minimum size a single generation call can batch together? <br>
*
* Minimum detail level is 4 (the size of a MC chunk) <br>
* Default detail level is 4 <br>
* For more information on what detail levels represent see: {@link EDhApiDetailLevel}.
*
* @see EDhApiDetailLevel
* @since API 1.0.0
*/
default byte getMinGenerationGranularity() { return EDhApiDetailLevel.CHUNK.detailLevel; }
/**
* When creating generation requests the system will attempt to group nearby tasks together. <br><br>
* What is the maximum size a single generation call can batch together? <br>
*
* Minimum detail level is 4 (the size of a MC chunk) <br>
* Default detail level is 6 (4x4 chunks) <br>
* For more information on what detail levels represent see: {@link EDhApiDetailLevel}.
*
* @see EDhApiDetailLevel
* @since API 1.0.0
*/
default byte getMaxGenerationGranularity() { return (byte) (EDhApiDetailLevel.CHUNK.detailLevel + 2); }
/**
* Starting in API 3.0.0 DH now handles future queuing/management internally. <br><br>
*
* Previous description: <br>
* true if the generator is unable to accept new generation requests. <br>
*
* @since API 1.0.0
* @deprecated API 3.0.0
*/
@Deprecated
default boolean isBusy() { return false; }
/**
* Only used if {@link #getReturnType()} returns {@link EDhApiWorldGeneratorReturnType#API_CHUNKS}. <Br>
* If true DH will run additional validation on the {@link DhApiChunk}'s returned. <Br>
* Used if {@link #getReturnType()} returns {@link EDhApiWorldGeneratorReturnType#API_CHUNKS} or {@link EDhApiWorldGeneratorReturnType#API_DATA_SOURCES}. <Br>
* If true DH will run additional validation on the {@link DhApiChunk} or {@link IDhApiFullDataSource}'s returned. <Br>
* This should be disabled during release but should be enabled during development to help spot issues with your data format.
*
* @see #getReturnType()
* @see DhApiChunk
* @see IDhApiFullDataSource
* @see EDhApiWorldGeneratorReturnType#API_CHUNKS
* @since API 3.0.0
* @since API 4.0.0
*/
default boolean runApiChunkValidation() { return true; }
default boolean runApiValidation() { return true; }
@@ -143,9 +104,9 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
*
* @param chunkPosMinX the chunk X position closest to negative infinity
* @param chunkPosMinZ the chunk Z position closest to negative infinity
* @param granularity TODO find a central location to store the definition of granularity. For now it is stored in the Core method: WorldGenerationQueue#startGenerationEvent
* @param generationRequestChunkWidthCount how many chunks wide you should generate
* @param targetDataDetail the LOD Detail level requested to generate. See {@link EDhApiDetailLevel} for additional information.
* @param generatorMode how far into the world gen pipeline this method run. See {@link EDhApiDistantGeneratorMode} for additional documentation.
* @param generatorMode how far into the world gen pipeline this method should run. See {@link EDhApiDistantGeneratorMode} for additional documentation.
* @param worldGeneratorThreadPool the thread pool that should be used when generating the returned {@link CompletableFuture}.
* @param resultConsumer the consumer that should be fired whenever a chunk finishes generating.
*
@@ -156,7 +117,7 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
default CompletableFuture<Void> generateChunks(
int chunkPosMinX,
int chunkPosMinZ,
byte granularity,
int generationRequestChunkWidthCount,
byte targetDataDetail,
EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool,
@@ -165,7 +126,7 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
{
throw new UnsupportedOperationException();
}
/**
* This method is called by Distant Horizons to generate terrain over a given area when
* {@link #getReturnType()} returns {@link EDhApiWorldGeneratorReturnType#API_CHUNKS}. <br><br>
@@ -179,9 +140,9 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
*
* @param chunkPosMinX the chunk X position closest to negative infinity
* @param chunkPosMinZ the chunk Z position closest to negative infinity
* @param granularity TODO find a central location to store the definition of granularity. For now it is stored in the Core method: WorldGenerationQueue#startGenerationEvent
* @param generationRequestChunkWidthCount how many chunks wide you should generate
* @param targetDataDetail the LOD Detail level requested to generate. See {@link EDhApiDetailLevel} for additional information.
* @param generatorMode how far into the world gen pipeline this method run. See {@link EDhApiDistantGeneratorMode} for additional documentation.
* @param generatorMode how far into the world gen pipeline this method should run. See {@link EDhApiDistantGeneratorMode} for additional documentation.
* @param worldGeneratorThreadPool the thread pool that should be used when generating the returned {@link CompletableFuture}.
* @param resultConsumer the consumer that should be fired whenever a chunk finishes generating.
*
@@ -192,7 +153,7 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
default CompletableFuture<Void> generateApiChunks(
int chunkPosMinX,
int chunkPosMinZ,
byte granularity,
int generationRequestChunkWidthCount,
byte targetDataDetail,
EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool,
@@ -201,11 +162,51 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
{
throw new UnsupportedOperationException();
}
/**
* This method is called by Distant Horizons to generate terrain over a given area when
* {@link #getReturnType()} returns {@link EDhApiWorldGeneratorReturnType#API_DATA_SOURCES}. <br><br>
*
* After the {@link IDhApiWorldGenerator} has been generated, it should be passed into the
* resultConsumer's {@link Consumer#accept(Object)} method.
* Note: if air blocks aren't included in the with the {@link DhApiChunk} with proper lighting, lower detail levels will appear as black/unlit.
*
* @implNote the default implementation of this method throws an {@link UnsupportedOperationException},
* and must be overridden when {@link #getReturnType()} returns {@link EDhApiWorldGeneratorReturnType#API_CHUNKS}.
*
* @param chunkPosMinX the chunk X position closest to negative infinity
* @param chunkPosMinZ the chunk Z position closest to negative infinity
* @param lodPosX the LOD's X position, relative to the given {@link EDhApiDetailLevel}
* @param lodPosZ the LOD's Z position, relative to the given {@link EDhApiDetailLevel}
* @param detailLevel the LOD Detail level requested to generate. See {@link EDhApiDetailLevel} for additional information.
* @param pooledFullDataSource The data source you should populate during your world generation.
* This data source is pooled by DH and may be reused multiple times by different internal DH systems. <br>
* This data source should <strong>not</strong> be referenced or stored outside of this method nor the executor provided by worldGeneratorThreadPool.
* <strong>Attempting to do so will corrupt DH's data.</strong>
* @param generatorMode how far into the world gen pipeline this method should run. See {@link EDhApiDistantGeneratorMode} for additional documentation.
* @param worldGeneratorThreadPool the thread pool that should be used when generating the returned {@link CompletableFuture}.
* @param resultConsumer the consumer that should be fired whenever a chunk finishes generating.
*
* @return a future that should run on the worldGeneratorThreadPool and complete once the given generation task has completed.
*
* @since API 4.0.0
*/
default CompletableFuture<Void> generateLod(
int chunkPosMinX, int chunkPosMinZ,
int lodPosX, int lodPosZ, byte detailLevel,
IDhApiFullDataSource pooledFullDataSource,
EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool,
Consumer<IDhApiFullDataSource> resultConsumer
)
{
throw new UnsupportedOperationException();
}
/**
* This method controls how Distant Horizons requests generated chunks.
* By default, the return value is {@link EDhApiWorldGeneratorReturnType#VANILLA_CHUNKS},
* which means that {@link #generateChunks(int, int, byte, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* which means that {@link #generateChunks(int, int, int, byte, EDhApiDistantGeneratorMode, ExecutorService, Consumer)}
* will be invoked whenever Distant Horizons wants to generate terrain with this world generator.
*
* @since API 2.0.0
@@ -238,4 +239,5 @@ public interface IDhApiWorldGenerator extends Closeable, IDhApiOverrideable
void close();
}
@@ -0,0 +1,45 @@
package com.seibel.distanthorizons.api.objects.data;
import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator;
import java.util.List;
/**
* Represents a single full LOD backed by Distant Horizons' ID system.
*
* @see IDhApiWorldGenerator
* @since API 4.0.0
*/
public interface IDhApiFullDataSource
{
/** @return how many data columns wide this data source is */
int getWidthInDataColumns();
/**
* Sets the data column at the relative X and Z position to the list given.
* The given list may be resorted based on the internal format DH requires.
*
* @param relX can be in the range 0 to {@link IDhApiFullDataSource#getWidthInDataColumns()}-1 (both inclusive)
* @param relZ can be in the range 0 to {@link IDhApiFullDataSource#getWidthInDataColumns()}-1 (both inclusive)
*
* @return the same columnDataPoints list after it has been imported into the data source.
* The returned list and contained objects can then be re-used.
*
* @throws IndexOutOfBoundsException if the relative positions are negative or outside the bounds of this data source.
*/
List<DhApiTerrainDataPoint> setApiDataPointColumn(int relX, int relZ, List<DhApiTerrainDataPoint> columnDataPoints)
throws IndexOutOfBoundsException, IllegalArgumentException;
/**
* @param relX can be in the range 0 to {@link IDhApiFullDataSource#getWidthInDataColumns()}-1 (both inclusive)
* @param relZ can be in the range 0 to {@link IDhApiFullDataSource#getWidthInDataColumns()}-1 (both inclusive)
*
* @return a {@link List} of {@link DhApiTerrainDataPoint} representing the data for the given relative position.
*
* @throws IndexOutOfBoundsException if the relative positions are negative or outside the bounds of this data source.
*/
List<DhApiTerrainDataPoint> getApiDataPointColumn(int relX, int relZ) throws IndexOutOfBoundsException;
}
@@ -21,6 +21,8 @@ package com.seibel.distanthorizons.core.dataObjects.fullData.sources;
import com.seibel.distanthorizons.api.enums.config.EDhApiWorldCompressionMode;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.api.objects.data.DhApiTerrainDataPoint;
import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.FullDataPointIdMap;
import com.seibel.distanthorizons.core.dataObjects.transformers.LodDataBuilder;
import com.seibel.distanthorizons.core.file.AbstractDataSourceHandler;
@@ -29,6 +31,7 @@ import com.seibel.distanthorizons.core.file.IDataSource;
import com.seibel.distanthorizons.core.level.IDhLevel;
import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.util.DhApiTerrainDataPointUtil;
import com.seibel.distanthorizons.core.util.FullDataPointUtil;
import com.seibel.distanthorizons.core.util.LodUtil;
import com.seibel.distanthorizons.core.util.RenderDataPointUtil;
@@ -40,7 +43,9 @@ import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* This data source contains every datapoint over its given {@link DhSectionPos}. <br><br>
@@ -48,7 +53,7 @@ import java.util.Arrays;
* @see FullDataPointUtil
* @see FullDataSourceV1
*/
public class FullDataSourceV2 implements IDataSource<IDhLevel>
public class FullDataSourceV2 implements IDataSource<IDhLevel>, IDhApiFullDataSource
{
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
/** useful for debugging, but can slow down update operations quite a bit due to being called so often. */
@@ -112,6 +117,9 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
public boolean isEmpty;
public boolean applyToParent = false;
/** should only be used by methods exposed by the DH API */
private boolean runApiChunkValidation = false;
//==============//
@@ -870,6 +878,10 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
Arrays.fill(dataSource.columnGenerationSteps, (byte) 0);
Arrays.fill(dataSource.columnWorldCompressionMode, (byte) 0);
}
// default to API validation disabled so it's opt-in
// to reduce the chance of performance loss with unnecessary validation
dataSource.setRunApiChunkValidation(false);
}
@@ -916,6 +928,59 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
//=============//
// API methods //
//=============//
public void setRunApiChunkValidation(boolean runValidation) { this.runApiChunkValidation = runValidation; }
@Override
public int getWidthInDataColumns() { return WIDTH; }
@Override
public List<DhApiTerrainDataPoint> setApiDataPointColumn(int relX, int relZ, List<DhApiTerrainDataPoint> columnDataPoints)
throws IndexOutOfBoundsException, IllegalArgumentException
{
try
{
LodDataBuilder.correctDataColumnOrder(columnDataPoints);
if (this.runApiChunkValidation)
{
LodDataBuilder.validateOrThrowApiDataColumn(columnDataPoints);
}
LongArrayList packedDataPoints = LodDataBuilder.convertApiDataPointListToPackedLongArray(columnDataPoints, this, 0);
// TODO there should be an "unknown" compression and generation step, or be defined via the datapoints
this.setSingleColumn(packedDataPoints, relX, relZ, EDhApiWorldGenerationStep.LIGHT, EDhApiWorldCompressionMode.MERGE_SAME_BLOCKS);
return columnDataPoints;
}
catch (DataCorruptedException e)
{
throw new IllegalArgumentException(e.getMessage(), e);
}
}
@Override
public List<DhApiTerrainDataPoint> getApiDataPointColumn(int relX, int relZ) throws IndexOutOfBoundsException
{
LongArrayList dataColumn = this.get(relX, relZ);
ArrayList<DhApiTerrainDataPoint> apiList = new ArrayList<>();
for (int i = 0; i < dataColumn.size(); i++)
{
long datapoint = dataColumn.getLong(i);
DhApiTerrainDataPoint apiDataPoint = DhApiTerrainDataPointUtil.createApiDatapoint(this.levelMinY, this.mapping, DhSectionPos.getDetailLevel(this.pos), datapoint);
apiList.add(apiDataPoint);
}
return apiList;
}
//================//
// base overrides //
//================//
@@ -967,8 +1032,6 @@ public class FullDataSourceV2 implements IDataSource<IDhLevel>
@Override
public void close() throws Exception
{
DATA_SOURCE_POOL.returnPooledDataSource(this);
}
{ DATA_SOURCE_POOL.returnPooledDataSource(this); }
}
@@ -45,6 +45,7 @@ import com.seibel.distanthorizons.core.wrapperInterfaces.world.IBiomeWrapper;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
public class LodDataBuilder
{
@@ -70,7 +71,7 @@ public class LodDataBuilder
int sectionPosZ = getXOrZSectionPosFromChunkPos(chunkWrapper.getChunkPos().getZ());
long pos = DhSectionPos.encode(DhSectionPos.SECTION_BLOCK_DETAIL_LEVEL, sectionPosX, sectionPosZ);
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos);
FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos);
dataSource.isEmpty = false;
@@ -311,44 +312,19 @@ public class LodDataBuilder
int relSourceBlockX = Math.floorMod(apiChunk.chunkPosX, 4) * LodUtil.CHUNK_WIDTH;
int relSourceBlockZ = Math.floorMod(apiChunk.chunkPosZ, 4) * LodUtil.CHUNK_WIDTH;
FullDataSourceV2 dataSource = FullDataSourceV2.createEmpty(pos);
FullDataSourceV2 dataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(pos);
for (int relBlockZ = 0; relBlockZ < LodUtil.CHUNK_WIDTH; relBlockZ++)
{
for (int relBlockX = 0; relBlockX < LodUtil.CHUNK_WIDTH; relBlockX++)
{
List<DhApiTerrainDataPoint> columnDataPoints = apiChunk.getDataPoints(relBlockX, relBlockZ);
LodDataBuilder.correctDataColumnOrder(columnDataPoints);
if (runAdditionalValidation)
{
validateOrThrowDataColumn(columnDataPoints);
validateOrThrowApiDataColumn(columnDataPoints);
}
// this null check does 2 nice things at the same time:
// if columnDataPoints is null,
// then packedDataPoints will be of length 0
// AND the below loop won't run.
int size = (columnDataPoints != null) ? columnDataPoints.size() : 0;
// TODO make missing air LODs
// TODO merge duplicate datapoints
LongArrayList packedDataPoints = new LongArrayList(new long[size]);
for (int index = 0; index < size; index++)
{
DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index);
int id = dataSource.mapping.addIfNotPresentAndGetId(
(IBiomeWrapper) (dataPoint.biomeWrapper),
(IBlockStateWrapper) (dataPoint.blockStateWrapper)
);
packedDataPoints.set(index, FullDataPointUtil.encode(
id,
dataPoint.topYBlockPos - dataPoint.bottomYBlockPos,
dataPoint.bottomYBlockPos - apiChunk.bottomYBlockPos,
(byte) (dataPoint.blockLightLevel),
(byte) (dataPoint.skyLightLevel)
));
}
LongArrayList packedDataPoints = convertApiDataPointListToPackedLongArray(columnDataPoints, dataSource, apiChunk.bottomYBlockPos);
// TODO add the ability for API users to define a different compression mode
// or add a "unkown" compression mode
@@ -361,7 +337,50 @@ public class LodDataBuilder
}
return dataSource;
}
private static void validateOrThrowDataColumn(List<DhApiTerrainDataPoint> dataPoints) throws IllegalArgumentException
//================//
// public helpers //
//================//
/** @see FullDataPointUtil */
public static LongArrayList convertApiDataPointListToPackedLongArray(
@Nullable List<DhApiTerrainDataPoint> columnDataPoints, FullDataSourceV2 dataSource,
int bottomYBlockPos) throws DataCorruptedException
{
// this null check does 2 nice things at the same time:
// if columnDataPoints is null,
// then packedDataPoints will be of length 0
// AND the below loop won't run.
int size = (columnDataPoints != null) ? columnDataPoints.size() : 0;
// TODO make missing air LODs
// TODO merge duplicate datapoints
LongArrayList packedDataPoints = new LongArrayList(new long[size]);
for (int index = 0; index < size; index++)
{
DhApiTerrainDataPoint dataPoint = columnDataPoints.get(index);
int id = dataSource.mapping.addIfNotPresentAndGetId(
(IBiomeWrapper) (dataPoint.biomeWrapper),
(IBlockStateWrapper) (dataPoint.blockStateWrapper)
);
packedDataPoints.set(index, FullDataPointUtil.encode(
id,
dataPoint.topYBlockPos - dataPoint.bottomYBlockPos,
dataPoint.bottomYBlockPos - bottomYBlockPos,
(byte) (dataPoint.blockLightLevel),
(byte) (dataPoint.skyLightLevel)
));
}
return packedDataPoints;
}
/** also corrects the order if it's backwards */
public static void correctDataColumnOrder(List<DhApiTerrainDataPoint> dataPoints)
{
// order doesn't need to be checked if there is 0 or 1 items
if (dataPoints.size() > 1)
@@ -376,9 +395,10 @@ public class LodDataBuilder
}
}
}
public static void validateOrThrowApiDataColumn(List<DhApiTerrainDataPoint> dataPoints) throws IllegalArgumentException
{
// check that each datapoint is valid
int lastBottomYPos = Integer.MIN_VALUE;
for (int i = 0; i < dataPoints.size(); i++) // standard for-loop used instead of an enhanced for-loop to slightly reduce GC overhead due to iterator allocation
@@ -432,7 +452,6 @@ public class LodDataBuilder
//================//
// helper methods //
//================//
@@ -52,7 +52,7 @@ public class DelayedFullDataSourceSaveCache
{
if (temporaryDataSource == null)
{
temporaryDataSource = FullDataSourceV2.createEmpty(inputPos);
temporaryDataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(inputPos);
}
temporaryDataSource.update(inputDataSource);
@@ -104,6 +104,14 @@ public class DelayedFullDataSourceSaveCache
public int getUnsavedCount() { return this.dataSourceByPosition.size(); }
public void flush()
{
this.saveTimerTasksBySectionPos.forEach((pos, timerTask)->
{
timerTask.run();
});
}
//================//
@@ -199,6 +199,9 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
if (this.delayedFullDataSourceSaveCache.getUnsavedCount() >= maxQueueCount)
{
// flushing since we're waiting for this timer to expire anyway
this.delayedFullDataSourceSaveCache.flush();
// don't queue additional world gen requests if there are
// a lot of data sources in memory
// (this is done to prevent infinite memory growth)
@@ -391,6 +394,7 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
// TODO may not be needed
private class WorldGenTaskTracker implements IWorldGenTaskTracker
{
/** just used when debugging/troubleshooting */
private final long pos;
public WorldGenTaskTracker(long pos) { this.pos = pos; }
@@ -413,7 +417,19 @@ public class GeneratedFullDataSourceProvider extends FullDataSourceProviderV2 im
// allows us to reduce cross-chunk lighting issues by lighting the whole 4x4 LOD at once
DhLightingEngine.INSTANCE.bakeDataSourceSkyLight(fullDataSource, LodUtil.MAX_MC_LIGHT);
GeneratedFullDataSourceProvider.this.updateDataSourceAsync(fullDataSource);
GeneratedFullDataSourceProvider.this.updateDataSourceAsync(fullDataSource)
.thenRun(() ->
{
try
{
// send this datasource back to the pool to hopefully reduce GC overhead
fullDataSource.close();
}
catch (Exception e)
{
LOGGER.error("Unexpected issue closing full data source", e);
}
});
}
@@ -83,11 +83,6 @@ public class BatchGenerator implements IDhApiWorldGenerator
@Override
public byte getLargestDataDetailLevel() { return LodUtil.BLOCK_DETAIL_LEVEL; }
@Override
public byte getMinGenerationGranularity() { return LodUtil.CHUNK_DETAIL_LEVEL; }
@Override
public byte getMaxGenerationGranularity() { return LodUtil.CHUNK_DETAIL_LEVEL + 2; }
@@ -97,7 +92,7 @@ public class BatchGenerator implements IDhApiWorldGenerator
@Override
public CompletableFuture<Void> generateChunks(
int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode,
int chunkPosMinX, int chunkPosMinZ, int generationRequestChunkWidthCount, byte targetDataDetail, EDhApiDistantGeneratorMode generatorMode,
ExecutorService worldGeneratorThreadPool, Consumer<Object[]> resultConsumer)
{
EDhApiWorldGenerationStep targetStep = null;
@@ -118,13 +113,11 @@ public class BatchGenerator implements IDhApiWorldGenerator
break;
}
int genChunkSize = BitShiftUtil.powerOfTwo(granularity - 4); // minus 4 is equal to dividing by 16 to convert to chunk scale
// the consumer needs to be wrapped like this because the API can't use DH core objects (and IChunkWrapper can't be easily put into the API project)
Consumer<IChunkWrapper> consumerWrapper = (chunkWrapper) -> resultConsumer.accept(new Object[]{chunkWrapper});
try
{
return this.generationEnvironment.generateChunks(chunkPosMinX, chunkPosMinZ, genChunkSize, targetStep, worldGeneratorThreadPool, consumerWrapper);
return this.generationEnvironment.generateChunks(chunkPosMinX, chunkPosMinZ, generationRequestChunkWidthCount, targetStep, worldGeneratorThreadPool, consumerWrapper);
}
catch (Exception e)
{
@@ -23,6 +23,7 @@ import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiDistantGenerat
import com.seibel.distanthorizons.api.interfaces.override.worldGenerator.IDhApiWorldGenerator;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGeneratorReturnType;
import com.seibel.distanthorizons.api.objects.data.DhApiChunk;
import com.seibel.distanthorizons.api.objects.data.IDhApiFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
import com.seibel.distanthorizons.core.generation.tasks.*;
@@ -43,7 +44,7 @@ import com.seibel.distanthorizons.core.util.threading.ThreadPoolUtil;
import com.seibel.distanthorizons.core.world.DhApiWorldProxy;
import com.seibel.distanthorizons.core.wrapperInterfaces.IWrapperFactory;
import com.seibel.distanthorizons.core.wrapperInterfaces.chunk.IChunkWrapper;
import it.unimi.dsi.fastutil.longs.LongArrayFIFOQueue;
import com.seibel.distanthorizons.coreapi.util.BitShiftUtil;
import org.apache.logging.log4j.Logger;
import java.awt.*;
@@ -74,10 +75,6 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
private final ConcurrentHashMap<Long, InProgressWorldGenTaskGroup> inProgressGenTasksByLodPos = new ConcurrentHashMap<>();
// granularity is the detail level for batching world generator requests together
public final byte maxGranularity;
public final byte minGranularity;
/** largest numerical detail level allowed */
public final byte lowestDataDetail;
/** smallest numerical detail level allowed */
@@ -95,15 +92,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
private final ExecutorService queueingThread = ThreadUtil.makeSingleThreadPool("World Gen Queue");
private boolean generationQueueRunning = false;
private DhBlockPos2D generationTargetPos = DhBlockPos2D.ZERO;
/** can be used for debugging how many tasks are currently in the queue */
private int numberOfTasksQueued = 0;
// debug variables to test for duplicate world generator requests //
/** limits how many of the previous world gen requests we should track */
private static final int MAX_ALREADY_GENERATED_COUNT = 100;
private final HashMap<Long, StackTraceElement[]> alreadyGeneratedPosHashSet = new HashMap<>(MAX_ALREADY_GENERATED_COUNT);
private final LongArrayFIFOQueue alreadyGeneratedPosQueue = new LongArrayFIFOQueue();
/** just used for rendering to the F3 menu */
private int estimatedTotalTaskCount = 0;
@@ -117,20 +106,9 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
{
LOGGER.info("Creating world gen queue");
this.generator = generator;
this.maxGranularity = generator.getMaxGenerationGranularity();
this.minGranularity = generator.getMinGenerationGranularity();
this.lowestDataDetail = generator.getLargestDataDetailLevel();
this.highestDataDetail = generator.getSmallestDataDetailLevel();
if (this.minGranularity < LodUtil.CHUNK_DETAIL_LEVEL)
{
throw new IllegalArgumentException(IDhApiWorldGenerator.class.getSimpleName() + ": min granularity must be at least 4 (Chunk sized)!");
}
if (this.maxGranularity < this.minGranularity)
{
throw new IllegalArgumentException(IDhApiWorldGenerator.class.getSimpleName() + ": max granularity smaller than min granularity!");
}
DebugRenderer.register(this, Config.Client.Advanced.Debugging.DebugWireframe.showWorldGenQueue);
LOGGER.info("Created world gen queue");
}
@@ -349,44 +327,15 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
{
byte taskDetailLevel = newTaskGroup.group.dataDetail;
long taskPos = newTaskGroup.group.pos;
byte granularity = (byte) (DhSectionPos.getDetailLevel(taskPos) - taskDetailLevel);
LodUtil.assertTrue(granularity >= this.minGranularity && granularity <= this.maxGranularity);
LodUtil.assertTrue(taskDetailLevel >= this.highestDataDetail && taskDetailLevel <= this.lowestDataDetail);
DhChunkPos chunkPosMin = new DhChunkPos(DhSectionPos.getSectionBBoxPos(taskPos).getCornerBlockPos());
// check if this is a duplicate generation task
if (this.alreadyGeneratedPosHashSet.containsKey(newTaskGroup.group.pos))
{
// temporary solution to prevent generating the same section multiple times
//LOGGER.trace("Duplicate generation section " + taskPos + " with granularity [" + granularity + "] at " + chunkPosMin + ". Skipping...");
// sending a success result is necessary to make sure the render sections are reloaded correctly
newTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(DhSectionPos.encode(granularity, DhSectionPos.getX(taskPos), DhSectionPos.getZ(taskPos)))));
return false;
}
this.alreadyGeneratedPosHashSet.put(newTaskGroup.group.pos, Thread.currentThread().getStackTrace());
this.alreadyGeneratedPosQueue.enqueue(newTaskGroup.group.pos);
// remove extra tracked duplicate positions
while (this.alreadyGeneratedPosQueue.size() > MAX_ALREADY_GENERATED_COUNT)
{
long posToRemove = this.alreadyGeneratedPosQueue.dequeueLong();
this.alreadyGeneratedPosHashSet.remove(posToRemove);
}
//LOGGER.info("Generating section "+taskPos+" with granularity "+granularity+" at "+chunkPosMin);
this.numberOfTasksQueued++;
newTaskGroup.genFuture = this.startGenerationEvent(chunkPosMin, taskPos, granularity, taskDetailLevel, newTaskGroup.group::consumeDataSource);
newTaskGroup.genFuture = this.startGenerationEvent(taskPos, taskDetailLevel, newTaskGroup.group::consumeDataSource);
LodUtil.assertTrue(newTaskGroup.genFuture != null);
newTaskGroup.genFuture.whenComplete((voidObj, exception) ->
{
try
{
this.numberOfTasksQueued--;
if (exception != null)
{
// don't log the shutdown exceptions
@@ -399,7 +348,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
}
else
{
newTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(DhSectionPos.encode(granularity, DhSectionPos.getX(taskPos), DhSectionPos.getZ(taskPos)))));
newTaskGroup.group.worldGenTasks.forEach(worldGenTask -> worldGenTask.future.complete(WorldGenResult.CreateSuccess(taskPos)));
}
boolean worked = this.inProgressGenTasksByLodPos.remove(taskPos, newTaskGroup);
LodUtil.assertTrue(worked, "Unable to find in progress generator task with position ["+DhSectionPos.toString(taskPos)+"]");
@@ -413,35 +362,15 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
this.inProgressGenTasksByLodPos.put(taskPos, newTaskGroup);
return true;
}
/**
* The chunkPos is always aligned to the granularity.
* For example: if the granularity is 4 (chunk sized) with a data detail level of 0 (block sized),
* the chunkPos will be aligned to 16x16 blocks. <br> <br>
*
*
* <strong>Full Granularity definition (as of 2023-6-21): </strong> <br> <br>
*
* world gen actually supports (in theory) generating stuff with a data detail that's higher than the per-block. <br> <br>
*
* Granularity basically means, on a single generation task, how big such group should be, in terms of the data points it will make. <br> <br>
*
* For example, a granularity of 4 means the task will generate a 16 by 16 data points.
* Now, those data points might be per block, or per 4 by 4 blocks. Granularity doesn't say what detail those would be. <br> <br>
*
* Note: currently the core system sends data via the chunk sized container,
* which has the locked granularity of 4 (16 by 16 data columns), and thus generators should at least have min granularity of 4.
* (Gen chunk width in that context means how many 'chunk sized containers' it will fill up.
* Again, note that a 'chunk sized container' isn't necessary 16 by 16 Minecraft blocks wide.
* It only has to contain 16 by 16 columns of data points, in whatever data detail it might be in.)
* (So, with a generator whose only gen data detail is 0, it is the same as a MC chunk.)
*/
private CompletableFuture<Void> startGenerationEvent(
DhChunkPos chunkPosMin,
byte granularity,
long requestPos,
byte targetDataDetail,
Consumer<FullDataSourceV2> dataSourceConsumer
)
{
DhChunkPos chunkPosMin = new DhChunkPos(DhSectionPos.getSectionBBoxPos(requestPos).getCornerBlockPos());
int generationRequestChunkWidthCount = BitShiftUtil.powerOfTwo(DhSectionPos.getDetailLevel(requestPos) - targetDataDetail - 4); // minus 4 is equal to dividing by 16 to convert to chunk scale
EDhApiDistantGeneratorMode generatorMode = Config.Client.Advanced.WorldGenerator.distantGeneratorMode.get();
EDhApiWorldGeneratorReturnType returnType = this.generator.getReturnType();
switch (returnType)
@@ -449,9 +378,8 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
case VANILLA_CHUNKS:
{
return this.generator.generateChunks(
chunkPosMin.getX(),
chunkPosMin.getZ(),
granularity,
chunkPosMin.getX(), chunkPosMin.getZ(),
generationRequestChunkWidthCount,
targetDataDetail,
generatorMode,
ThreadPoolUtil.getWorldGenExecutor(),
@@ -475,9 +403,8 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
case API_CHUNKS:
{
return this.generator.generateApiChunks(
chunkPosMin.getX(),
chunkPosMin.getZ(),
granularity,
chunkPosMin.getX(), chunkPosMin.getZ(),
generationRequestChunkWidthCount,
targetDataDetail,
generatorMode,
ThreadPoolUtil.getWorldGenExecutor(),
@@ -485,7 +412,7 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
{
try
{
FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints, this.generator.runApiChunkValidation());
FullDataSourceV2 dataSource = LodDataBuilder.createFromApiChunkData(dataPoints, this.generator.runApiValidation());
dataSourceConsumer.accept(dataSource);
}
catch (DataCorruptedException | IllegalArgumentException e)
@@ -501,6 +428,39 @@ public class WorldGenerationQueue implements IFullDataSourceRetrievalQueue, IDeb
}
);
}
case API_DATA_SOURCES:
{
// done to reduce GC overhead
FullDataSourceV2 pooledDataSource = FullDataSourceV2.DATA_SOURCE_POOL.getPooledSource(requestPos);
// set here so the API user doesn't have to pass in this value anywhere themselves
pooledDataSource.setRunApiChunkValidation(this.generator.runApiValidation());
return this.generator.generateLod(
chunkPosMin.getX(), chunkPosMin.getZ(),
DhSectionPos.getX(requestPos), DhSectionPos.getZ(requestPos),
(byte) (DhSectionPos.getDetailLevel(requestPos) - DhSectionPos.SECTION_MINIMUM_DETAIL_LEVEL),
pooledDataSource,
generatorMode,
ThreadPoolUtil.getWorldGenExecutor(),
(IDhApiFullDataSource dataSource) ->
{
try
{
dataSourceConsumer.accept((FullDataSourceV2)dataSource);
}
catch (IllegalArgumentException e)
{
LOGGER.error("World generator returned a corrupt data source. Error: [" + e.getMessage() + "]. World generator disabled.", e);
Config.Client.Advanced.WorldGenerator.enableDistantGeneration.set(false);
}
catch (ClassCastException e)
{
LOGGER.error("World generator return type incorrect. Error: [" + e.getMessage() + "]. World generator disabled.", e);
Config.Client.Advanced.WorldGenerator.enableDistantGeneration.set(false);
}
}
);
}
default:
{
Config.Client.Advanced.WorldGenerator.enableDistantGeneration.set(false);
@@ -59,11 +59,6 @@ public class TestWorldGenerator implements IDhApiWorldGenerator
@Override
public byte getLargestDataDetailLevel() { return LodUtil.BLOCK_DETAIL_LEVEL; }
@Override
public byte getMinGenerationGranularity() { return LodUtil.CHUNK_DETAIL_LEVEL; }
@Override
public byte getMaxGenerationGranularity() { return LodUtil.CHUNK_DETAIL_LEVEL + 2; }
//===================//
@@ -74,10 +69,7 @@ public class TestWorldGenerator implements IDhApiWorldGenerator
public void close() { }
@Override
public boolean isBusy() { return false; }
@Override
public CompletableFuture<Void> generateChunks(int chunkPosMinX, int chunkPosMinZ, byte granularity, byte targetDataDetail, EDhApiDistantGeneratorMode maxGenerationStep, ExecutorService executorService, Consumer<Object[]> resultConsumer) { return null; }
public CompletableFuture<Void> generateChunks(int chunkPosMinX, int chunkPosMinZ, int generationRequestChunkWidthCount, byte targetDataDetail, EDhApiDistantGeneratorMode maxGenerationStep, ExecutorService executorService, Consumer<Object[]> resultConsumer) { return null; }
@Override
public void preGeneratorTaskStart() { }