wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Making multi-level works a bit better... Needs more changes to work well though...

Browse Source
This commit is contained in:
TomTheFurry
2022-08-20 21:24:42 +08:00
parent 90fe892291
commit 78ccb8aaf2
6 changed files with 193 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/seibel/lod/core/a7/datatype/full/FullDataSource.java
+38 -16
View File
@@ -18,7 +18,10 @@ import org.apache.logging.log4j.Logger;
import java.io.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
public class FullDataSource extends FullArrayView implements LodDataSource { // 1 chunk
private static final Logger LOGGER = DhLoggerBuilder.getLogger();
@@ -197,25 +200,44 @@ public class FullDataSource extends FullArrayView implements LodDataSource { //
return new FullDataSource(pos);
}
public static boolean neededForPosition(DhSectionPos posToWrite, DhSectionPos posToTest) {
if (!posToWrite.overlaps(posToTest)) return false;
if (posToTest.sectionDetail > posToWrite.sectionDetail) return false;
if (posToWrite.sectionDetail - posToTest.sectionDetail <= SECTION_SIZE_OFFSET) return true;
byte sectPerData = (byte) (1 << (posToWrite.sectionDetail - posToTest.sectionDetail - SECTION_SIZE_OFFSET));
return posToTest.sectionX % sectPerData == 0 && posToTest.sectionZ % sectPerData == 0;
}
public static FullDataSource createFromLower(DhSectionPos pos, FullDataSource[] lower) {
FullDataSource newData = new FullDataSource(pos);
LodUtil.assertTrue(lower.length == 4, "Non direct convert from lower is not yet implemented");
int halfSize = SECTION_SIZE/2;
DhSectionPos cornerPos = pos.getChild(0);
for (FullDataSource lowerSection : lower) {
if (lowerSection==null || lowerSection.isEmpty) continue;
int offsetX = lowerSection.sectionPos.sectionX == cornerPos.sectionX ? 0 : halfSize;
int offsetZ = lowerSection.sectionPos.sectionZ == cornerPos.sectionZ ? 0 : halfSize;
newData.isEmpty = false;
for (int ox = 0; ox < halfSize; ox++) {
for (int oz = 0; oz < halfSize; oz++) {
FullArrayView quadView = lowerSection.subView(2, ox*2, oz*2);
SingleFullArrayView target = newData.get(ox+offsetX, oz+offsetZ);
target.downsampleFrom(quadView);
public void writeFromLower(FullDataSource subData) {
LodUtil.assertTrue(sectionPos.overlaps(subData.sectionPos));
LodUtil.assertTrue(subData.sectionPos.sectionDetail < sectionPos.sectionDetail);
if (!neededForPosition(sectionPos, subData.sectionPos)) return;
DhSectionPos lowerSectPos = subData.sectionPos;
byte detailDiff = (byte) (sectionPos.sectionDetail - subData.sectionPos.sectionDetail);
byte targetDataDetail = getDataDetail();
DhLodPos minDataPos = sectionPos.getCorner(targetDataDetail);
if (detailDiff <= SECTION_SIZE_OFFSET) {
int count = 1 << detailDiff;
int dataPerCount = SECTION_SIZE / count;
DhLodPos subDataPos = lowerSectPos.getSectionBBoxPos().getCorner(targetDataDetail);
int dataOffsetX = subDataPos.x - minDataPos.x;
int dataOffsetZ = subDataPos.z - minDataPos.z;
LodUtil.assertTrue(dataOffsetX >= 0 && dataOffsetX < SECTION_SIZE && dataOffsetZ >= 0 && dataOffsetZ < SECTION_SIZE);
for (int ox = 0; ox < count; ox++) {
for (int oz = 0; oz < count; oz++) {
SingleFullArrayView column = this.get(ox + dataOffsetX, oz + dataOffsetZ);
column.downsampleFrom(subData.subView(dataPerCount, ox * dataPerCount, oz * dataPerCount));
}
}
} else {
// Count == 1
DhLodPos subDataPos = lowerSectPos.getSectionBBoxPos().convertUpwardsTo(targetDataDetail);
int dataOffsetX = subDataPos.x - minDataPos.x;
int dataOffsetZ = subDataPos.z - minDataPos.z;
LodUtil.assertTrue(dataOffsetX >= 0 && dataOffsetX < SECTION_SIZE && dataOffsetZ >= 0 && dataOffsetZ < SECTION_SIZE);
subData.get(0,0).deepCopyTo(get(dataOffsetX, dataOffsetZ));
}
return newData;
}
}
src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java
+1 -1
View File
@@ -20,7 +20,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class GenerationQueue implements AutoCloseable {
final ConcurrentQuadCombinableProviderTree<GenerationResult> cqcpTree = new ConcurrentQuadCombinableProviderTree<>();
IGenerator generator = null;
IGenerator generator = null; //FIXME: This is volatile and need atomic control
private final Logger logger = DhLoggerBuilder.getLogger();
private final ConcurrentHashMap<DhLodPos, CompletableFuture<GenerationResult>> taskMap = new ConcurrentHashMap<>();
private final AtomicReference<ConcurrentHashMap<DhLodPos, CompletableFuture<GenerationResult>>> inProgress = new AtomicReference<>(null);
src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java
+7 -3
View File
@@ -107,12 +107,12 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
} catch (Exception e) {
LOGGER.error("Error closing world generator", e);
}
worldGenerator = null;
renderBufferHandler.close();
renderBufferHandler = null;
renderFileHandler.flushAndSave(); //Ignore the completion feature so that this action is async
renderFileHandler.close();
renderFileHandler = null;
worldGenerator = null;
}
@Override
@@ -142,8 +142,12 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
@Override
public CompletableFuture<Void> save() {
return renderFileHandler == null ? dataFileHandler.flushAndSave() : renderFileHandler.flushAndSave();
//Note: saving renderFileHandler will also save the dataFileHandler.
if (renderFileHandler != null) {
return renderFileHandler.flushAndSave().thenCompose(v -> dataFileHandler.flushAndSave());
} else {
return dataFileHandler.flushAndSave();
}
}
@Override
public void close() {
src/main/java/com/seibel/lod/core/a7/pos/DhLodPos.java
+6 -1
View File
@@ -31,7 +31,12 @@ public class DhLodPos implements Comparable<DhLodPos> {
public int getWidth() {
return 1 << detail;
}
public static int getWidth(byte detail) {
public int getWidth(byte detail) {
LodUtil.assertTrue(detail <= this.detail);
return 1 << (this.detail - detail);
}
public static int blockWidth(byte detail) {
return 1 << detail;
}
src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java
+137 -17
View File
@@ -108,12 +108,7 @@ public class DataFileHandler implements IDataSourceProvider {
}
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
@Override
public CompletableFuture<LodDataSource> read(DhSectionPos pos) {
topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetail));
private DataMetaFile atomicGetOrMakeFile(DhSectionPos pos) {
DataMetaFile metaFile = files.get(pos);
if (metaFile == null) {
File file = computeDefaultFilePath(pos);
@@ -123,22 +118,147 @@ public class DataFileHandler implements IDataSourceProvider {
DataMetaFile newMetaFile = new DataMetaFile(level, file, pos, gen);
metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
if (metaFile == null) {
//FIXME: First check for lower detail level and use them first.
dataSourceCreator.apply(pos).handle((source, ex) -> {
if (ex != null) {
LOGGER.error("Failed to create data source for {}", pos, ex);
gen.completeExceptionally(ex);
} else {
gen.complete(source);
}
return null;
});
buildFile(pos, gen);
metaFile = newMetaFile;
} else {
gen.cancel(true);
}
}
return metaFile;
}
private void selfSearch(DhSectionPos basePos, DhSectionPos pos, ArrayList<DataMetaFile> existFiles, ArrayList<DhSectionPos> missing) {
byte detail = pos.sectionDetail;
boolean allEmpty = true;
outerLoop:
while (--detail >= minDetailLevel) {
DhLodPos min = pos.getCorner().getCorner(detail);
int count = pos.getSectionBBoxPos().getWidth(detail);
for (int ox = 0; ox<count; ox++) {
for (int oz = 0; oz<count; oz++) {
DhSectionPos subPos = new DhSectionPos(detail, ox+min.x, oz+min.z);
LodUtil.assertTrue(pos.overlaps(basePos) && subPos.overlaps(pos));
//TODO: The following check is temp as we only samples corner points per data, which means
// on a very different level, we may not need the entire section at all.
if (!FullDataSource.neededForPosition(basePos, subPos)) continue;
if (files.containsKey(subPos)) {
allEmpty = false;
break outerLoop;
}
}
}
}
if (allEmpty) {
missing.add(pos);
} else {
{
DhSectionPos childPos = pos.getChild(0);
if (FullDataSource.neededForPosition(basePos, childPos)) {
DataMetaFile metaFile = files.get(childPos);
if (metaFile != null) {
existFiles.add(metaFile);
} else if (childPos.sectionDetail == minDetailLevel) {
missing.add(childPos);
} else {
selfSearch(basePos, childPos, existFiles, missing);
}
}
}
{
DhSectionPos childPos = pos.getChild(1);
if (FullDataSource.neededForPosition(basePos, childPos)) {
DataMetaFile metaFile = files.get(childPos);
if (metaFile != null) {
existFiles.add(metaFile);
} else if (childPos.sectionDetail == minDetailLevel) {
missing.add(childPos);
} else {
selfSearch(basePos, childPos, existFiles, missing);
}
}
}
{
DhSectionPos childPos = pos.getChild(2);
if (FullDataSource.neededForPosition(basePos, childPos)) {
DataMetaFile metaFile = files.get(childPos);
if (metaFile != null) {
existFiles.add(metaFile);
} else if (childPos.sectionDetail == minDetailLevel) {
missing.add(childPos);
} else {
selfSearch(basePos, childPos, existFiles, missing);
}
}
}
{
DhSectionPos childPos = pos.getChild(3);
if (FullDataSource.neededForPosition(basePos, childPos)) {
DataMetaFile metaFile = files.get(childPos);
if (metaFile != null) {
existFiles.add(metaFile);
} else if (childPos.sectionDetail == minDetailLevel) {
missing.add(childPos);
} else {
selfSearch(basePos, childPos, existFiles, missing);
}
}
}
}
}
private void buildFile(DhSectionPos pos, CompletableFuture<LodDataSource> gen) {
ArrayList<DataMetaFile> existFiles = new ArrayList<>();
ArrayList<DhSectionPos> missing = new ArrayList<>();
selfSearch(pos, pos, existFiles, missing);
LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) {
dataSourceCreator.apply(pos).whenComplete((f, ex) -> {
if (ex != null) {
gen.completeExceptionally(ex);
} else {
gen.complete(f);
}
});
return;
}
LOGGER.info("Creating file at {} using {} existing files and {} new files.", pos, existFiles.size(), missing.size());
ArrayList<CompletableFuture<LodDataSource>> futures = new ArrayList<>(existFiles.size() + missing.size());
for (DhSectionPos missingPos : missing) {
existFiles.add(atomicGetOrMakeFile(missingPos));
}
FullDataSource fullDataSource = FullDataSource.createEmpty(pos);
for (DataMetaFile metaFile : existFiles) {
futures.add(
metaFile.loadOrGetCached(fileReaderThread).whenComplete((data, ex) -> {
if (ex != null) return;
if (!(data instanceof FullDataSource)) return;
fullDataSource.writeFromLower((FullDataSource) data);
})
);
}
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.whenComplete((v, ex) -> {
if (ex != null) {
gen.completeExceptionally(ex);
} else {
gen.complete(fullDataSource);
}
});
}
/*
* This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
*/
@Override
public CompletableFuture<LodDataSource> read(DhSectionPos pos) {
topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetail));
DataMetaFile metaFile = atomicGetOrMakeFile(pos);
return metaFile.loadOrGetCached(fileReaderThread);
}
src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java
+4 -4
View File
@@ -41,10 +41,10 @@ public class RenderMetaFile extends MetaFile {
}
public CompletableFuture<Void> flushAndSave(ExecutorService renderCacheThread) {
if (!path.exists()) return CompletableFuture.completedFuture(null); // No need to save if the file doesn't exist.
CompletableFuture<LodRenderSource> source = _readCached(data.get());
if (source == null) return CompletableFuture.completedFuture(null);
return source.thenAccept((a)->{});
//TODO: Should we save the data or let user re-calculate it on new load?
if (source == null) return CompletableFuture.completedFuture(null); // If there is no cached data, there is no need to save.
return source.thenAccept((a)->{}); // Otherwise, wait for the data to be read (which also flushes changes to the file).
}
@FunctionalInterface
@@ -94,7 +94,7 @@ public class RenderMetaFile extends MetaFile {
}
}
//==== Cached file out of scrope. ====
//==== Cached file out of scope. ====
// Someone is already trying to complete it. so just return the obj.
if ((obj instanceof CompletableFuture<?>)) {
return (CompletableFuture<LodRenderSource>)obj;