Merge

core/src/main/java/com/seibel/lod/core/a7/datatype/LodDataSource.java

@@ -14,7 +14,6 @@ import java.io.OutputStream;
 public interface LodDataSource {
     DhSectionPos getSectionPos();
     byte getDataDetail();
-    void setLocalVersion(int localVer);
     byte getDataVersion();
 
     void update(ChunkSizedData data);

core/src/main/java/com/seibel/lod/core/a7/datatype/LodRenderSource.java

@@ -13,12 +13,12 @@ import java.util.concurrent.atomic.AtomicReference;
 
 public interface LodRenderSource {
     DhSectionPos getSectionPos();
+    byte getDataDetail();
 
     void enableRender(IClientLevel level, LodQuadTree quadTree);
     void disableRender();
     boolean isRenderReady();
     void dispose(); // notify the container that the parent lodSection is now disposed (can be in loaded or unloaded state)
-    byte getDetailOffset();
 
 
     /**
@@ -32,7 +32,9 @@ public interface LodRenderSource {
 
     void saveRender(IClientLevel level, RenderMetaFile file, OutputStream dataStream) throws IOException;
 
+    @Deprecated
     void write(ChunkSizedData chunkData);
+    @Deprecated
     void flushWrites(IClientLevel level);
 
     byte getRenderVersion();
@@ -41,4 +43,7 @@ public interface LodRenderSource {
      * Whether this object is still valid. If not, a new one should be created.
      */
     boolean isValid();
+
+    // Only override the data that has not been written directly using write(), and skip those that are empty
+    void weakWrite(LodRenderSource source);
 }

core/src/main/java/com/seibel/lod/core/a7/datatype/PlaceHolderRenderSource.java

@@ -23,6 +23,11 @@ public class PlaceHolderRenderSource implements LodRenderSource {
         return pos;
     }
 
+    @Override
+    public byte getDataDetail() {
+        return 0;
+    }
+
     @Override
     public void enableRender(IClientLevel level, LodQuadTree quadTree) {
     }
@@ -37,10 +42,6 @@ public class PlaceHolderRenderSource implements LodRenderSource {
     @Override
     public void dispose() {}
     @Override
-    public byte getDetailOffset() {
-        return 0;
-    }
-    @Override
     public boolean trySwapRenderBuffer(LodQuadTree quadTree, AtomicReference<RenderBuffer> referenceSlotsOpaque, AtomicReference<RenderBuffer> referenceSlotsTransparent) {
         return false;
     }
@@ -68,5 +69,10 @@ public class PlaceHolderRenderSource implements LodRenderSource {
         return isValid;
     }
 
+    @Override
+    public void weakWrite(LodRenderSource source) {
+
+    }
+
 
 }

core/src/main/java/com/seibel/lod/core/a7/datatype/RenderSourceLoader.java

@@ -2,6 +2,7 @@ package com.seibel.lod.core.a7.datatype;
 
 import com.google.common.collect.HashMultimap;
 import com.seibel.lod.core.a7.level.IClientLevel;
+import com.seibel.lod.core.a7.level.ILevel;
 import com.seibel.lod.core.a7.save.io.render.RenderMetaFile;
 
 import java.io.IOException;
@@ -53,7 +54,7 @@ public abstract class RenderSourceLoader {
     }
 
     // Can return null as meaning the file is out of date or something
-    public abstract LodRenderSource loadRender(RenderMetaFile renderFile, InputStream data, IClientLevel level) throws IOException;
+    public abstract LodRenderSource loadRender(RenderMetaFile renderFile, InputStream data, ILevel level) throws IOException;
     public abstract LodRenderSource createRender(LodDataSource dataSource, IClientLevel level);
 
 

core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderLoader.java

@@ -2,11 +2,14 @@ package com.seibel.lod.core.a7.datatype.column;
 
 import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.full.FullDataSource;
+import com.seibel.lod.core.a7.datatype.full.SparseDataSource;
 import com.seibel.lod.core.a7.datatype.transform.FullToColumnTransformer;
 import com.seibel.lod.core.a7.level.IClientLevel;
 import com.seibel.lod.core.a7.datatype.LodRenderSource;
 import com.seibel.lod.core.a7.datatype.RenderSourceLoader;
+import com.seibel.lod.core.a7.level.ILevel;
 import com.seibel.lod.core.a7.save.io.render.RenderMetaFile;
+import com.seibel.lod.core.util.LodUtil;
 
 import java.io.DataInputStream;
 import java.io.IOException;
@@ -18,12 +21,12 @@ public class ColumnRenderLoader extends RenderSourceLoader {
     }
 
     @Override
-    public LodRenderSource loadRender(RenderMetaFile dataFile, InputStream data, IClientLevel level) throws IOException {
+    public LodRenderSource loadRender(RenderMetaFile dataFile, InputStream data, ILevel level) throws IOException {
         try (
                 //TODO: Add decompressor here
                 DataInputStream dis = new DataInputStream(data);
         ) {
-            return new ColumnRenderSource(dataFile.pos, dis, dataFile.loaderVersion, level);
+            return new ColumnRenderSource(dataFile.pos, dis, dataFile.metaData.loaderVersion, level);
         }
     }
 
@@ -31,7 +34,10 @@ public class ColumnRenderLoader extends RenderSourceLoader {
     public LodRenderSource createRender(LodDataSource dataSource, IClientLevel level) {
         if (dataSource instanceof FullDataSource) {
             return FullToColumnTransformer.transformFullDataToColumnData(level, (FullDataSource) dataSource);
+        } else if (dataSource instanceof SparseDataSource) {
+            return FullToColumnTransformer.transformSparseDataToColumnData(level, (SparseDataSource) dataSource);
         }
+        LodUtil.assertNotReach();
         return null;
     }
 

core/src/main/java/com/seibel/lod/core/a7/datatype/column/ColumnRenderSource.java

@@ -210,6 +210,9 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype {
             for (int j = 0; j < verticalSize; j++)
             {
                 long current = dataContainer[i * verticalSize + j];
+				// TODO
+//                if (ColumnFormat.doesItExist(current))
+//                    current = ColumnFormat.overrideGenerationMode(current, (byte) 1);
                 output.writeLong(Long.reverseBytes(current));
             }
             if (!ColumnFormat.doesItExist(dataContainer[i]))
@@ -402,4 +405,26 @@ public class ColumnRenderSource implements LodRenderSource, IColumnDatatype {
     public boolean isValid() {
         return true;
     }
+
+    @Override
+    public void weakWrite(LodRenderSource source) {
+        LodUtil.assertTrue(source instanceof ColumnRenderSource);
+        ColumnRenderSource src = (ColumnRenderSource) source;
+
+        LodUtil.assertTrue(src.sectionPos.equals(sectionPos));
+        LodUtil.assertTrue(src.verticalSize == verticalSize);
+
+        if (src.isEmpty) return;
+        isEmpty = false;
+
+        for (int i=0; i<dataContainer.length; i+=verticalSize) {
+            int genMode = ColumnFormat.getGenerationMode(dataContainer[i]);
+            int srcGenMode = ColumnFormat.getGenerationMode(src.dataContainer[i]);
+            if (srcGenMode == 0) continue;
+            if (genMode <= srcGenMode) {
+                new ColumnArrayView(dataContainer, verticalSize, i, verticalSize).copyFrom(
+                    new ColumnArrayView(src.dataContainer, verticalSize, i, verticalSize));
+            }
+        }
+    }
 }

core/src/main/java/com/seibel/lod/core/a7/datatype/full/FullDataSource.java

@@ -30,7 +30,6 @@ public class FullDataSource extends FullArrayView implements LodDataSource { //
     public static final byte LATEST_VERSION = 0;
     public static final long TYPE_ID = "FullDataSource".hashCode();
     private final DhSectionPos sectionPos;
-    private int localVersion = 0;
     public boolean isEmpty = true;
     protected FullDataSource(DhSectionPos sectionPos) {
         super(new IdBiomeBlockStateMap(), new long[SECTION_SIZE*SECTION_SIZE][0], SECTION_SIZE);
@@ -45,10 +44,6 @@ public class FullDataSource extends FullArrayView implements LodDataSource { //
     public byte getDataDetail() {
         return (byte) (sectionPos.sectionDetail-SECTION_SIZE_OFFSET);
     }
-    @Override
-    public void setLocalVersion(int localVer) {
-        localVersion = localVer;
-    }
 
     @Override
     public byte getDataVersion() {
@@ -145,8 +140,8 @@ public class FullDataSource extends FullArrayView implements LodDataSource { //
     public static FullDataSource loadData(DataMetaFile dataFile, InputStream dataStream, ILevel level) throws IOException {
         try (DataInputStream dos = new DataInputStream(dataStream)) {
             int dataDetail = dos.readInt();
-            if(dataDetail != dataFile.dataLevel)
-                throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.dataLevel));
+            if(dataDetail != dataFile.metaData.dataLevel)
+                throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
             int size = dos.readInt();
             if (size != SECTION_SIZE)
                 throw new IOException(LodUtil.formatLog(

core/src/main/java/com/seibel/lod/core/a7/datatype/full/SparseDataSource.java

@@ -66,10 +66,7 @@ public class SparseDataSource implements LodDataSource {
     public byte getDataDetail() {
         return (byte) (sectionPos.sectionDetail-SECTION_SIZE_OFFSET);
     }
-    @Override
-    public void setLocalVersion(int localVer) {
-        //TODO: implement
-    }
+
     @Override
     public byte getDataVersion() {
         return LATEST_VERSION;
@@ -104,9 +101,59 @@ public class SparseDataSource implements LodDataSource {
                 }
             }
         }
+        isEmpty = false;
         sparseData[arrayOffset] = newArray;
     }
 
+    public void sampleFrom(SparseDataSource sparseSource) {
+        DhSectionPos pos = sparseSource.sectionPos;
+        LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail);
+        LodUtil.assertTrue(pos.overlaps(sectionPos));
+        if (sparseSource.isEmpty) return;
+
+        // Downsample needed
+        DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
+        DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL);
+        int offsetX = dataPos.x-basePos.x;
+        int offsetZ = dataPos.z-basePos.z;
+        LodUtil.assertTrue(offsetX >=0 && offsetX < chunks && offsetZ >=0 && offsetZ < chunks);
+
+        for (int ox = 0; ox < sparseSource.chunks; ox++) {
+            for (int oz = 0; oz < sparseSource.chunks; oz++) {
+                FullArrayView sourceChunk = sparseSource.sparseData[ox*sparseSource.chunks + oz];
+                if (sourceChunk != null) {
+                    FullArrayView buff = new FullArrayView(mapping, new long[dataPerChunk * dataPerChunk][], dataPerChunk);
+                    buff.downsampleFrom(sourceChunk);
+                    sparseData[(ox+offsetX)* chunks + (oz+offsetZ)] = buff;
+                }
+            }
+        }
+    }
+    public void sampleFrom(FullDataSource fullSource) {
+        DhSectionPos pos = fullSource.getSectionPos();
+        LodUtil.assertTrue(pos.sectionDetail < sectionPos.sectionDetail);
+        LodUtil.assertTrue(pos.overlaps(sectionPos));
+        if (fullSource.isEmpty) return;
+        // Downsample needed
+        DhLodPos basePos = sectionPos.getCorner(SPARSE_UNIT_DETAIL);
+        DhLodPos dataPos = pos.getCorner(SPARSE_UNIT_DETAIL);
+        int coveredChunks = pos.getWidth(SPARSE_UNIT_DETAIL).value;
+        int sourceDataPerChunk = SPARSE_UNIT_SIZE >>> fullSource.getDataDetail();
+        LodUtil.assertTrue(coveredChunks*sourceDataPerChunk == FullDataSource.SECTION_SIZE);
+        int offsetX = dataPos.x-basePos.x;
+        int offsetZ = dataPos.z-basePos.z;
+        LodUtil.assertTrue(offsetX >=0 && offsetX < chunks && offsetZ >=0 && offsetZ < chunks);
+
+        for (int ox = 0; ox < coveredChunks; ox++) {
+            for (int oz = 0; oz < coveredChunks; oz++) {
+                FullArrayView sourceChunk = fullSource.subView(sourceDataPerChunk, ox*sourceDataPerChunk, oz*sourceDataPerChunk);
+                FullArrayView buff = new FullArrayView(mapping, new long[dataPerChunk * dataPerChunk][], dataPerChunk);
+                buff.downsampleFrom(sourceChunk);
+                sparseData[(ox+offsetX)* chunks + (oz+offsetZ)] = buff;
+            }
+        }
+    }
+
     @Override
     public void saveData(ILevel level, DataMetaFile file, OutputStream dataStream) throws IOException {
         try (DataOutputStream dos = new DataOutputStream(dataStream)) {
@@ -160,8 +207,8 @@ public class SparseDataSource implements LodDataSource {
         LodUtil.assertTrue(dataFile.pos.sectionDetail <= MAX_SECTION_DETAIL);
         try (DataInputStream dos = new DataInputStream(dataStream)) {
             int dataDetail = dos.readShort();
-            if(dataDetail != dataFile.dataLevel)
-                throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.dataLevel));
+            if(dataDetail != dataFile.metaData.dataLevel)
+                throw new IOException(LodUtil.formatLog("Data level mismatch: {} != {}", dataDetail, dataFile.metaData.dataLevel));
             int sparseDetail = dos.readShort();
             if (sparseDetail != SPARSE_UNIT_DETAIL)
                 throw new IOException((LodUtil.formatLog("Unexpected sparse detail level: {} != {}",
@@ -204,8 +251,8 @@ public class SparseDataSource implements LodDataSource {
             for (int i = set.nextSetBit(0); i >= 0 && i < dataChunks.length; i = set.nextSetBit(i + 1)) {
                 long[][] dataColumns = new long[dataPerChunk*dataPerChunk][];
                 dataChunks[i] = dataColumns;
-                for (int j = 0; j < dataColumns.length; j++) {
-                    dataColumns[i] = new long[dos.readByte()];
+                for (int i2 = 0; i2 < dataColumns.length; i2++) {
+                    dataColumns[i2] = new long[dos.readByte()];
                 }
                 for (int k = 0; k < dataColumns.length; k++) {
                     if (dataColumns[k].length == 0) continue;
@@ -240,9 +287,43 @@ public class SparseDataSource implements LodDataSource {
                 FullArrayView array = sparseData[x*chunks+z];
                 if (array == null) continue;
                 // Otherwise, apply data to dataSource
+                dataSource.isEmpty = false;
                 FullArrayView view = dataSource.subView(dataPerChunk, x*dataPerChunk, z*dataPerChunk);
                 array.shadowCopyTo(view);
             }
         }
     }
+
+
+    public LodDataSource promote(LodDataSource generatedData) {
+        if (!(generatedData instanceof FullDataSource) && !(generatedData instanceof SparseDataSource))
+            throw new UnsupportedOperationException("Requires FullDataSource for the promotion!");
+        if (generatedData instanceof FullDataSource) {
+            applyToFullDataSource((FullDataSource) generatedData);
+            return generatedData;
+        } else {
+            LodUtil.assertToDo(); //TODO
+            return null;
+        }
+    }
+
+    public LodDataSource trySelfPromote() {
+        if (isEmpty) return this;
+        for (FullArrayView array : sparseData) {
+            if (array == null) return this;
+        }
+        FullDataSource newSource = FullDataSource.createEmpty(sectionPos);
+        applyToFullDataSource(newSource);
+        return newSource;
+    }
+
+    // Return null if doesn't exist
+    public SingleFullArrayView tryGet(int x, int z) {
+        LodUtil.assertTrue(x>=0 && x<SECTION_SIZE && z>=0 && z<SECTION_SIZE);
+        int chunkX = x / dataPerChunk;
+        int chunkZ = z / dataPerChunk;
+        FullArrayView chunk = sparseData[chunkX * chunks + chunkZ];
+        if (chunk == null) return null;
+        return chunk.get(chunkX % dataPerChunk, chunkZ % dataPerChunk);
+    }
 }

core/src/main/java/com/seibel/lod/core/a7/datatype/full/accessor/FullArrayView.java

@@ -2,6 +2,7 @@ package com.seibel.lod.core.a7.datatype.full.accessor;
 
 import com.seibel.lod.core.a7.datatype.full.FullFormat;
 import com.seibel.lod.core.a7.datatype.full.IdBiomeBlockStateMap;
+import com.seibel.lod.core.util.LodUtil;
 
 public class FullArrayView implements IFullDataView {
     protected final long[][] dataArrays;
@@ -80,4 +81,15 @@ public class FullArrayView implements IFullDataView {
             }
         }
     }
+
+    public void downsampleFrom(FullArrayView source) {
+        LodUtil.assertTrue(source.size > size && source.size % size == 0);
+        int dataPerUnit = source.size / size;
+        for (int ox = 0; ox < size; ox++) {
+            for (int oz = 0; oz < size; oz++) {
+                SingleFullArrayView column = get(ox, oz);
+                column.downsampleFrom(source.subView(dataPerUnit, ox * dataPerUnit, oz * dataPerUnit));
+            }
+        }
+    }
 }

core/src/main/java/com/seibel/lod/core/a7/datatype/transform/FullToColumnTransformer.java

@@ -1,13 +1,11 @@
 package com.seibel.lod.core.a7.datatype.transform;
 
+import com.seibel.lod.core.a7.datatype.LodRenderSource;
 import com.seibel.lod.core.a7.datatype.column.accessor.ColumnFormat;
 import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource;
 import com.seibel.lod.core.a7.datatype.column.accessor.ColumnArrayView;
 import com.seibel.lod.core.a7.datatype.column.accessor.ColumnQuadView;
-import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
-import com.seibel.lod.core.a7.datatype.full.FullDataSource;
-import com.seibel.lod.core.a7.datatype.full.FullFormat;
-import com.seibel.lod.core.a7.datatype.full.IdBiomeBlockStateMap;
+import com.seibel.lod.core.a7.datatype.full.*;
 import com.seibel.lod.core.a7.datatype.full.accessor.SingleFullArrayView;
 import com.seibel.lod.core.a7.level.IClientLevel;
 import com.seibel.lod.core.a7.pos.DhSectionPos;
@@ -43,7 +41,7 @@ public class FullToColumnTransformer {
                 for (int z = 0; z < pos.getWidth(dataDetail).value; z++) {
                     ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z);
                     SingleFullArrayView fullArrayView = data.get(x, z);
-                    convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView);
+                    convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView, 1);
                     if (fullArrayView.doesItExist()) LodUtil.assertTrue(columnSource.doesItExist(x, z));
                 }
             }
@@ -67,6 +65,33 @@ public class FullToColumnTransformer {
         return columnSource;
     }
 
+    public static LodRenderSource transformSparseDataToColumnData(IClientLevel level, SparseDataSource data) {
+        final DhSectionPos pos = data.getSectionPos();
+        final byte dataDetail = data.getDataDetail();
+        final int vertSize = Config.Client.Graphics.Quality.verticalQuality.get().calculateMaxVerticalData(data.getDataDetail());
+        final ColumnRenderSource columnSource = new ColumnRenderSource(pos, vertSize, level.getMinY());
+        if (data.isEmpty) return columnSource;
+        columnSource.isEmpty = false;
+
+        if (dataDetail == columnSource.getDataDetail()) {
+            int baseX = pos.getCorner().getCorner().x;
+            int baseZ = pos.getCorner().getCorner().z;
+            for (int x = 0; x < pos.getWidth(dataDetail).value; x++) {
+                for (int z = 0; z < pos.getWidth(dataDetail).value; z++) {
+                    SingleFullArrayView fullArrayView = data.tryGet(x, z);
+                    if (fullArrayView == null) continue;
+                    ColumnArrayView columnArrayView = columnSource.getVerticalDataView(x, z);
+                    convertColumnData(level, baseX + x, baseZ + z, columnArrayView, fullArrayView, 1);
+                    if (fullArrayView.doesItExist()) LodUtil.assertTrue(columnSource.doesItExist(x, z));
+                }
+            }
+        } else {
+            throw new UnsupportedOperationException("To be implemented");
+            //FIXME: Implement different size creation of renderData
+        }
+        return columnSource;
+    }
+
     public static void writeFullDataChunkToColumnData(ColumnRenderSource render, IClientLevel level, ChunkSizedData data) {
         if (data.dataDetail != 0)
             throw new UnsupportedOperationException("To be implemented");
@@ -87,7 +112,7 @@ public class FullToColumnTransformer {
                     SingleFullArrayView fullArrayView = data.get(x, z);
                     convertColumnData(level, blockX + perRenderWidth * (renderOffsetX+x),
                             blockZ + perRenderWidth * (renderOffsetZ+z),
-                            columnArrayView, fullArrayView);
+                            columnArrayView, fullArrayView, 2);
                     if (fullArrayView.doesItExist()) LodUtil.assertTrue(render.doesItExist(renderOffsetX + x, renderOffsetZ + z));
                 }
             }
@@ -108,7 +133,7 @@ public class FullToColumnTransformer {
                             SingleFullArrayView fullArrayView = data.get(x*dataPerRender+ox, z*dataPerRender+oz);
                             convertColumnData(level, blockX + perRenderWidth * (renderOffsetX+x) + perDataWidth * ox,
                                     blockZ + perRenderWidth * (renderOffsetZ+z) + perDataWidth * oz,
-                                    columnArrayView, fullArrayView);
+                                    columnArrayView, fullArrayView, 2);
                         }
                     }
                     ColumnArrayView downSampledArrayView = render.getVerticalDataView(renderOffsetX + x, renderOffsetZ + z);
@@ -118,19 +143,17 @@ public class FullToColumnTransformer {
         }
     }
 
-    private static void convertColumnData(IClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleFullArrayView fullArrayView) {
+    private static void convertColumnData(IClientLevel level, int blockX, int blockZ, ColumnArrayView columnArrayView, SingleFullArrayView fullArrayView, int genMode) {
         if (!fullArrayView.doesItExist()) return;
-        // TODO: Set gen mode
-        int genModeValue = 1;
         int dataTotalLength = fullArrayView.getSingleLength();
         if (dataTotalLength == 0) return;
 
         if (dataTotalLength > columnArrayView.verticalSize()) {
             ColumnArrayView totalColumnData = new ColumnArrayView(new long[dataTotalLength], dataTotalLength, 0, dataTotalLength);
-            iterateAndConvert(level, blockX, blockZ, genModeValue, totalColumnData, fullArrayView);
+            iterateAndConvert(level, blockX, blockZ, genMode, totalColumnData, fullArrayView);
             columnArrayView.changeVerticalSizeFrom(totalColumnData);
         } else {
-            iterateAndConvert(level, blockX, blockZ, genModeValue, columnArrayView, fullArrayView); //Directly use the arrayView since it fits.
+            iterateAndConvert(level, blockX, blockZ, genMode, columnArrayView, fullArrayView); //Directly use the arrayView since it fits.
         }
     }
 

core/src/main/java/com/seibel/lod/core/a7/generation/GenerationQueue.java

@@ -1,213 +1,398 @@
 package com.seibel.lod.core.a7.generation;
 
-import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
-import com.seibel.lod.core.a7.datatype.full.FullDataSource;
 import com.seibel.lod.core.a7.pos.DhBlockPos2D;
 import com.seibel.lod.core.a7.pos.DhLodPos;
-import com.seibel.lod.core.a7.pos.DhSectionPos;
-import com.seibel.lod.core.a7.util.ConcurrentQuadCombinableProviderTree;
-import com.seibel.lod.core.a7.util.UncheckedInterruptedException;
 import com.seibel.lod.core.logging.DhLoggerBuilder;
 import com.seibel.lod.core.objects.DHChunkPos;
 import com.seibel.lod.core.util.LodUtil;
-import com.seibel.lod.core.util.gridList.ArrayGridList;
 import org.apache.logging.log4j.Logger;
 
+import java.io.Closeable;
 import java.util.*;
 import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+public class GenerationQueue implements Closeable {
+    public static final int SHUTDOWN_TIMEOUT_SEC = 10;
+
 
-public class GenerationQueue implements AutoCloseable {
-    final ConcurrentQuadCombinableProviderTree<GenerationResult> cqcpTree = new ConcurrentQuadCombinableProviderTree<>();
-    IGenerator generator = null; //FIXME: This is volatile and need atomic control
     private final Logger logger = DhLoggerBuilder.getLogger();
-    private final ConcurrentHashMap<DhLodPos, CompletableFuture<GenerationResult>> taskMap = new ConcurrentHashMap<>();
-    private final AtomicReference<ConcurrentHashMap<DhLodPos, CompletableFuture<GenerationResult>>> inProgress = new AtomicReference<>(null);
+    public static abstract class GenTaskTracker {
+        public abstract boolean isValid();
+        public abstract Consumer<ChunkSizedData> getConsumer();
+    }
+
+    final IGenerator generator;
+    static final class GenTask {
+        final DhLodPos pos;
+        final byte dataDetail;
+        final GenTaskTracker taskTracker;
+        final CompletableFuture<Boolean> future;
+        GenTask(DhLodPos pos, byte dataDetail, GenTaskTracker taskTracker, CompletableFuture<Boolean> future) {
+            this.dataDetail = dataDetail;
+            this.pos = pos;
+            this.taskTracker = taskTracker;
+            this.future = future;
+        }
+    }
+    static final class TaskGroup {
+        final DhLodPos pos;
+        byte dataDetail;
+        final LinkedList<GenTask> members = new LinkedList<>(); // Accessed by gen poller thread only
+        TaskGroup(DhLodPos pos, byte dataDetail) {
+            this.pos = pos;
+            this.dataDetail = dataDetail;
+        }
+
+        void accept(ChunkSizedData data) {
+            Iterator<GenTask> iter = members.iterator();
+            while (iter.hasNext()) {
+                GenTask task = iter.next();
+                Consumer<ChunkSizedData> consumer = task.taskTracker.getConsumer();
+                if (consumer == null) {
+                    iter.remove();
+                    task.future.complete(false);
+                } else {
+                    consumer.accept(data);
+                }
+            }
+        }
+    }
+    static final class InProgressTask {
+        final TaskGroup group;
+        CompletableFuture<Void> genFuture = null;
+        InProgressTask(TaskGroup group) {
+            this.group = group;
+        }
+    }
+
+    static class SplitTask extends GenTaskTracker {
+        final GenTaskTracker parentTracker;
+        final CompletableFuture<Boolean> parentFuture;
+        boolean cachedValid = true;
+        SplitTask(GenTaskTracker parentTracker, CompletableFuture<Boolean> parentFuture) {
+            this.parentTracker = parentTracker;
+            this.parentFuture = parentFuture;
+        }
+        boolean recheckState() {
+            if (!cachedValid) return false;
+            cachedValid = parentTracker.isValid();
+            if (!cachedValid) parentFuture.complete(false);
+            return cachedValid;
+        }
+        @Override
+        public boolean isValid() {
+            return cachedValid;
+        }
+        @Override
+        public Consumer<ChunkSizedData> getConsumer() {
+            return parentTracker.getConsumer();
+        }
+    }
+
+    private final ConcurrentLinkedQueue<GenTask> looseTasks = new ConcurrentLinkedQueue<>();
+    private final HashMap<DhLodPos, TaskGroup> taskGroups = new HashMap<>(); // Accessed by poller only
+    private final ConcurrentHashMap<DhLodPos, InProgressTask> inProgress = new ConcurrentHashMap<>();
+
+    private final byte maxGranularity;
+    private final byte minGranularity;
+    private final byte maxDataDetail;
+    private final byte minDataDetail;
+    private volatile CompletableFuture<Void> closer = null;
+
+    public GenerationQueue(IGenerator generator) {
+        this.generator = generator;
+        maxGranularity = generator.getMaxGenerationGranularity();
+        minGranularity = generator.getMinGenerationGranularity();
+        maxDataDetail = generator.getMaxDataDetail();
+        minDataDetail = generator.getMinDataDetail();
+        if (minGranularity < 4) throw new IllegalArgumentException("DH-IGenerator: min granularity must be at least 4!");
+        if (maxGranularity < minGranularity) throw new IllegalArgumentException("DH-IGenerator: max granularity smaller than min granularity!");
+    }
+
+    public CompletableFuture<Boolean> submitGenTask(DhLodPos pos, byte requiredDataDetail, GenTaskTracker tracker) {
+        if (closer != null) return CompletableFuture.completedFuture(false);
+        if (requiredDataDetail < minDataDetail) {
+            throw new UnsupportedOperationException("Current generator does not meet requiredDataDetail level");
+        }
+        if (requiredDataDetail > maxDataDetail) requiredDataDetail = maxDataDetail;
+
+        LodUtil.assertTrue(pos.detail > requiredDataDetail+4);
+        byte granularity = (byte) (pos.detail - requiredDataDetail);
+
+        if (granularity > maxGranularity) {
+            // Too big of a chunk. We need to split it up
+            byte subDetail = (byte) (maxGranularity + requiredDataDetail);
+            int subPosCount = pos.getWidth(subDetail);
+            DhLodPos cornerSubPos = pos.getCorner(subDetail);
+            CompletableFuture<Boolean>[] subFutures = new CompletableFuture[subPosCount*subPosCount];
+            ArrayList<GenTask> subTasks = new ArrayList<>(subPosCount*subPosCount);
+            SplitTask splitTask = new SplitTask(tracker, new CompletableFuture<>());
+            {
+                int i = 0;
+                for (int ox = 0; ox < subPosCount; ox++) {
+                    for (int oz = 0; oz < subPosCount; oz++) {
+                        CompletableFuture<Boolean> subFuture = new CompletableFuture<>();
+                        subFutures[i++] = subFuture;
+                        subTasks.add(new GenTask(cornerSubPos.offset(ox, oz), requiredDataDetail, splitTask, subFuture));
+                    }
+                }
+            }
+            CompletableFuture.allOf(subFutures).whenComplete((v,ex) -> {
+                if (ex != null) splitTask.parentFuture.completeExceptionally(ex);
+                if (!splitTask.recheckState()) return; // Auto join future
+                for (CompletableFuture<Boolean> subFuture: subFutures) {
+                    boolean successful = subFuture.join();
+                    if (!successful) {
+                        splitTask.parentFuture.complete(false);
+                        return;
+                    }
+                }
+                splitTask.parentFuture.complete(true);
+            });
+            looseTasks.addAll(subTasks);
+            if (closer != null) return CompletableFuture.completedFuture(false);
+            else return splitTask.parentFuture;
+        } else if (granularity < minGranularity) {
+            // Too small of a chunk. We'll just over-size the generation.
+            byte parentDetail = (byte) (minGranularity + requiredDataDetail);
+            DhLodPos parentPos = pos.convertUpwardsTo(parentDetail);
+            CompletableFuture<Boolean> future = new CompletableFuture<>();
+            looseTasks.add(new GenTask(parentPos, requiredDataDetail, tracker, future));
+            if (closer != null) return CompletableFuture.completedFuture(false);
+            else return future;
+        } else {
+            CompletableFuture<Boolean> future = new CompletableFuture<>();
+            looseTasks.add(new GenTask(pos, requiredDataDetail, tracker, future));
+            if (closer != null) return CompletableFuture.completedFuture(false);
+            else return future;
+        }
+    }
+
+    private void addAndCombineGroup(TaskGroup target) {
+        byte granularity = (byte) (target.pos.detail - target.dataDetail);
+        LodUtil.assertTrue(granularity <= maxGranularity && granularity >= minGranularity);
+        LodUtil.assertTrue(!taskGroups.containsKey(target.pos));
+
+        // Check and merge all those who has exactly the same dataDetail, and overlaps the position, but have lower granularity than us
+        if (granularity > minGranularity) {
+            // TODO: Optimize this check
+            Iterator<TaskGroup> groupIter = taskGroups.values().iterator();
+            while (groupIter.hasNext()) {
+                TaskGroup group = groupIter.next();
+                if (group.dataDetail != target.dataDetail) continue;
+                if (!group.pos.overlaps(target.pos)) continue;
+
+                // We should have already ALWAYS selected the higher granularity.
+                LodUtil.assertTrue(group.pos.detail < target.pos.detail);
+                groupIter.remove(); // Remove and consume all from that lower granularity request
+                target.members.addAll(group.members);
+            }
+        }
+
+        // Now, Check if we are the missing piece in the 4 quadrants, and if so, combine the four into a new higher granularity group
+        if (granularity < maxGranularity) { // Obviously, only do so if we aren't at the maxGranularity already
+            // Check for merging and upping the granularity
+            DhLodPos corePos = target.pos;
+            DhLodPos parentPos = corePos.convertUpwardsTo((byte) (corePos.detail+1));
+            int targetChildId = target.pos.getChildIndexOfParent();
+            boolean allPassed = true;
+            for (int i = 0; i < 4; i++) {
+                if (i == targetChildId) continue;
+                TaskGroup group = taskGroups.get(parentPos.getChild(i));
+                if (group == null || group.dataDetail != target.dataDetail) {
+                    allPassed = false;
+                    break;
+                }
+            }
+            if (allPassed) {
+                LodUtil.assertTrue(!taskGroups.containsKey(parentPos) || taskGroups.get(parentPos).dataDetail != target.dataDetail);
+                TaskGroup[] groups = new TaskGroup[4];
+                for (int i = 0; i < 4; i++) {
+                    if (i==targetChildId) groups[i] = target;
+                    else groups[i] = taskGroups.remove(parentPos.getChild(i));
+                    LodUtil.assertTrue(groups[i] != null && groups[i].dataDetail == target.dataDetail);
+                }
+
+                TaskGroup newGroup = taskGroups.get(parentPos);
+                if (newGroup != null) {
+                    LodUtil.assertTrue(newGroup.dataDetail != target.dataDetail); // if it is equal, we should have been merged ages ago
+                    if (newGroup.dataDetail < target.dataDetail) {
+                        // We can just append us into the existing list.
+                        for (TaskGroup g : groups) newGroup.members.addAll(g.members);
+                    } else {
+                        // We need to upgrade the requested dataDetail of the group.
+                        newGroup.dataDetail = target.dataDetail;
+                        boolean worked = taskGroups.remove(parentPos, newGroup); // Pop it off for later proper merge check
+                        LodUtil.assertTrue(worked);
+                        for (TaskGroup g : groups) newGroup.members.addAll(g.members);
+                        addAndCombineGroup(newGroup); // Recursive check the new group
+                    }
+                } else {
+                    // There should not be any higher granularity to check, as otherwise we would have merged ages ago
+                    newGroup = new TaskGroup(parentPos, target.dataDetail);
+                    for (TaskGroup g : groups) newGroup.members.addAll(g.members);
+                    addAndCombineGroup(newGroup); // Recursive check the new group
+                }
+                return; // We have merged. So no need to add the target group
+            }
+        }
+
+        // Finally, we should be safe to add the target group into the list
+        TaskGroup v = taskGroups.put(target.pos, target);
+        LodUtil.assertTrue(v == null); // should never be replacing other things
+    }
+
+    private void processLooseTasks() {
+        while (!looseTasks.isEmpty()) {
+            GenTask task = looseTasks.poll();
+            byte taskDataDetail = task.dataDetail;
+            byte taskGranularity = (byte) (task.pos.detail - taskDataDetail);
+            LodUtil.assertTrue(taskGranularity >= 4 && taskGranularity >= minGranularity && taskGranularity <= maxGranularity);
+
+            // Check existing one
+            TaskGroup group = taskGroups.get(task.pos);
+            if (group != null) {
+                if (group.dataDetail <= taskDataDetail) {
+                    // We can just append us into the existing list.
+                    group.members.add(task);
+                } else {
+                    // We need to upgrade the requested dataDetail of the group.
+                    group.dataDetail = taskDataDetail;
+                    boolean worked = taskGroups.remove(task.pos, group); // Pop it off for later proper merge check
+                    LodUtil.assertTrue(worked);
+                    group.members.add(task);
+                    addAndCombineGroup(group);
+                }
+            } else {
+
+                // Check higher granularity one
+                byte granularity = taskGranularity;
+                boolean didAnything = false;
+                while (++granularity <= maxGranularity) {
+                    group = taskGroups.get(task.pos.convertUpwardsTo((byte) (taskDataDetail + granularity)));
+                    if (group != null && group.dataDetail == taskDataDetail) {
+                        // We can just append to the higher granularity group one
+                        group.members.add(task);
+                        didAnything = true;
+                        break;
+                    }
+                }
+                if (!didAnything) {
+                    group = new TaskGroup(task.pos, taskDataDetail);
+                    group.members.add(task);
+                    addAndCombineGroup(group);
+                }
+            }
+
+        }
+
+
+    }
+
+    private void removeOutdatedGroups() {
+        // Remove all invalid genTasks and groups
+        Iterator<TaskGroup> groupIter = taskGroups.values().iterator();
+        while (groupIter.hasNext()) {
+            TaskGroup group = groupIter.next();
+            Iterator<GenTask> taskIter = group.members.iterator();
+            while (taskIter.hasNext()) {
+                GenTask task = taskIter.next();
+                if (!task.taskTracker.isValid()) {
+                    taskIter.remove();
+                    task.future.complete(false);
+                }
+            }
+            if (group.members.isEmpty()) groupIter.remove();
+        }
+    }
+
+    private void pollAndStartNext(DhBlockPos2D targetPos) {
+        // Select the one with the highest data detail level and closest to the target pos
+        TaskGroup best = null;
+        long cachedDist = Long.MAX_VALUE;
+        for (TaskGroup group : taskGroups.values()) {
+            if (best != null) {
+                if (group.dataDetail < best.dataDetail) continue;
+                long dist = group.pos.getCenter().distSquared(targetPos);
+                if (cachedDist <= dist) continue;
+                cachedDist = dist;
+            }
+            best = group;
+        }
+        if (best != null) {
+            InProgressTask startedTask = new InProgressTask(best);
+            InProgressTask casTask = inProgress.putIfAbsent(best.pos, startedTask);
+            boolean worked = taskGroups.remove(best.pos, best); // Remove the selected task from the group
+            LodUtil.assertTrue(worked);
+            if (casTask != null) {
+                // Note: Due to concurrency reasons, even if the currently running task is compatible with selected task,
+                //         we cannot use it, as some chunks may have already been written into.
+                pollAndStartNext(targetPos); // Poll next one.
+                TaskGroup exchange = taskGroups.put(best.pos, best); // put back the task.
+                LodUtil.assertTrue(exchange == null);
+            } else {
+                startTaskGroup(startedTask);
+            }
+        }
+
+    }
 
-    public GenerationQueue() {}
     public void pollAndStartClosest(DhBlockPos2D targetPos) {
         if (generator == null) throw new IllegalStateException("generator is null");
         if (generator.isBusy()) return;
-        DhLodPos closest = null;
-        long closestDist = Long.MAX_VALUE;
-        int smallestDetail = Integer.MAX_VALUE;
-        for (DhLodPos key : taskMap.keySet()) {
-            if (key.detail > smallestDetail) continue;
-            long dist = key.getCenter().distSquared(targetPos);
-            if (key.detail == smallestDetail && dist >= closestDist) continue;
-            closest = key;
-            closestDist = dist;
-            smallestDetail = key.detail;
-        }
-        if (closest != null) {
-            CompletableFuture<GenerationResult> future = taskMap.remove(closest);
-            startFuture(closest, future);
-        }
+        removeOutdatedGroups();
+        processLooseTasks();
+        pollAndStartNext(targetPos);
     }
 
-    public void setGenerator(IGenerator generator) {
-        LodUtil.assertTrue(generator != null);
-        LodUtil.assertTrue(this.generator == null);
-        this.generator = generator;
-        inProgress.set(new ConcurrentHashMap<>(16));
-    }
-    public void removeGenerator() {
-        LodUtil.assertTrue(generator != null);
-        this.generator = null;
-        ConcurrentHashMap<DhLodPos, CompletableFuture<GenerationResult>> swapped = this.inProgress.getAndSet(null);
-        swapped.forEach((k,f) -> f.cancel(true));
-    }
+    private void startTaskGroup(InProgressTask task) {
+        byte dataDetail = task.group.dataDetail;
+        DhLodPos pos = task.group.pos;
+        byte granularity = (byte) (pos.detail - dataDetail);
+        LodUtil.assertTrue(granularity >= minGranularity && granularity <= maxGranularity);
+        LodUtil.assertTrue(dataDetail >= minDataDetail && dataDetail <= maxDataDetail);
 
-    private CompletableFuture<GenerationResult> createFuture(DhLodPos pos) {
-        logger.info("Creating gen future for {}", pos);
-        CompletableFuture<GenerationResult> future = new CompletableFuture<>();
-        CompletableFuture<GenerationResult> swapped = taskMap.put(pos, future);
-        LodUtil.assertTrue(swapped == null);
-        return future;
-    }
-
-    private void startFuture(DhLodPos pos, CompletableFuture<GenerationResult> resultFuture) {
-        byte dataDetail = generator.getDataDetail();
-        byte minGenGranularity = generator.getMinGenerationGranularity();
-        byte maxGenGranularity = generator.getMaxGenerationGranularity();
-        if (minGenGranularity < 4 || maxGenGranularity < 4) {
-            throw new IllegalStateException("Generation granularity must be at least 4!");
-        }
-        byte minUnitDetail = (byte) (dataDetail + minGenGranularity);
-        byte maxUnitDetail = (byte) (dataDetail + maxGenGranularity);
-        LodUtil.assertTrue(pos.detail >= minUnitDetail && pos.detail <= maxUnitDetail);
-        byte genGranularity = (byte) (pos.detail - dataDetail);
         DHChunkPos chunkPosMin = new DHChunkPos(pos.getCorner());
-        logger.info("Generating section {} with granularity {} at {}", pos, genGranularity, chunkPosMin);
-        int perCallChunksWidth = 1 << (genGranularity - 4);
-
-        CompletableFuture<ArrayGridList<ChunkSizedData>> dataFuture = generator.generate(chunkPosMin, genGranularity);
-        final ConcurrentHashMap<DhLodPos, CompletableFuture<GenerationResult>> map = this.inProgress.get();
-        map.put(pos, //FIXME: Slight race condition issue here with map.clear()!
-            dataFuture.handle((data, ex) -> {
-                if (ex != null) {
-                    if (ex instanceof CompletionException) {
-                        ex = ex.getCause();
-                    }
-                    UncheckedInterruptedException.rethrowIfIsInterruption(ex);
-                    logger.error("Error generating data for section {}", pos, ex);
-                    throw new CompletionException("Generation failed", ex);
-                }
-                LodUtil.assertTrue(data != null);
-                if (data.gridSize < (1 << (genGranularity-4))) {
-                    logger.error(
-                            "Generator at {} returned {} by {} chunks but requested granularity was {}, which expect at least {} by {} chunks! ",
-                            pos, data.gridSize, data.gridSize, genGranularity, perCallChunksWidth, perCallChunksWidth);
-                    throw new RuntimeException("Generation failed. Generator returned less data than requested!");
-                }
-                logger.info("Completed generating {} by {} chunks to sections that overlaps {}",
-                        data.gridSize, data.gridSize, pos);
-                return data;
-            }).thenApply((list) -> {
-                GenerationResult result = new GenerationResult();
-                result.dataList.addAll(list);
-                return result;
-            }).handle((r, e) -> {
-                if (e!=null) resultFuture.completeExceptionally(e); else resultFuture.complete(r);
-                map.remove(pos);
-                return null;
-            })
-        );
+        logger.info("Generating section {} with granularity {} at {}", pos, granularity, chunkPosMin);
+        task.genFuture = generator.generate(
+                chunkPosMin, granularity, dataDetail, task.group::accept);
+        task.genFuture.whenComplete((v, ex) -> {
+           if (ex != null) {
+               logger.error("Error generating data for section {}", pos, ex);
+               task.group.members.forEach(m -> m.future.complete(false));
+           } else {
+               logger.info("Section generation at {} complated", pos);
+               task.group.members.forEach(m -> m.future.complete(true));
+           }
+           boolean worked = inProgress.remove(pos, task);
+           LodUtil.assertTrue(worked);
+        });
     }
 
-    public CompletableFuture<LodDataSource> generate(DhSectionPos sectionPos) {
-        byte maxGen = (byte) (generator.getMaxGenerationGranularity() + generator.getDataDetail());
-        if (sectionPos.sectionDetail > maxGen) {
-            int count = 1 << (sectionPos.sectionDetail - maxGen);
-            DhLodPos minPos = sectionPos.getCorner(maxGen);
-            ArrayList<CompletableFuture<GenerationResult>> futures = new ArrayList<>(count*count);
-            for (int x = 0; x < count; x++) {
-                for (int z = 0; z < count; z++) {
-                    DhLodPos subPos = new DhLodPos(maxGen, minPos.x + x, minPos.z + z);
-                    futures.add(cqcpTree.createOrUseExisting(subPos, this::createFuture));
-                }
-            }
-            // FIXME: Does `allOf` have correct behaviour when one of the futures fails?
-            return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
-                    .thenApply((v) -> {
-                        FullDataSource newSource = FullDataSource.createEmpty(sectionPos);
-                        for (CompletableFuture<GenerationResult> future : futures) {
-                            try {
-                                GenerationResult result = future.join();
-                                for (ChunkSizedData data : result.dataList) {
-                                    if (data.getBBoxLodPos().overlaps(sectionPos.getSectionBBoxPos())) newSource.update(data);
-                                }
-                            } catch (Exception e) {
-                                UncheckedInterruptedException.rethrowIfIsInterruption(e);
-                                // else log
-                                logger.error("Error generating data for section {}", sectionPos, e);
-                            }
-                        }
-                        return newSource;
-            });
-        } else {
-            DhLodPos lodPos = sectionPos.getSectionBBoxPos();
-            return cqcpTree.createOrUseExisting(lodPos, this::createFuture).thenApply(
-                    (result) -> {
-                        if (result == null || result.dataList.isEmpty()) return FullDataSource.createEmpty(sectionPos);
-                        FullDataSource newSource = FullDataSource.createEmpty(sectionPos);
-                        for (ChunkSizedData data : result.dataList) {
-                            if (data.getBBoxLodPos().overlaps(sectionPos.getSectionBBoxPos())) newSource.update(data);
-                        }
-                        return newSource;
-                    });
+    public CompletableFuture<Void> startClosing(boolean cancelCurrentGeneration, boolean alsoInterruptRunning) {
+        taskGroups.values().forEach(g -> g.members.forEach(t -> t.future.complete(false)));
+        taskGroups.clear();
+        ArrayList<CompletableFuture<Void>> array = new ArrayList<>(inProgress.size());
+        inProgress.values().forEach(runningTask -> array.add(runningTask.genFuture));
+        closer = CompletableFuture.allOf(array.toArray(CompletableFuture[]::new));
+        if (cancelCurrentGeneration) {
+            array.forEach(f -> f.cancel(alsoInterruptRunning));
         }
+        looseTasks.forEach(t -> t.future.complete(false));
+        looseTasks.clear();
+        return closer;
     }
 
     @Override
     public void close() {
-        //TODO
+        if (closer == null) startClosing(true, true);
+        LodUtil.assertTrue(closer != null);
+        try {
+            closer.orTimeout(SHUTDOWN_TIMEOUT_SEC, TimeUnit.SECONDS).join();
+        } catch (Throwable e) {
+            logger.error("Failed to close generation queue: ", e);
+        }
     }
-
-//
-//    DhBlockPos2D lastPlayerPos = new DhBlockPos2D(0, 0);
-//    final ConcurrentHashMap<DhSectionPos, WeakReference<PlaceHolderRenderSource>> trackers = new ConcurrentHashMap<>();
-//    final BiConsumer<DhSectionPos, ChunkSizedData> writeConsumer;
-//    final HashSet<Request, CompletableFuture<?>> inProgressSections = new HashSet<>();
-
-//
-//    public void track(PlaceHolderRenderSource source) {
-//        //logger.info("Tracking source {} at {}", source, source.getSectionPos());
-//        trackers.put(source.getSectionPos(), new WeakReference<>(source));
-//    }
-//
-//    private void update() {
-//        LinkedList<DhSectionPos> toRemove = new LinkedList<>();
-//        for (DhSectionPos pos : trackers.keySet()) {
-//            WeakReference<PlaceHolderRenderSource> ref = trackers.get(pos);
-//            if (ref.get() == null) {
-//                toRemove.add(pos);
-//            }
-//        }
-//        for (DhSectionPos pos : toRemove) {
-//            trackers.remove(pos);
-//        }
-//    }
-
-//    //FIXME: Do optimizations on polling closest to player. (Currently its a O(n) search!)
-//    //FIXME: Do not return sections that is already being generated.
-//    //FIXME: Optimize the checks for inProgressSections.
-//    private DhSectionPos pollClosest(DhBlockPos2D playerPos) {
-//        update();
-//        DhSectionPos closest = null;
-//        long closestDist = Long.MAX_VALUE;
-//        for (DhSectionPos pos : trackers.keySet()) {
-//            if (inProgressSections.contains(pos)) {
-//                continue;
-//            }
-//            long distSqr = pos.getCenter().getCenter().distSquared(playerPos);
-//            if (distSqr < closestDist) {
-//                closest = pos;
-//                closestDist = distSqr;
-//            }
-//        }
-//        if (closest != null) inProgressSections.add(closest);
-//        return closest;
-//    }
 }

core/src/main/java/com/seibel/lod/core/a7/generation/IChunkGenerator.java

@@ -7,16 +7,15 @@ import com.seibel.lod.core.util.gridList.ArrayGridList;
 import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
 
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
 
 public interface IChunkGenerator extends IGenerator {
-    CompletableFuture<ArrayGridList<IChunkWrapper>> generateChunks(DHChunkPos chunkPosMin, byte granularity);
+    CompletableFuture<Void> generateChunks(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer<IChunkWrapper> resultConsumer);
 
     @Override
-    default CompletableFuture<ArrayGridList<ChunkSizedData>> generate(DHChunkPos chunkPosMin, byte granularity) {
-        return generateChunks(chunkPosMin, granularity).thenApply(chunks -> {
-            ArrayGridList<ChunkSizedData> chunkData = new ArrayGridList<>(chunks.gridSize);
-            chunks.forEachPos((x, y) -> chunkData.set(x, y, LodDataBuilder.createChunkData(chunks.get(x, y))));
-            return chunkData;
+    default CompletableFuture<Void> generate(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer<ChunkSizedData> resultConsumer) {
+        return generateChunks(chunkPosMin, granularity, targetDataDetail, (chunk) -> {
+            resultConsumer.accept(LodDataBuilder.createChunkData(chunk));
         });
     }
 

core/src/main/java/com/seibel/lod/core/a7/generation/IGenerator.java

@@ -5,18 +5,22 @@ import com.seibel.lod.core.objects.DHChunkPos;
 import com.seibel.lod.core.util.gridList.ArrayGridList;
 
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
 
 public interface IGenerator extends AutoCloseable {
     // What is the detail / resolution of the data? (This will offset the generation granularity)
     // (minimum detail is 0, maximum detail is 255) (though that high isn't really... realistic)
     // (0 = 1x1 block per data, 1 = 2x2 block per data, 2 = 4x4 block per data... etc.)
     // TODO: System currently only supports 1x1 block per data.
-    byte getDataDetail();
+    byte getMinDataDetail();
+    byte getMaxDataDetail();
+    int getPriority();
     // What is the min batch size of a single generation?
     // (minimum return value is 4 since that's the MC chunk size)
     // (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.)
     byte getMinGenerationGranularity();
-    // What is the max batch size of a single generation?
+
+    // What is the max batch size of a single generation? The system will try to group tasks to the max batch size if possible
     // (minimum return value is 4 since that's the MC chunk size)
     // (4 -> 16x16 data per call, 5 -> 32x32 data per call, 6 -> 64x64 data per call... etc.)
     byte getMaxGenerationGranularity();
@@ -24,7 +28,7 @@ public interface IGenerator extends AutoCloseable {
     // Start a generation event
     // (Note that the chunkPos is always aligned to the granularity)
     // (For example, if the granularity is 4, data detail is 0, the chunkPos will be aligned to 16x16 blocks)
-    CompletableFuture<ArrayGridList<ChunkSizedData>> generate(DHChunkPos chunkPosMin, byte granularity);
+    CompletableFuture<Void> generate(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer<ChunkSizedData> resultConsumer);
 
     // Return whether the generator is currently busy and cannot accept new generation requests.
     boolean isBusy();

core/src/main/java/com/seibel/lod/core/a7/level/DhClientServerLevel.java

@@ -1,7 +1,6 @@
 package com.seibel.lod.core.a7.level;
 
 import com.seibel.lod.core.a7.generation.GenerationQueue;
-import com.seibel.lod.core.a7.generation.IGenerator;
 import com.seibel.lod.core.a7.render.LodQuadTree;
 import com.seibel.lod.core.a7.save.io.file.GeneratedDataFileHandler;
 import com.seibel.lod.core.a7.util.FileScanner;
@@ -24,6 +23,7 @@ import com.seibel.lod.core.wrapperInterfaces.world.IBiomeWrapper;
 import com.seibel.lod.core.wrapperInterfaces.world.IClientLevelWrapper;
 import com.seibel.lod.core.wrapperInterfaces.world.ILevelWrapper;
 import com.seibel.lod.core.wrapperInterfaces.world.IServerLevelWrapper;
+import net.minecraft.world.entity.ambient.Bat;
 import org.apache.logging.log4j.Logger;
 
 import java.util.concurrent.CompletableFuture;
@@ -32,23 +32,22 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
     private static final Logger LOGGER = DhLoggerBuilder.getLogger();
     private static final IMinecraftClientWrapper MC_CLIENT = SingletonInjector.INSTANCE.get(IMinecraftClientWrapper.class);
     public final LocalSaveStructure save;
-    public final DataFileHandler dataFileHandler;
-    public GenerationQueue generationQueue = null;
+    public final GeneratedDataFileHandler dataFileHandler;
+    public volatile GenerationQueue generationQueue = null;
     public RenderFileHandler renderFileHandler = null;
     public RenderBufferHandler renderBufferHandler = null; //TODO: Should this be owned by renderer?
     public final IServerLevelWrapper serverLevel;
     public IClientLevelWrapper clientLevel;
     public a7LodRenderer renderer = null;
     public LodQuadTree tree = null;
-    public BatchGenerator worldGenerator = null;
+    public volatile BatchGenerator worldGenerator = null;
 
     public DhClientServerLevel(LocalSaveStructure save, IServerLevelWrapper level) {
         this.serverLevel = level;
         this.save = save;
         save.getDataFolder(level).mkdirs();
         save.getRenderCacheFolder(level).mkdirs();
-        generationQueue = new GenerationQueue();
-        dataFileHandler = new GeneratedDataFileHandler(this, save.getDataFolder(level), generationQueue);
+        dataFileHandler = new GeneratedDataFileHandler(this, save.getDataFolder(level));
         FileScanner.scanFile(save, serverLevel, dataFileHandler, null);
         LOGGER.info("Started DHLevel for {} with saves at {}", level, save);
     }
@@ -74,7 +73,8 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
         this.clientLevel = clientLevel;
         // TODO: Make a registry for generators for modding support.
         worldGenerator = new BatchGenerator(this);
-        generationQueue.setGenerator(worldGenerator);
+        generationQueue = new GenerationQueue(worldGenerator);
+        dataFileHandler.setGenerationQueue(generationQueue);
         renderFileHandler = new RenderFileHandler(dataFileHandler, this, save.getRenderCacheFolder(serverLevel));
         tree = new LodQuadTree(this, Config.Client.Graphics.Quality.lodChunkRenderDistance.get()*16,
                 MC_CLIENT.getPlayerBlockPos().x, MC_CLIENT.getPlayerBlockPos().z, renderFileHandler);
@@ -102,18 +102,25 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
         }
         tree.close();
         tree = null;
-        generationQueue.removeGenerator();
-        try {
-            worldGenerator.close();
-        } catch (Exception e) {
-            LOGGER.error("Error closing world generator", e);
-        }
+        dataFileHandler.popGenerationQueue();
+        final BatchGenerator f_worldGen = worldGenerator;
+        CompletableFuture<Void> closer = generationQueue.startClosing(true, true)
+                .exceptionally(ex -> {
+                    LOGGER.error("Error closing geberation queue", ex);
+                    return null;
+                }).thenRun(f_worldGen::close)
+                .exceptionally(ex -> {
+                    LOGGER.error("Error closing geberation queue", ex);
+                    return null;
+                });
+        generationQueue = null;
         worldGenerator = null;
         renderBufferHandler.close();
         renderBufferHandler = null;
         renderFileHandler.flushAndSave(); //Ignore the completion feature so that this action is async
         renderFileHandler.close();
         renderFileHandler = null;
+        closer.join(); // TODO: Could this cause deadlocks? we are blocking in main thread.
     }
 
     @Override
@@ -158,6 +165,7 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
     }
     @Override
     public void close() {
+        if (generationQueue != null) generationQueue.close();
         if (worldGenerator != null) worldGenerator.close();
         if (renderer != null) renderer.close();
         if (tree != null) tree.close();
@@ -170,10 +178,12 @@ public class DhClientServerLevel implements IClientLevel, IServerLevel {
 
     @Override
     public void doWorldGen() {
-        if (worldGenerator != null) {
-            worldGenerator.update();
-            if (generationQueue != null)
-                generationQueue.pollAndStartClosest(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos()));
+        final BatchGenerator f_worldGen = worldGenerator;
+        if (f_worldGen != null) {
+            f_worldGen.update();
+            final GenerationQueue f_genQueue = generationQueue;
+            if (f_genQueue != null)
+                f_genQueue.pollAndStartClosest(new DhBlockPos2D(MC_CLIENT.getPlayerBlockPos()));
         }
     }
 

core/src/main/java/com/seibel/lod/core/a7/level/DhServerLevel.java

@@ -1,5 +1,6 @@
 package com.seibel.lod.core.a7.level;
 
+import com.seibel.lod.core.a7.save.io.file.RemoteDataFileHandler;
 import com.seibel.lod.core.a7.util.FileScanner;
 import com.seibel.lod.core.a7.save.io.file.DataFileHandler;
 import com.seibel.lod.core.a7.save.structure.LocalSaveStructure;
@@ -22,7 +23,7 @@ public class DhServerLevel implements IServerLevel
         this.save = save;
         this.level = level;
         save.getDataFolder(level).mkdirs();
-        dataFileHandler = new DataFileHandler(this, save.getDataFolder(level), null); //FIXME: GenerationQueue
+        dataFileHandler = new DataFileHandler(this, save.getDataFolder(level)); //FIXME: GenerationQueue
         FileScanner.scanFile(save, level, dataFileHandler, null);
         LOGGER.info("Started DHLevel for {} with saves at {}", level, save);
     }

core/src/main/java/com/seibel/lod/core/a7/pos/DhLodPos.java

@@ -94,6 +94,9 @@ public class DhLodPos implements Comparable<DhLodPos> {
         if (width.detail < detail) throw new IllegalArgumentException("add called with width.detail < pos detail");
         return new DhLodPos(detail, x + width.convertTo(detail).value, z + width.convertTo(detail).value);
     }
+    public DhLodPos offset(int ox, int oz) {
+        return new DhLodPos(detail, x+ox, z+oz);
+    }
 
     @Override
     public int compareTo(@NotNull DhLodPos o) {

core/src/main/java/com/seibel/lod/core/a7/render/LodRenderSection.java

@@ -69,6 +69,7 @@ public class LodRenderSection {
             }
         }
         if (lodRenderSource != null) {
+            provider.refreshRenderSource(lodRenderSource);
             lodRenderSource.flushWrites(level);
         }
     }

core/src/main/java/com/seibel/lod/core/a7/save/io/MetaFile.java

@@ -2,14 +2,13 @@ package com.seibel.lod.core.a7.save.io;
 
 import java.io.*;
 import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.FileChannel;
+import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.Consumer;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.Adler32;
 import java.util.zip.CheckedOutputStream;
 
@@ -37,7 +36,7 @@ public class MetaFile
     //
     //    8 bytes: datatype identifier
     //
-    //    8 bytes: timestamp
+    //    8 bytes: data version
 
     // Used size: 40 bytes
     // Remaining space: 24 bytes
@@ -53,28 +52,38 @@ public class MetaFile
     public final DhSectionPos pos;
 
     public File path;
-    public int checksum;
-    public long timestamp;
-    public byte dataLevel;
 
-    //Loader stuff
-    public long dataTypeId;
-    public byte loaderVersion;
+    @FunctionalInterface
+    public interface IOConsumer<T> {
+        void accept(T t) throws IOException;
+    }
 
-    private final ReentrantReadWriteLock assertLock = new ReentrantReadWriteLock();
+    public static class MetaData {
+        public DhSectionPos pos;
+        public int checksum;
+        public AtomicLong dataVersion;
+        public byte dataLevel;
+        //Loader stuff
+        public long dataTypeId;
+        public byte loaderVersion;
 
-    // Load a metaFile in this path. It also automatically read the metadata.
-    protected MetaFile(File path) throws IOException {
-        this.path = path;
-        validateFile();
-        LodUtil.assertTrue(assertLock.readLock().tryLock());
-        try (FileChannel channel = FileChannel.open(path.toPath(), StandardOpenOption.READ)) {
+        public MetaData(DhSectionPos pos, int checksum, long dataVersion, byte dataLevel, long dataTypeId, byte loaderVersion) {
+            this.pos = pos;
+            this.checksum = checksum;
+            this.dataVersion = new AtomicLong(dataVersion);
+            this.dataLevel = dataLevel;
+            this.dataTypeId = dataTypeId;
+            this.loaderVersion = loaderVersion;
+        }
+    }
+    public volatile MetaData metaData = null;
+    private static MetaData readMeta(File file) throws IOException {
+        try (FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
             ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE);
             channel.read(buffer, 0);
             channel.close();
             buffer.flip();
 
-            this.path = path;
             int magic = buffer.getInt();
             if (magic != METADATA_MAGIC_BYTES) {
                 throw new IOException("Invalid file: Magic bytes check failed.");
@@ -82,26 +91,19 @@ public class MetaFile
             int x = buffer.getInt();
             int y = buffer.getInt(); // Unused
             int z = buffer.getInt();
-            checksum = buffer.getInt();
+            int checksum = buffer.getInt();
             byte detailLevel = buffer.get();
-            dataLevel = buffer.get();
-            loaderVersion = buffer.get();
+            byte dataLevel = buffer.get();
+            byte loaderVersion = buffer.get();
             byte unused = buffer.get();
-            dataTypeId = buffer.getLong();
-            timestamp = buffer.getLong();
+            long dataTypeId = buffer.getLong();
+            long timestamp = buffer.getLong();
             LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE);
-            pos = new DhSectionPos(detailLevel, x, z);
-        } finally {
-            assertLock.readLock().unlock();
+            DhSectionPos dataPos = new DhSectionPos(detailLevel, x, z);
+            return new MetaData(dataPos, checksum, timestamp, dataLevel, dataTypeId, loaderVersion);
         }
     }
 
-    // Make a new MetaFile. It doesn't load or write any metadata itself.
-    protected MetaFile(File path, DhSectionPos pos) {
-        this.path = path;
-        this.pos = pos;
-    }
-
     private void validateFile() throws IOException {
         if (!path.exists()) throw new IOException("File missing");
         if (!path.isFile()) throw new IOException("Not a file");
@@ -109,42 +111,32 @@ public class MetaFile
         if (!path.canWrite()) throw new IOException("File not writable");
     }
 
-    protected void updateMetaData() throws IOException {
+    // Create a metaFile in this path. If the path has a file, throws FileAlreadyExistsException
+    protected MetaFile(File path, DhSectionPos pos) throws IOException {
+        this.path = path;
+        this.pos = pos;
+        if (path.exists()) throw new FileAlreadyExistsException(path.toString());
+    }
+    // Load a metaFile in this path
+    protected MetaFile(File path) throws IOException {
+        this.path = path;
+        if (!path.exists()) throw new FileNotFoundException("File not found at " + path);
         validateFile();
-        LodUtil.assertTrue(assertLock.readLock().tryLock());
-        try (FileChannel channel = FileChannel.open(path.toPath(), StandardOpenOption.READ)) {
-            ByteBuffer buffer = ByteBuffer.allocate(METADATA_SIZE);
-            channel.read(buffer, 0);
-            channel.close();
-            buffer.flip();
+        metaData = readMeta(path);
+        pos = metaData.pos;
+    }
 
-            int magic = buffer.getInt();
-            if (magic != METADATA_MAGIC_BYTES) {
-                throw new IOException("Invalid file: Magic bytes check failed.");
-            }
-            int x = buffer.getInt();
-            int y = buffer.getInt(); // Unused
-            int z = buffer.getInt();
-            checksum = buffer.getInt();
-            byte detailLevel = buffer.get();
-            dataLevel = buffer.get();
-            byte loaderVersion = buffer.get();
-            byte unused = buffer.get();
-            dataTypeId = buffer.getLong();
-            timestamp = buffer.getLong();
-            LodUtil.assertTrue(buffer.remaining() == METADATA_RESERVED_SIZE);
-
-            DhSectionPos newPos = new DhSectionPos(detailLevel, x, z);
-            if (!newPos.equals(pos)) {
-                throw new IOException("Invalid file: Section position changed.");
-            }
-            this.loaderVersion = loaderVersion;
-        } finally {
-            assertLock.readLock().unlock();
+    protected void loadMetaData() throws IOException {
+        validateFile();
+        metaData = readMeta(path);
+        if (!metaData.pos.equals(pos)) {
+            LOGGER.warn("The file is from a different location than expected! Expects {} but got {}. Ignoring file tag.", pos, metaData.pos);
+            metaData.pos = pos;
         }
     }
 
-    protected void writeData(Consumer<OutputStream> dataWriter) throws IOException {
+    protected void writeData(IOConsumer<OutputStream> dataWriter) throws IOException {
+        LodUtil.assertTrue(metaData != null);
         if (path.exists()) validateFile();
         File writerFile;
         if (USE_ATOMIC_MOVE_REPLACE) {
@@ -153,7 +145,7 @@ public class MetaFile
         } else {
             writerFile = path;
         }
-        LodUtil.assertTrue(assertLock.writeLock().tryLock());
+
         try (FileChannel file = FileChannel.open(writerFile.toPath(),
                 StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
             {
@@ -174,11 +166,11 @@ public class MetaFile
                 buff.putInt(pos.sectionZ);
                 buff.putInt(checksum);
                 buff.put(pos.sectionDetail);
-                buff.put(dataLevel);
-                buff.put(loaderVersion);
+                buff.put(metaData.dataLevel);
+                buff.put(metaData.loaderVersion);
                 buff.put(Byte.MIN_VALUE); // Unused
-                buff.putLong(dataTypeId);
-                buff.putLong(timestamp);
+                buff.putLong(metaData.dataTypeId);
+                buff.putLong(metaData.dataVersion.get());
                 LodUtil.assertTrue(buff.remaining() == METADATA_RESERVED_SIZE);
                 buff.flip();
                 file.write(buff);
@@ -189,7 +181,6 @@ public class MetaFile
                 Files.move(writerFile.toPath(), path.toPath(), StandardCopyOption.ATOMIC_MOVE);
             }
         } finally {
-            assertLock.writeLock().unlock();
             try {
                 if (USE_ATOMIC_MOVE_REPLACE && writerFile.exists()) {
                     boolean i = writerFile.delete(); // Delete temp file. Ignore errors if fails.

core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataFileHandler.java

@@ -4,10 +4,11 @@ import com.google.common.collect.HashMultimap;
 import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
 import com.seibel.lod.core.a7.datatype.full.FullDataSource;
+import com.seibel.lod.core.a7.datatype.full.SparseDataSource;
 import com.seibel.lod.core.a7.level.ILevel;
-import com.seibel.lod.core.a7.level.IServerLevel;
 import com.seibel.lod.core.a7.pos.DhLodPos;
 import com.seibel.lod.core.a7.pos.DhSectionPos;
+import com.seibel.lod.core.a7.save.io.MetaFile;
 import com.seibel.lod.core.logging.DhLoggerBuilder;
 import com.seibel.lod.core.util.LodUtil;
 import org.apache.logging.log4j.Logger;
@@ -20,9 +21,10 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Function;
+import java.util.function.Consumer;
 
 public class DataFileHandler implements IDataSourceProvider {
     // Note: Single main thread only for now. May make it multi-thread later, depending on the usage.
@@ -33,14 +35,10 @@ public class DataFileHandler implements IDataSourceProvider {
     final File saveDir;
     AtomicInteger topDetailLevel = new AtomicInteger(-1);
     final int minDetailLevel = FullDataSource.SECTION_SIZE_OFFSET;
-    final Function<DhSectionPos, CompletableFuture<LodDataSource>> dataSourceCreator;
 
-
-    public DataFileHandler(ILevel level, File saveRootDir,
-                           Function<DhSectionPos, CompletableFuture<LodDataSource>> dataSourceCreator) {
+    public DataFileHandler(ILevel level, File saveRootDir) {
         this.saveDir = saveRootDir;
         this.level = level;
-        this.dataSourceCreator = dataSourceCreator;
     }
 
     /*
@@ -55,7 +53,7 @@ public class DataFileHandler implements IDataSourceProvider {
         { // Sort files by pos.
             for (File file : detectedFiles) {
                 try {
-                    DataMetaFile metaFile = new DataMetaFile(level, file);
+                    DataMetaFile metaFile = new DataMetaFile(this, level, file);
                     filesByPos.put(metaFile.pos, metaFile);
                 } catch (IOException e) {
                     LOGGER.error("Failed to read file {}. File will be deleted.", file, e);
@@ -71,7 +69,7 @@ public class DataFileHandler implements IDataSourceProvider {
             Collection<DataMetaFile> metaFiles = filesByPos.get(pos);
             DataMetaFile fileToUse;
             if (metaFiles.size() > 1) {
-                fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.timestamp));
+                fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get()));
                 {
                     StringBuilder sb = new StringBuilder();
                     sb.append("Multiple files with the same pos: ");
@@ -108,26 +106,23 @@ public class DataFileHandler implements IDataSourceProvider {
         }
     }
 
-    private DataMetaFile atomicGetOrMakeFile(DhSectionPos pos) {
+    protected DataMetaFile atomicGetOrMakeFile(DhSectionPos pos) {
         DataMetaFile metaFile = files.get(pos);
         if (metaFile == null) {
-            File file = computeDefaultFilePath(pos);
-            //FIXME: Handle file already exists issue. Possibly by renaming the file.
-            LodUtil.assertTrue(!file.exists(), "File {} already exist for path {}", file, pos);
-            CompletableFuture<LodDataSource> gen = new CompletableFuture<>();
-            DataMetaFile newMetaFile = new DataMetaFile(level, file, pos, gen);
-            metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
-            if (metaFile == null) {
-                buildFile(pos, gen);
-                metaFile = newMetaFile;
-            } else {
-                gen.cancel(true);
+            DataMetaFile newMetaFile;
+            try {
+                newMetaFile = new DataMetaFile(this, level, pos);
+            } catch (IOException e) {
+                LOGGER.error("IOException on creating new data file at {}", pos, e);
+                return null;
             }
+            metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
+            if (metaFile == null) metaFile = newMetaFile;
         }
         return metaFile;
     }
 
-    private void selfSearch(DhSectionPos basePos, DhSectionPos pos, ArrayList<DataMetaFile> existFiles, ArrayList<DhSectionPos> missing) {
+    protected void selfSearch(DhSectionPos basePos, DhSectionPos pos, ArrayList<DataMetaFile> existFiles, ArrayList<DhSectionPos> missing) {
         byte detail = pos.sectionDetail;
         boolean allEmpty = true;
         outerLoop:
@@ -210,48 +205,6 @@ public class DataFileHandler implements IDataSourceProvider {
     }
 
 
-    private void buildFile(DhSectionPos pos, CompletableFuture<LodDataSource> gen) {
-        ArrayList<DataMetaFile> existFiles = new ArrayList<>();
-        ArrayList<DhSectionPos> missing = new ArrayList<>();
-        selfSearch(pos, pos, existFiles, missing);
-        LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
-        if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) {
-            dataSourceCreator.apply(pos).whenComplete((f, ex) -> {
-                if (ex != null) {
-                    gen.completeExceptionally(ex);
-                } else {
-                    gen.complete(f);
-                }
-            });
-            return;
-        }
-
-
-        LOGGER.info("Creating file at {} using {} existing files and {} new files.", pos, existFiles.size(), missing.size());
-        ArrayList<CompletableFuture<LodDataSource>> futures = new ArrayList<>(existFiles.size() + missing.size());
-        for (DhSectionPos missingPos : missing) {
-            existFiles.add(atomicGetOrMakeFile(missingPos));
-        }
-        FullDataSource fullDataSource = FullDataSource.createEmpty(pos);
-        for (DataMetaFile metaFile : existFiles) {
-            futures.add(
-                    metaFile.loadOrGetCached(fileReaderThread).whenComplete((data, ex) -> {
-                        if (ex != null) return;
-                        if (!(data instanceof FullDataSource)) return;
-                        fullDataSource.writeFromLower((FullDataSource) data);
-                    })
-            );
-        }
-        CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
-                .whenComplete((v, ex) -> {
-            if (ex != null) {
-                gen.completeExceptionally(ex);
-            } else {
-                gen.complete(fullDataSource);
-            }
-        });
-    }
-
     /*
     * This call is concurrent. I.e. it supports multiple threads calling this method at the same time.
      */
@@ -259,7 +212,8 @@ public class DataFileHandler implements IDataSourceProvider {
     public CompletableFuture<LodDataSource> read(DhSectionPos pos) {
         topDetailLevel.updateAndGet(v -> Math.max(v, pos.sectionDetail));
         DataMetaFile metaFile = atomicGetOrMakeFile(pos);
-        return metaFile.loadOrGetCached(fileReaderThread);
+        if (metaFile == null) return CompletableFuture.completedFuture(null);
+        return metaFile.loadOrGetCached();
     }
 
     /*
@@ -289,23 +243,81 @@ public class DataFileHandler implements IDataSourceProvider {
     public CompletableFuture<Void> flushAndSave() {
         ArrayList<CompletableFuture<Void>> futures = new ArrayList<>();
         for (DataMetaFile metaFile : files.values()) {
-            futures.add(metaFile.flushAndSave(fileReaderThread));
+            futures.add(metaFile.flushAndSave());
         }
         return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
     }
 
     @Override
-    public boolean isCacheValid(DhSectionPos sectionPos, long timestamp) {
+    public long getLatestCacheVersion(DhSectionPos sectionPos) {
         DataMetaFile file = files.get(sectionPos);
-        if (file == null) return false;
-        //TODO
-        return true;
+        if (file == null) return 0;
+        return file.getDataVersion();
     }
 
-    private File computeDefaultFilePath(DhSectionPos pos) { //TODO: Temp code as we haven't decided on the file naming & location yet.
+    @Override
+    public CompletableFuture<LodDataSource> onCreateDataFile(DataMetaFile file) {
+        DhSectionPos pos = file.pos;
+        ArrayList<DataMetaFile> existFiles = new ArrayList<>();
+        ArrayList<DhSectionPos> missing = new ArrayList<>();
+        selfSearch(pos, pos, existFiles, missing);
+        LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
+        if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) {
+            // None exist.
+            SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
+            return CompletableFuture.completedFuture(dataSource);
+        } else {
+
+            for (DhSectionPos missingPos : missing) {
+                DataMetaFile newfile = atomicGetOrMakeFile(missingPos);
+                if (newfile != null) existFiles.add(newfile);
+            }
+            final ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(existFiles.size());
+            final SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
+
+            for (DataMetaFile f : existFiles) {
+                futures.add(f.loadOrGetCached()
+                        .exceptionally((ex) -> null)
+                        .thenAccept((data) -> {
+                            if (data != null) {
+                                if (data instanceof SparseDataSource)
+                                    dataSource.sampleFrom((SparseDataSource) data);
+                                else if (data instanceof FullDataSource)
+                                    dataSource.sampleFrom((FullDataSource) data);
+                                else LodUtil.assertNotReach();
+                            }
+                        })
+                );
+            }
+            return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
+                    .thenApply((v) -> dataSource.trySelfPromote());
+        }
+    }
+
+    @Override
+    public LodDataSource onDataFileLoaded(LodDataSource source, Consumer<LodDataSource> updater) {
+        updater.accept(source);
+        if (source instanceof SparseDataSource) return ((SparseDataSource) source).trySelfPromote();
+        return source;
+    }
+    @Override
+    public LodDataSource onDataFileRefresh(LodDataSource source, Consumer<LodDataSource> updater) {
+        updater.accept(source);
+        if (source instanceof SparseDataSource) return ((SparseDataSource) source).trySelfPromote();
+        return source;
+    }
+
+    @Override
+    public File computeDataFilePath(DhSectionPos pos) {
         return new File(saveDir, pos.serialize() + ".lod");
     }
 
+    @Override
+    public Executor getIOExecutor() {
+        return fileReaderThread;
+    }
+
+
     @Override
     public void close() {
         DataMetaFile.debugCheck();

core/src/main/java/com/seibel/lod/core/a7/save/io/file/DataMetaFile.java

@@ -1,28 +1,19 @@
 package com.seibel.lod.core.a7.save.io.file;
 
-import java.awt.*;
 import java.io.*;
 import java.lang.ref.*;
-import java.security.Provider;
-import java.sql.Ref;
-import java.util.List;
 import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.Executor;
+import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.Function;
-import java.util.function.Supplier;
 
 import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.DataSourceLoader;
+import com.seibel.lod.core.a7.datatype.LodRenderSource;
 import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
-import com.seibel.lod.core.a7.datatype.full.FullDataSource;
 import com.seibel.lod.core.a7.pos.DhLodPos;
 import com.seibel.lod.core.a7.save.io.MetaFile;
 import com.seibel.lod.core.a7.level.ILevel;
@@ -36,14 +27,15 @@ public class DataMetaFile extends MetaFile
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger(DataMetaFile.class.getSimpleName());
 	
 	private final ILevel level;
+	private final IDataSourceProvider handler;
+	private boolean doesFileExist;
+
 	public DataSourceLoader loader;
 	public Class<? extends LodDataSource> dataType;
-	AtomicInteger localVersion = new AtomicInteger(); // This MUST be atomic
-	
 	// The '?' type should either be:
-	//    SoftReference<LodDataSource>, or		- Non-dirty file that can be GCed
-	//    CompletableFuture<LodDataSource>, or  - File that is being loaded
-	//    null									- Nothing is loaded or being loaded
+	//    SoftReference<LodDataSource>, or		    - Non-dirty file that can be GCed
+	//    CompletableFuture<LodDataSource>, or      - File that is being loaded. No guarantee that the type is promotable or not
+	//    null									    - Nothing is loaded or being loaded
 	AtomicReference<Object> data = new AtomicReference<Object>(null);
 
 	//TODO: use ConcurrentAppendSingleSwapContainer<LodDataSource> instead of below:
@@ -51,14 +43,18 @@ public class DataMetaFile extends MetaFile
 		ReentrantReadWriteLock appendLock = new ReentrantReadWriteLock();
 		ConcurrentLinkedQueue<ChunkSizedData> queue = new ConcurrentLinkedQueue<>();
 	}
+
+	// ===Concurrent Write stuff===
 	AtomicReference<GuardedMultiAppendQueue> writeQueue =
 			new AtomicReference<>(new GuardedMultiAppendQueue());
 	GuardedMultiAppendQueue _backQueue = new GuardedMultiAppendQueue();
-	private final AtomicBoolean inCacheWriteLock = new AtomicBoolean(false);
+	// ===========================
 
+	private final AtomicBoolean inCacheWriteAccessAsserter = new AtomicBoolean(false);
+
+	// ===Object lifetime stuff===
 	private static final ReferenceQueue<LodDataSource> lifeCycleDebugQueue = new ReferenceQueue<>();
 	private static final Set<DataObjTracker> lifeCycleDebugSet = ConcurrentHashMap.newKeySet();
-
 	private static class DataObjTracker extends PhantomReference<LodDataSource> implements Closeable {
 		private final DhSectionPos pos;
 		DataObjTracker(LodDataSource data) {
@@ -72,6 +68,50 @@ public class DataMetaFile extends MetaFile
 			lifeCycleDebugSet.remove(this);
 		}
 	}
+    // ===========================
+
+
+
+	// Create a new metaFile
+	public DataMetaFile(IDataSourceProvider handler, ILevel level, DhSectionPos pos) throws IOException {
+		super(handler.computeDataFilePath(pos), pos);
+		debugCheck();
+		this.handler = handler;
+		this.level = level;
+		LodUtil.assertTrue(metaData == null);
+		doesFileExist = false;
+	}
+
+	public DataMetaFile(IDataSourceProvider handler, ILevel level, File path) throws IOException {
+		super(path);
+		debugCheck();
+		this.handler = handler;
+		this.level = level;
+		LodUtil.assertTrue(metaData != null);
+		loader = DataSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion);
+		if (loader == null) {
+			throw new IOException("Invalid file: Data type loader not found: "
+					+ metaData.dataTypeId + "(v" + metaData.loaderVersion + ")");
+		}
+		dataType = loader.clazz;
+		doesFileExist = true;
+	}
+
+	public CompletableFuture<Void> flushAndSave() {
+		debugCheck();
+		boolean isEmpty = writeQueue.get().queue.isEmpty();
+		if (!isEmpty) {
+			return loadOrGetCached().thenApply((unused) -> null); // This will flush the data to disk.
+		} else {
+			return CompletableFuture.completedFuture(null);
+		}
+	}
+
+	public long getDataVersion() {
+		debugCheck();
+		MetaData getData = metaData;
+		return getData == null ? 0 : metaData.dataVersion.get();
+	}
 
 	public void addToWriteQueue(ChunkSizedData datatype) {
 		debugCheck();
@@ -90,68 +130,82 @@ public class DataMetaFile extends MetaFile
 			appendLock.unlock();
 		}
 	}
-	private void swapWriteQueue() {
-		GuardedMultiAppendQueue queue = writeQueue.getAndSet(_backQueue);
-		// Acquire write lock and then release it again as we only need to ensure that the queue
-		// is not being appended to by another thread. Note that the above atomic swap &
-		// the guarantee that all append first acquire the appendLock means after the locK() call,
-		// there will be no other threads able to or is currently appending to the queue.
-		// Note: The above needs the getAndSet() to have at least Release Memory order.
-		// (not that java supports anything non volatile for getAndSet()...)
-		queue.appendLock.writeLock().lock();
-		queue.appendLock.writeLock().unlock();
-		_backQueue = queue;
-	}
 
-	// Load a metaFile in this path. It also automatically read the metadata.
-	public DataMetaFile(ILevel level, File path) throws IOException {
-		super(path);
+	// Cause: Generic Type runtime casting cannot safety check it.
+	// However, the Union type ensures the 'data' should only contain the listed type.
+	public CompletableFuture<LodDataSource> loadOrGetCached() {
 		debugCheck();
-		this.level = level;
-		loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
-		if (loader == null) {
-			throw new IOException("Invalid file: Data type loader not found: "
-					+ dataTypeId + "(v" + loaderVersion + ")");
-		}
-		dataType = loader.clazz;
-	}
+		Object obj = data.get();
+
+		CompletableFuture<LodDataSource> cached = _readCached(obj);
+		if (cached != null) return cached;
 
-	// Make a new MetaFile. It doesn't load or write any metadata itself.
-	public DataMetaFile(ILevel level, File path, DhSectionPos pos, CompletableFuture<LodDataSource> creator) {
-		super(path, pos);
-		debugCheck();
-		this.level = level;
 		CompletableFuture<LodDataSource> future = new CompletableFuture<>();
-		data.set(future);
-		creator.thenApply((f) -> {
-				applyWriteQueue(f);
-				return f;
-		}).whenComplete((f, e) -> {
-			if (e != null) {
-				LOGGER.error("Uncaught error on creation {}: ", path, e);
-				future.complete(null);
-				data.set(null);
-			} else {
-				future.complete(f);
-				new DataObjTracker(f);
-				data.set(new SoftReference<>(f));
-			}
-		});
-	}
-	
-	public boolean isValid(int version) {
-		debugCheck();
-		boolean isValid;
-		// First check if write queue is empty, then check if localVersion is equal to version.
-		// Must be done in this order as writer will increment localVersion before polling in the write queue.
-		// Note: Be careful with the localVerion read's memory order if we do switch over to java 1.9.
-		// It should be acquire or higher!
 
-		isValid = writeQueue.get().queue.isEmpty(); // The 'get()' & 'isEmpty()' enforce a memory barrier.
-													// Also, we are just querying the state, and this means no
-													// need to get any locks for the queue.
-		isValid &= localVersion.get() == version; // The 'get()' enforce a memory barrier.
-		return isValid;
+		// Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
+		boolean worked = data.compareAndSet(obj, future);
+		if (!worked) return loadOrGetCached();
+
+		// After cas. We are in exclusive control.
+		if (!doesFileExist) {
+			doesFileExist = true;
+			handler.onCreateDataFile(this)
+					.thenApply((data) -> {
+						metaData = makeMetaData(data);
+						return data;
+					})
+					.thenApply((data) -> handler.onDataFileLoaded(data, this::applyWriteQueue))
+					.whenComplete((v, e) -> {
+						if (e != null) {
+							LOGGER.error("Uncaught error on creation {}: ", path, e);
+							future.complete(null);
+							data.set(null);
+						} else {
+							future.complete(v);
+							new DataObjTracker(v);
+							data.set(new SoftReference<>(v));
+						}
+					});
+		} else {
+			CompletableFuture.supplyAsync(() -> {
+						if (metaData == null)
+							throw new IllegalStateException("Meta data not loaded!");
+						// Load the file.
+						LodDataSource data;
+						try (FileInputStream fio = getDataContent()){
+							data = loader.loadData(this, fio, level);
+						} catch (IOException e) {
+							throw new CompletionException(e);
+						}
+						// Apply the write queue
+						LodUtil.assertTrue(!inCacheWriteAccessAsserter.get(),"No one should be writing to the cache while we are in the process of " +
+								"loading one into the cache! Is this a deadlock?");
+						data = handler.onDataFileLoaded(data, this::applyWriteQueue);
+						// Finally, return the data.
+						return data;
+					}, handler.getIOExecutor())
+					.whenComplete((f, e) -> {
+						if (e != null) {
+							LOGGER.error("Error loading file {}: ", path, e);
+							future.complete(null);
+							data.set(null);
+						} else {
+							future.complete(f);
+							new DataObjTracker(f);
+							data.set(new SoftReference<>(f));
+						}
+					});
+		}
+
+		// Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :(
+		//return future.completeAsync(this::loadAndUpdateDataSource, fileReaderThreads);
+		return future;
+	}
+
+	private static MetaData makeMetaData(LodDataSource data) {
+		DataSourceLoader loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion());
+		return new MetaData(data.getSectionPos(), -1, 1,
+				data.getDataDetail(), loader == null ? 0 : loader.datatypeId, data.getDataVersion());
 	}
 
 	// "unchecked": Suppress casting of CompletableFuture<?> to CompletableFuture<LodDataSource>
@@ -174,13 +228,13 @@ public class DataMetaFile extends MetaFile
 					//       The latter give us immediate access to the data, but we need to ensure concurrent reads and
 					//       writes doesn't cause unexpected behavior down the line.
 					//       For now, I'll go for the latter option and just hope nothing goes wrong...
-					if (inCacheWriteLock.getAndSet(true) == false) {
+					if (inCacheWriteAccessAsserter.getAndSet(true) == false) {
 						try {
-							applyWriteQueue((LodDataSource) inner);
+							inner = handler.onDataFileRefresh((LodDataSource) inner, this::applyWriteQueue);
 						} catch (Exception e) {
 							LOGGER.error("Error while applying changes to LodDataSource at {}: ", pos, e);
 						} finally {
-							inCacheWriteLock.set(false);
+							inCacheWriteAccessAsserter.set(false);
 						}
 					}
 				}
@@ -197,36 +251,17 @@ public class DataMetaFile extends MetaFile
 		return null;
 	}
 
-	// Cause: Generic Type runtime casting cannot safety check it.
-	// However, the Union type ensures the 'data' should only contain the listed type.
-	public CompletableFuture<LodDataSource> loadOrGetCached(Executor fileReaderThreads) {
-		debugCheck();
-		Object obj = data.get();
-		
-		CompletableFuture<LodDataSource> cached = _readCached(obj);
-		if (cached != null) return cached;
-
-		CompletableFuture<LodDataSource> future = new CompletableFuture<>();
-		
-		// Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
-		boolean worked = data.compareAndSet(obj, future);
-		if (!worked) return loadOrGetCached(fileReaderThreads);
-		
-		// Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :(
-		//return future.completeAsync(this::loadAndUpdateDataSource, fileReaderThreads);
-		CompletableFuture.supplyAsync(this::loadAndUpdateDataSource, fileReaderThreads)
-				.whenComplete((f, e) -> {
-			if (e != null) {
-				LOGGER.error("Uncaught error loading file {}: ", path, e);
-				future.complete(null);
-				data.set(null);
-			} else {
-				future.complete(f);
-				new DataObjTracker(f);
-				data.set(new SoftReference<>(f));
-			}
-		});
-		return future;
+	private void swapWriteQueue() {
+		GuardedMultiAppendQueue queue = writeQueue.getAndSet(_backQueue);
+		// Acquire write lock and then release it again as we only need to ensure that the queue
+		// is not being appended to by another thread. Note that the above atomic swap &
+		// the guarantee that all append first acquire the appendLock means after the locK() call,
+		// there will be no other threads able to or is currently appending to the queue.
+		// Note: The above needs the getAndSet() to have at least Release Memory order.
+		// (not that java supports anything non volatile for getAndSet()...)
+		queue.appendLock.writeLock().lock();
+		queue.appendLock.writeLock().unlock();
+		_backQueue = queue;
 	}
 
 	// Return whether any write has happened to the data
@@ -235,54 +270,30 @@ public class DataMetaFile extends MetaFile
 		// First check if write queue is empty, then swap the write queue.
 		// Must be done in this order to ensure isValid work properly. See isValid() for details.
 		boolean isEmpty = writeQueue.get().queue.isEmpty();
-		int localVer;
 		if (!isEmpty) {
-			localVer = localVersion.incrementAndGet();
 			swapWriteQueue();
 			int count = _backQueue.queue.size();
 			for (ChunkSizedData chunk : _backQueue.queue) {
 				data.update(chunk);
 			}
 			_backQueue.queue.clear();
-			write(data);
-			LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count);
-		} else localVer = localVersion.get();
-		data.setLocalVersion(localVer);
-	}
-	
-	private LodDataSource loadAndUpdateDataSource() {
-		LodDataSource data = loadFile();
-		if (data == null) data = FullDataSource.createEmpty(pos);
-		// Apply the write queue
-		LodUtil.assertTrue(!inCacheWriteLock.get(),"No one should be writing to the cache while we are in the process of " +
-				"loading one into the cache! Is this a deadlock?");
-		applyWriteQueue(data);
-		// Finally, return the data.
-		return data;
+			try {
+				// Write/Update data
+				LodUtil.assertTrue(metaData != null);
+				metaData.dataLevel = data.getDataDetail();
+				loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion());
+				LodUtil.assertTrue(loader != null, "No loader for {} (v{})", data.getClass(), data.getDataVersion());
+				dataType = data.getClass();
+				metaData.dataTypeId = loader == null ? 0 : loader.datatypeId;
+				metaData.loaderVersion = data.getDataVersion();
+				super.writeData((out) -> data.saveData(level, this, out));
+				LOGGER.info("Updated Data file at {} for sect {} with {} chunk writes.", path, pos, count);
+			} catch (IOException e) {
+				LOGGER.error("Failed to save updated data file at {} for sect {}", path, pos, e);
+			}
+		}
 	}
 
-	private LodDataSource loadFile() {
-		if (!path.exists()) return null;
-		// Refresh the metadata.
-		try {
-			super.updateMetaData();
-		} catch (Exception e) {
-			LOGGER.warn("Metadata for file {} changed unexpectedly and in an invalid state. Dropping file.", path, e);
-			return null;
-		}
-		if (loader == null) {
-			//LOGGER.warn("No loader for file {}. Dropping file.", path); // Disable as data lod has no loader yet.
-			return null;
-		}
-
-		// Load the file.
-		try (FileInputStream fio = getDataContent()){
-			return loader.loadData(this, fio, level);
-		} catch (Exception e) {
-			LOGGER.warn("Failed to load file {}. Dropping file.", path, e);
-			return null;
-		}
-	}
 	private FileInputStream getDataContent() throws IOException {
 		FileInputStream fin = new FileInputStream(path);
 		int toSkip = METADATA_SIZE;
@@ -300,51 +311,6 @@ public class DataMetaFile extends MetaFile
 	}
 
 
-	public CompletableFuture<Void> flushAndSave(Executor fileWriterThreads) {
-		debugCheck();
-		boolean isEmpty = writeQueue.get().queue.isEmpty();
-		if (!isEmpty) {
-			return loadOrGetCached(fileWriterThreads).thenApply((unused) -> null); // This will flush the data to disk.
-		} else {
-			return CompletableFuture.completedFuture(null);
-		}
-	}
-
-	@Override
-	protected void updateMetaData() throws IOException {
-		super.updateMetaData();
-		loader = DataSourceLoader.getLoader(dataTypeId, loaderVersion);
-		if (loader == null) {
-			throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
-		}
-		dataType = loader.clazz;
-		dataTypeId = loader.datatypeId;
-	}
-
-	private void write(LodDataSource data) {
-		try {
-			dataLevel = data.getDataDetail();
-			loader = DataSourceLoader.getLoader(data.getClass(), data.getDataVersion());
-			// FIXME: Uncomment this and fix id when we have FullDataSource loader!
-			//LodUtil.assertTrue(loader != null, "No loader for {} (v{})", data.getClass(), data.getDataVersion());
-			dataType = data.getClass();
-			dataTypeId = loader == null ? 0 : loader.datatypeId;
-			loaderVersion = data.getDataVersion();
-			timestamp = System.currentTimeMillis(); // TODO: Do we need to use server synced time?
-			// Warn: This may become an attack vector! Be careful!
-			super.writeData((out) -> {
-				try {
-					data.saveData(level, this, out);
-				} catch (IOException e) {
-					LOGGER.error("Failed to save data for file {}", path, e);
-				}
-			});
-		} catch (IOException e) {
-			LOGGER.error("Failed to write data for file {}", path, e);
-		}
-	}
-
-
 	public static void debugCheck() {
 		DataObjTracker phantom = (DataObjTracker) lifeCycleDebugQueue.poll();
 		while (phantom != null) {

core/src/main/java/com/seibel/lod/core/a7/save/io/file/GeneratedDataFileHandler.java

@@ -1,12 +1,149 @@
 package com.seibel.lod.core.a7.save.io.file;
 
+import com.seibel.lod.core.a7.datatype.LodDataSource;
+import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
+import com.seibel.lod.core.a7.datatype.full.FullDataSource;
+import com.seibel.lod.core.a7.datatype.full.SparseDataSource;
 import com.seibel.lod.core.a7.generation.GenerationQueue;
 import com.seibel.lod.core.a7.level.IServerLevel;
+import com.seibel.lod.core.a7.pos.DhSectionPos;
+import com.seibel.lod.core.logging.DhLoggerBuilder;
+import com.seibel.lod.core.util.LodUtil;
+import org.apache.logging.log4j.Logger;
 
 import java.io.File;
+import java.lang.ref.WeakReference;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
 public class GeneratedDataFileHandler extends DataFileHandler {
-    public GeneratedDataFileHandler(IServerLevel level, File saveRootDir, GenerationQueue queue) {
-        super(level, saveRootDir, queue::generate);
+    private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+
+    AtomicReference<GenerationQueue> queue = new AtomicReference<>(null);
+    // TODO: Should I include a lib that impl weak concurrent hash map?
+    final Map<LodDataSource, GenTask> genQueue = Collections.synchronizedMap(new WeakHashMap<>());
+
+    class GenTask extends GenerationQueue.GenTaskTracker {
+        final DhSectionPos pos;
+        WeakReference<LodDataSource> targetData;
+        LodDataSource loadedTargetData = null;
+        GenTask(DhSectionPos pos, WeakReference<LodDataSource> targetData) {
+            this.pos = pos;
+            this.targetData = targetData;
+        }
+        @Override
+        public boolean isValid() {
+            return targetData.get() != null;
+        }
+        @Override
+        public Consumer<ChunkSizedData> getConsumer() {
+            if (loadedTargetData == null) {
+                loadedTargetData = targetData.get();
+                if (loadedTargetData == null) return null;
+            }
+            return (chunk) -> {
+                if (chunk.getBBoxLodPos().overlaps(loadedTargetData.getSectionPos().getSectionBBoxPos()))
+                    write(loadedTargetData.getSectionPos(), chunk);
+            };
+        }
+
+        void releaseStrongReference() {
+            loadedTargetData = null;
+        }
+    }
+
+
+    public GeneratedDataFileHandler(IServerLevel level, File saveRootDir) {
+        super(level, saveRootDir);
+    }
+
+    public void setGenerationQueue(GenerationQueue newQueue) {
+        boolean worked = queue.compareAndSet(null, newQueue);
+        LodUtil.assertTrue(worked, "previous queue is still here!");
+        synchronized (genQueue) {
+            for (Map.Entry<LodDataSource, GenTask> entry : genQueue.entrySet()) {
+                LodDataSource source = entry.getKey();
+                DhSectionPos pos = source.getSectionPos();
+                GenTask task = entry.getValue();
+                queue.get().submitGenTask(pos.getSectionBBoxPos(), source.getDataDetail(), task)
+                        .whenComplete(
+                                (b, ex) -> {
+                                    if (ex != null) LOGGER.error("Uncaught Gen Task Exception at {}:", pos, ex);
+                                    LodDataSource data = task.targetData.get();
+                                    if (ex == null && b) {
+                                        files.get(task.pos).metaData.dataVersion.incrementAndGet();
+                                        genQueue.remove(data, task);
+                                        return;
+                                    }
+                                    task.releaseStrongReference();
+                                }
+                        );
+            }
+        }
+    }
+
+    public GenerationQueue popGenerationQueue() {
+        GenerationQueue cas = queue.getAndSet(null);
+        LodUtil.assertTrue(cas != null, "there are no previous live generation queue!");
+        return cas;
+    }
+
+    @Override
+    public CompletableFuture<LodDataSource> onCreateDataFile(DataMetaFile file) {
+        DhSectionPos pos = file.pos;
+        ArrayList<DataMetaFile> existFiles = new ArrayList<>();
+        ArrayList<DhSectionPos> missing = new ArrayList<>();
+        selfSearch(pos, pos, existFiles, missing);
+        LodUtil.assertTrue(!missing.isEmpty() || !existFiles.isEmpty());
+        if (missing.size() == 1 && existFiles.isEmpty() && missing.get(0).equals(pos)) {
+            // None exist.
+            SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
+            GenerationQueue getQueue = queue.get();
+            GenTask task = new GenTask(pos, new WeakReference<>(dataSource));
+            genQueue.put(dataSource, task);
+            if (getQueue != null) {
+                getQueue.submitGenTask(dataSource.getSectionPos().getSectionBBoxPos(),
+                        dataSource.getDataDetail(), task)
+                        .whenComplete(
+                        (b, ex) -> {
+                            if (ex != null) LOGGER.error("Uncaught Gen Task Exception at {}:", pos, ex);
+                            LodDataSource data = task.targetData.get();
+                            if (ex == null && b) {
+                                files.get(task.pos).metaData.dataVersion.incrementAndGet();
+                                genQueue.remove(data, task);
+                                return;
+                            }
+                            task.releaseStrongReference();
+                        }
+                );
+            }
+            return CompletableFuture.completedFuture(dataSource);
+        } else {
+            for (DhSectionPos missingPos : missing) {
+                DataMetaFile newfile = atomicGetOrMakeFile(missingPos);
+                if (newfile != null) existFiles.add(newfile);
+            }
+            final ArrayList<CompletableFuture<Void>> futures = new ArrayList<>(existFiles.size());
+            final SparseDataSource dataSource = SparseDataSource.createEmpty(pos);
+
+            for (DataMetaFile f : existFiles) {
+                futures.add(f.loadOrGetCached()
+                        .exceptionally((ex) -> null)
+                        .thenAccept((data) -> {
+                            if (data != null) {
+                                if (data instanceof SparseDataSource)
+                                    dataSource.sampleFrom((SparseDataSource) data);
+                                else if (data instanceof FullDataSource)
+                                    dataSource.sampleFrom((FullDataSource) data);
+                                else LodUtil.assertNotReach();
+                            }
+                        })
+                );
+            }
+            return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new))
+                    .thenApply((v) -> dataSource.trySelfPromote());
+        }
     }
 }

core/src/main/java/com/seibel/lod/core/a7/save/io/file/IDataSourceProvider.java

@@ -3,12 +3,17 @@ package com.seibel.lod.core.a7.save.io.file;
 import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
 import com.seibel.lod.core.a7.datatype.full.FullFormat;
+import com.seibel.lod.core.a7.pos.DhLodPos;
 import com.seibel.lod.core.a7.pos.DhSectionPos;
 import com.seibel.lod.core.objects.DHChunkPos;
 
+import javax.annotation.Nullable;
 import java.io.File;
+import java.nio.file.Path;
 import java.util.Collection;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
 
 public interface IDataSourceProvider extends AutoCloseable {
     void addScannedFile(Collection<File> detectedFiles);
@@ -17,5 +22,12 @@ public interface IDataSourceProvider extends AutoCloseable {
     void write(DhSectionPos sectionPos, ChunkSizedData chunkData);
     CompletableFuture<Void> flushAndSave();
 
-    boolean isCacheValid(DhSectionPos sectionPos, long timestamp);
+    long getLatestCacheVersion(DhSectionPos sectionPos);
+
+    CompletableFuture<LodDataSource> onCreateDataFile(DataMetaFile file);
+    LodDataSource onDataFileLoaded(LodDataSource source, Consumer<LodDataSource> updater);
+    LodDataSource onDataFileRefresh(LodDataSource source, Consumer<LodDataSource> updater);
+    File computeDataFilePath(DhSectionPos pos);
+    Executor getIOExecutor();
+
 }

core/src/main/java/com/seibel/lod/core/a7/save/io/file/RemoteDataFileHandler.java

@@ -7,9 +7,6 @@ import java.io.File;
 
 public class RemoteDataFileHandler extends DataFileHandler {
     public RemoteDataFileHandler(ILevel level, File saveRootDir) {
-        super(level, saveRootDir, (pos) -> {
-            LodUtil.assertNotReach("TODO");
-            return null;
-        });
+        super(level, saveRootDir);
     }
 }

core/src/main/java/com/seibel/lod/core/a7/save/io/render/IRenderSourceProvider.java

@@ -13,4 +13,5 @@ public interface IRenderSourceProvider extends AutoCloseable {
     void addScannedFile(Collection<File> detectedFiles);
     void write(DhSectionPos sectionPos, ChunkSizedData chunkData);
     CompletableFuture<Void> flushAndSave();
+    boolean refreshRenderSource(LodRenderSource source);
 }

core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderFileHandler.java

@@ -1,21 +1,26 @@
 package com.seibel.lod.core.a7.save.io.render;
 
 import com.google.common.collect.HashMultimap;
+import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.PlaceHolderRenderSource;
 import com.seibel.lod.core.a7.datatype.LodRenderSource;
+import com.seibel.lod.core.a7.datatype.RenderSourceLoader;
 import com.seibel.lod.core.a7.datatype.column.ColumnRenderSource;
 import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
+import com.seibel.lod.core.a7.datatype.transform.DataRenderTransformer;
 import com.seibel.lod.core.a7.level.IClientLevel;
 import com.seibel.lod.core.a7.pos.DhLodPos;
 import com.seibel.lod.core.a7.save.io.file.IDataSourceProvider;
 import com.seibel.lod.core.a7.pos.DhSectionPos;
+import com.seibel.lod.core.a7.util.UncheckedInterruptedException;
+import com.seibel.lod.core.config.Config;
 import com.seibel.lod.core.logging.DhLoggerBuilder;
 import com.seibel.lod.core.util.LodUtil;
 import org.apache.logging.log4j.Logger;
 
-import javax.annotation.Nullable;
 import java.io.File;
 import java.io.IOException;
+import java.lang.ref.WeakReference;
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -45,11 +50,7 @@ public class RenderFileHandler implements IRenderSourceProvider {
         { // Sort files by pos.
             for (File file : detectedFiles) {
                 try {
-                    RenderMetaFile metaFile = new RenderMetaFile(
-                            dataSourceProvider::isCacheValid,
-                            dataSourceProvider::read,
-                            level, file
-                    );
+                    RenderMetaFile metaFile = new RenderMetaFile(this, file);
                     filesByPos.put(metaFile.pos, metaFile);
                 } catch (IOException e) {
                     throw new RuntimeException(e);
@@ -57,12 +58,12 @@ public class RenderFileHandler implements IRenderSourceProvider {
             }
         }
 
-        // Warn for multiple files with the same pos, and then select the one with latest timestamp.
+        // Warn for multiple files with the same pos, and then select the one with the latest timestamp.
         for (DhSectionPos pos : filesByPos.keySet()) {
             Collection<RenderMetaFile> metaFiles = filesByPos.get(pos);
             RenderMetaFile fileToUse;
             if (metaFiles.size() > 1) {
-                fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.timestamp));
+                fileToUse = Collections.max(metaFiles, Comparator.comparingLong(a -> a.metaData.dataVersion.get()));
                 {
                     StringBuilder sb = new StringBuilder();
                     sb.append("Multiple files with the same pos: ");
@@ -103,19 +104,25 @@ public class RenderFileHandler implements IRenderSourceProvider {
      */
     @Override
     public CompletableFuture<LodRenderSource> read(DhSectionPos pos) {
-        RenderMetaFile metaFile = files.computeIfAbsent(pos, (p) -> new RenderMetaFile(
-                dataSourceProvider::isCacheValid,
-                dataSourceProvider::read,
-                level, computeDefaultFilePath(p), p));
-
-        return metaFile.loadOrGetCached(renderCacheThread).handle(
+        RenderMetaFile metaFile = files.get(pos);
+        if (metaFile == null) {
+            RenderMetaFile newMetaFile;
+            try {
+                newMetaFile = new RenderMetaFile(this, pos);
+            } catch (IOException e) {
+                LOGGER.error("IOException on creating new render file at {}", pos, e);
+                return null;
+            }
+            metaFile = files.putIfAbsent(pos, newMetaFile); // This is a CAS with expected null value.
+            if (metaFile == null) metaFile = newMetaFile;
+        }
+        return metaFile.loadOrGetCached(renderCacheThread, level).handle(
                 (render, e) -> {
                     if (e != null) {
                         LOGGER.error("Uncaught error on {}:", pos, e);
                     }
                     if (render != null) return render;
-                    PlaceHolderRenderSource placeHolder = new PlaceHolderRenderSource(pos);
-                    return placeHolder;
+                    return new PlaceHolderRenderSource(pos);
                 }
         );
     }
@@ -125,8 +132,8 @@ public class RenderFileHandler implements IRenderSourceProvider {
      */
     @Override
     public void write(DhSectionPos sectionPos, ChunkSizedData chunkData) {
-        dataSourceProvider.write(sectionPos, chunkData);
         recursive_write(sectionPos,chunkData);
+        dataSourceProvider.write(sectionPos, chunkData);
     }
 
     private void recursive_write(DhSectionPos sectPos, ChunkSizedData chunkData) {
@@ -141,7 +148,6 @@ public class RenderFileHandler implements IRenderSourceProvider {
         if (metaFile != null) { // Fast path: if there is a file for this section, just write to it.
             metaFile.updateChunkIfNeeded(chunkData);
         }
-
     }
 
     /*
@@ -168,4 +174,89 @@ public class RenderFileHandler implements IRenderSourceProvider {
         }
         CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
     }
+
+    public File computeRenderFilePath(DhSectionPos pos) {
+        return new File(saveDir, pos.serialize() + ".lod");
+    }
+
+    public CompletableFuture<LodRenderSource> onCreateRenderFile(RenderMetaFile file) {
+        final int vertSize = Config.Client.Graphics.Quality.verticalQuality
+                .get().calculateMaxVerticalData((byte) (file.pos.sectionDetail - ColumnRenderSource.SECTION_SIZE_OFFSET));
+        return CompletableFuture.completedFuture(
+                new ColumnRenderSource(file.pos, vertSize, level.getMinY()));
+    }
+
+    private final ConcurrentHashMap<DhSectionPos, Object> cacheRecreationGuards = new ConcurrentHashMap<>();
+
+    private void updateCache(LodRenderSource data, RenderMetaFile file) {
+        if (cacheRecreationGuards.putIfAbsent(file.pos, new Object()) != null) return;
+        final WeakReference<LodRenderSource> dataRef = new WeakReference<>(data);
+        CompletableFuture<LodDataSource> dataFuture = dataSourceProvider.read(data.getSectionPos());
+        final long version = dataSourceProvider.getLatestCacheVersion(data.getSectionPos());
+                DataRenderTransformer.asyncTransformDataSource(
+                        dataFuture.thenApply((d) -> {
+                            if (dataRef.get() == null) throw new UncheckedInterruptedException();
+                            LodUtil.assertTrue(d != null);
+                            return d;
+                        }).exceptionally((ex) -> {
+                            if (ex != null)
+                                LOGGER.error("Uncaught exception when getting data for updateCache()", ex);
+                            return null;
+                        })
+                        , level)
+                .thenAccept((newData) -> write(dataRef.get(), file, newData, version))
+                .exceptionally((ex) -> {
+                    if (!UncheckedInterruptedException.isThrowableInterruption(ex))
+                        LOGGER.error("Exception when updating render file using data source: ", ex);
+                    return null;
+                }).thenRun(() -> cacheRecreationGuards.remove(file.pos));
+
+    }
+
+    public LodRenderSource onRenderFileLoaded(LodRenderSource data, RenderMetaFile file) {
+        long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos);
+        //NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
+        if (newCacheVersion - file.metaData.dataVersion.get() <= 0)
+            return data;
+        updateCache(data, file);
+        return data;
+    }
+
+    public LodRenderSource onLoadingRenderFile(RenderMetaFile file) {
+        return null; //Default behaviour
+    }
+
+    private void write(LodRenderSource target, RenderMetaFile file,
+                       LodRenderSource newData, long newDataVersion) {
+        if (target == null) return;
+        if (newData == null) return;
+        target.weakWrite(newData);
+        file.metaData.dataVersion.set(newDataVersion);
+        file.metaData.dataLevel = target.getDataDetail();
+        file.loader = RenderSourceLoader.getLoader(target.getClass(), target.getRenderVersion());
+        file.dataType = target.getClass();
+        file.metaData.dataTypeId = file.loader.renderTypeId;
+        file.metaData.loaderVersion = target.getRenderVersion();
+        file.save(target, level);
+    }
+
+    public void onReadRenderSourceFromCache(RenderMetaFile file, LodRenderSource data) {
+        long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos);
+        //NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
+        if (newCacheVersion - file.metaData.dataVersion.get() > 0)
+            updateCache(data, file);
+    }
+
+    public boolean refreshRenderSource(LodRenderSource source) {
+        RenderMetaFile file = files.get(source.getSectionPos());
+        LodUtil.assertTrue(file != null);
+        LodUtil.assertTrue(file.doesFileExist);
+        long newCacheVersion = dataSourceProvider.getLatestCacheVersion(file.pos);
+        //NOTE: Do this instead of direct compare so values that wrapped around still works correctly.
+        if (newCacheVersion - file.metaData.dataVersion.get() <= 0)
+            return false;
+        updateCache(source, file);
+        return true;
+    }
+
 }

core/src/main/java/com/seibel/lod/core/a7/save/io/render/RenderMetaFile.java

@@ -1,15 +1,19 @@
 package com.seibel.lod.core.a7.save.io.render;
 
+import com.seibel.lod.core.a7.datatype.DataSourceLoader;
 import com.seibel.lod.core.a7.datatype.LodDataSource;
 import com.seibel.lod.core.a7.datatype.LodRenderSource;
 import com.seibel.lod.core.a7.datatype.RenderSourceLoader;
 import com.seibel.lod.core.a7.datatype.full.ChunkSizedData;
 import com.seibel.lod.core.a7.datatype.transform.DataRenderTransformer;
 import com.seibel.lod.core.a7.level.IClientLevel;
+import com.seibel.lod.core.a7.level.ILevel;
 import com.seibel.lod.core.a7.pos.DhLodPos;
 import com.seibel.lod.core.a7.save.io.MetaFile;
 import com.seibel.lod.core.a7.pos.DhSectionPos;
 import com.seibel.lod.core.logging.DhLoggerBuilder;
+import com.seibel.lod.core.a7.save.io.file.DataMetaFile;
+import com.seibel.lod.core.a7.save.io.file.IDataSourceProvider;
 import com.seibel.lod.core.util.LodUtil;
 import org.apache.logging.log4j.Logger;
 
@@ -24,7 +28,8 @@ public class RenderMetaFile extends MetaFile
 {
 	private static final Logger LOGGER = DhLoggerBuilder.getLogger(RenderMetaFile.class.getSimpleName());
 	
-    private final IClientLevel level;
+    //private final IClientLevel level;
+	
     public RenderSourceLoader loader;
     public Class<? extends LodRenderSource> dataType;
 
@@ -62,29 +67,29 @@ public class RenderMetaFile extends MetaFile
     }
     CacheValidator validator;
     CacheSourceProducer source;
+    final RenderFileHandler handler;
+    boolean doesFileExist;
 
-    // Load a metaFile in this path. It also automatically read the metadata.
-    public RenderMetaFile(CacheValidator validator, CacheSourceProducer source,
-                          IClientLevel level, File path) throws IOException {
-        super(path);
-        this.level = level;
-        loader = RenderSourceLoader.getLoader(dataTypeId, loaderVersion);
-        if (loader == null) {
-            throw new IOException("Invalid file: Data type loader not found: "
-                    + dataTypeId + "(v" + loaderVersion + ")");
-        }
-        dataType = loader.clazz;
-        this.validator = validator;
-        this.source = source;
+
+    // Create a new metaFile
+    public RenderMetaFile(RenderFileHandler handler, DhSectionPos pos) throws IOException {
+        super(handler.computeRenderFilePath(pos), pos);
+        this.handler = handler;
+        LodUtil.assertTrue(metaData == null);
+        doesFileExist = false;
     }
 
-    // Make a new MetaFile. It doesn't load or write any metadata itself.
-    public RenderMetaFile(CacheValidator validator, CacheSourceProducer source,
-                          IClientLevel level, File path, DhSectionPos pos) {
-        super(path, pos);
-        this.level = level;
-        this.validator = validator;
-        this.source = source;
+    public RenderMetaFile(RenderFileHandler handler, File path) throws IOException {
+        super(path);
+        this.handler = handler;
+        LodUtil.assertTrue(metaData != null);
+        loader = RenderSourceLoader.getLoader(metaData.dataTypeId, metaData.loaderVersion);
+        if (loader == null) {
+            throw new IOException("Invalid file: Data type loader not found: "
+                    + metaData.dataTypeId + "(v" + metaData.loaderVersion + ")");
+        }
+        dataType = loader.clazz;
+        doesFileExist = true;
     }
 
     // Suppress casting of CompletableFuture<?> to CompletableFuture<LodRenderSource>
@@ -95,6 +100,7 @@ public class RenderMetaFile extends MetaFile
             Object inner = ((SoftReference<?>)obj).get();
             if (inner != null) {
                 LodUtil.assertTrue(inner instanceof LodRenderSource);
+                handler.onReadRenderSourceFromCache(this, (LodRenderSource) inner);
                 return CompletableFuture.completedFuture((LodRenderSource)inner);
             }
         }
@@ -109,7 +115,7 @@ public class RenderMetaFile extends MetaFile
 
     // Cause: Generic Type runtime casting cannot safety check it.
     // However, the Union type ensures the 'data' should only contain the listed type.
-    public CompletableFuture<LodRenderSource> loadOrGetCached(Executor fileReaderThreads) {
+    public CompletableFuture<LodRenderSource> loadOrGetCached(Executor fileReaderThreads, ILevel level) {
         Object obj = data.get();
 
         CompletableFuture<LodRenderSource> cached = _readCached(obj);
@@ -122,49 +128,68 @@ public class RenderMetaFile extends MetaFile
 
         // Would use faster and non-nesting Compare and exchange. But java 8 doesn't have it! :(
         boolean worked = data.compareAndSet(obj, future);
-        if (!worked) return loadOrGetCached(fileReaderThreads);
+        if (!worked) return loadOrGetCached(fileReaderThreads, level);
 
         // Now, there should only ever be one thread at a time here due to the CAS operation above.
 
-        // Would use CompletableFuture.completeAsync(...), But, java 8 doesn't have it! :(
-        //return future.completeAsync(this::loadAndUpdateRenderSource, fileReaderThreads);
-        CompletableFuture.supplyAsync(() -> buildFuture(fileReaderThreads), fileReaderThreads)
-                .thenCompose((sourceCompletableFuture) -> sourceCompletableFuture)
-                .whenComplete((renderSource, e) -> {
-            if (e != null) {
-                LOGGER.error("Uncaught error loading file {}: ", path, e);
-                future.complete(null);
-                data.set(null);
-            } else {
-                future.complete(renderSource);
-                data.set(new SoftReference<>(renderSource));
-            }
-        });
+
+        // After cas. We are in exclusive control.
+        if (!doesFileExist) {
+            doesFileExist = true;
+            handler.onCreateRenderFile(this)
+                    .thenApply((data) -> {
+                        metaData = makeMetaData(data);
+                        return data;
+                    })
+                    .thenApply((d) -> handler.onRenderFileLoaded(d, this))
+                    .whenComplete((v, e) -> {
+                        if (e != null) {
+                            LOGGER.error("Uncaught error on creation {}: ", path, e);
+                            future.complete(null);
+                            data.set(null);
+                        } else {
+                            future.complete(v);
+                            //new DataObjTracker(v); //TODO: Obj Tracker??? For debug?
+                            data.set(new SoftReference<>(v));
+                        }
+                    });
+
+
+        } else {
+            CompletableFuture.supplyAsync(() -> {
+                        if (metaData == null)
+                            throw new IllegalStateException("Meta data not loaded!");
+                        // Load the file.
+                        LodRenderSource data;
+                        data = handler.onLoadingRenderFile(this);
+                        if (data == null) {
+                            try (FileInputStream fio = getDataContent()) {
+                                data = loader.loadRender(this, fio, level);
+                            } catch (IOException e) {
+                                throw new CompletionException(e);
+                            }
+                        }
+                        data = handler.onRenderFileLoaded(data, this);
+                        return data;
+                    }, fileReaderThreads)
+                    .whenComplete((f, e) -> {
+                        if (e != null) {
+                            LOGGER.error("Error loading file {}: ", path, e);
+                            future.complete(null);
+                            data.set(null);
+                        } else {
+                            future.complete(f);
+                            data.set(new SoftReference<>(f));
+                        }
+                    });
+        }
         return future;
     }
 
-    private CompletableFuture<LodRenderSource> buildFuture(Executor executorService) {
-        if (path.exists()) {
-            try {
-                updateMetaData();
-                if (validator.isCacheValid(pos, timestamp)) {
-                    // Load the file.
-                    try (FileInputStream fio = getDataContent()) {
-                        return CompletableFuture.completedFuture(
-                                loader.loadRender(this, fio, level));
-                    }
-                }
-            } catch (IOException e) {
-                LOGGER.warn("Failed to read render cache at {}:", path, e);
-                LOGGER.warn("Will delete cache file.");
-                path.delete();
-            }
-        }
-        // Otherwise, re-query and make the RenderSource
-        CompletableFuture<LodDataSource> dataFuture = source.getSourceFuture(pos);
-        return dataFuture.thenCombineAsync(
-                DataRenderTransformer.asyncTransformDataSource(dataFuture, level),
-                this::write, executorService);
+    private static MetaData makeMetaData(LodRenderSource data) {
+        RenderSourceLoader loader = RenderSourceLoader.getLoader(data.getClass(), data.getRenderVersion());
+        return new MetaData(data.getSectionPos(), -1, -1,
+                data.getDataDetail(), loader == null ? 0 : loader.renderTypeId, data.getRenderVersion());
     }
 
     private FileInputStream getDataContent() throws IOException {
@@ -183,36 +208,13 @@ public class RenderMetaFile extends MetaFile
         return fin;
     }
 
-    @Override
-    protected void updateMetaData() throws IOException {
-        super.updateMetaData();
-        loader = RenderSourceLoader.getLoader(dataTypeId, loaderVersion);
-        if (loader == null) {
-            throw new IOException("Invalid file: Data type loader not found: " + dataTypeId + "(v" + loaderVersion + ")");
-        }
-        dataType = loader.clazz;
-        dataTypeId = loader.renderTypeId;
-    }
-
-    private LodRenderSource write(LodDataSource parent, LodRenderSource render) {
-        if (parent == null) return null;
+    public void save(LodRenderSource data, IClientLevel level) {
+        LodUtil.assertTrue(data == _readCached(this.data.get()).getNow(null));
+        LOGGER.info("Saving updated render file v[{}] at sect {}", metaData.dataVersion.get(), pos);
         try {
-            //TODO: Update Timestamp & stuff based on parent
-            dataLevel = parent.getDataDetail();
-            loader = RenderSourceLoader.getLoader(render.getClass(), render.getRenderVersion());
-            dataType = render.getClass();
-            dataTypeId = loader.renderTypeId;
-            loaderVersion = render.getRenderVersion();
-            super.writeData((out) -> {
-                try {
-                    render.saveRender(level, this, out);
-                } catch (IOException e) {
-                    LOGGER.error("Failed to save data for file {}", path, e);
-                }
-            });
+            super.writeData((out) -> data.saveRender(level, this, out));
         } catch (IOException e) {
-            LOGGER.error("Failed to write data for file {}", path, e);
+            LOGGER.error("Failed to save updated render file at {} for sect {}", path, pos, e);
         }
-        return render;
     }
 }

core/src/main/java/com/seibel/lod/core/builders/worldGeneration/BatchGenerator.java

@@ -40,6 +40,7 @@ import org.apache.logging.log4j.Logger;
 
 import java.lang.invoke.MethodHandles;
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
 
 public class BatchGenerator implements IChunkGenerator
 {
@@ -264,7 +265,7 @@ public class BatchGenerator implements IChunkGenerator
 	}
 
 	@Override
-	public CompletableFuture<ArrayGridList<IChunkWrapper>> generateChunks(DHChunkPos chunkPosMin, byte granularity) {
+	public CompletableFuture<Void> generateChunks(DHChunkPos chunkPosMin, byte granularity, byte targetDataDetail, Consumer<IChunkWrapper> resultConsumer) {
 		EDistanceGenerationMode mode = Config.Client.WorldGenerator.distanceGenerationMode.get();
 		Steps targetStep = null;
 		switch (mode) {
@@ -286,17 +287,26 @@ public class BatchGenerator implements IChunkGenerator
 				break;
 		};
 
-
 		int chunkXMin = chunkPosMin.x;
 		int chunkZMin = chunkPosMin.z;
 		int genChunkSize = 1 << (granularity - 4); // minus 4 for chunk size as its equal to div by 16
 		double runTimeRatio = Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get()>1 ? 1.0
 				: Config.Client.Advanced.Threading.numberOfWorldGenerationThreads.get();
-		return generationGroup.generateChunks(chunkXMin, chunkZMin, genChunkSize, targetStep, runTimeRatio);
+		return generationGroup.generateChunks(chunkXMin, chunkZMin, genChunkSize, targetStep, runTimeRatio, resultConsumer);
 	}
 
 	@Override
-	public byte getDataDetail() {
+	public byte getMinDataDetail() {
+		return 0;
+	}
+
+	@Override
+	public byte getMaxDataDetail() {
+		return 0;
+	}
+
+	@Override
+	public int getPriority() {
 		return 0;
 	}
 
@@ -307,7 +317,7 @@ public class BatchGenerator implements IChunkGenerator
 
 	@Override
 	public byte getMaxGenerationGranularity() {
-		return 8;
+		return 6;
 	}
 
 	@Override
@@ -318,5 +328,4 @@ public class BatchGenerator implements IChunkGenerator
 	public void update() {
 		generationGroup.updateAllFutures();
 	}
-
 }

core/src/main/java/com/seibel/lod/core/util/Atomics.java

@@ -4,6 +4,7 @@ import it.unimi.dsi.fastutil.booleans.BooleanObjectImmutablePair;
 
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.function.Predicate;
 
 public class Atomics {
     // While java 8 does have built in atomic operations, there doesn't seem to be any Compare And Exchange operation...
@@ -26,6 +27,23 @@ public class Atomics {
         }
     }
 
+    public static <T> T conditionalAndExchange(AtomicReference<T> atomic, Predicate<T> requirement, T newValue) {
+        while (true) {
+            T oldValue = atomic.get();
+            if (!requirement.test(oldValue)) return oldValue;
+            if (atomic.weakCompareAndSet(oldValue, newValue)) return oldValue;
+        }
+    }
+
+    public static <T> BooleanObjectImmutablePair<T> conditionalAndExchangeWeak(AtomicReference<T> atomic, Predicate<T> requirement, T newValue) {
+        T oldValue = atomic.get();
+        if (requirement.test(oldValue) && atomic.weakCompareAndSet(oldValue, newValue)) {
+            return new BooleanObjectImmutablePair<>(true, oldValue);
+        } else {
+            return new BooleanObjectImmutablePair<>(false, oldValue);
+        }
+    }
+
     // Additionally, we implement some helper methods for frequently used atomic operations.
 
     // Compare with expected value and set new value if equal. Then return whatever value the atomic now contains.

core/src/main/java/com/seibel/lod/core/util/LodUtil.java

@@ -432,6 +432,10 @@ public class LodUtil
 	public static void assertNotReach(String message, Object... args) {
 		throw new AssertFailureException("Assert Not Reach failed:\n " + formatLog(message, args));
 	}
+	public static void assertToDo() {
+		throw new AssertFailureException("TODO!");
+	}
+
 	public static ExecutorService makeSingleThreadPool(String name, int relativePriority) {
 		return Executors.newFixedThreadPool(1, new LodThreadFactory(name, Thread.NORM_PRIORITY+relativePriority));
 	}

core/src/main/java/com/seibel/lod/core/wrapperInterfaces/worldGeneration/AbstractBatchGenerationEnvionmentWrapper.java

@@ -20,10 +20,10 @@
 package com.seibel.lod.core.wrapperInterfaces.worldGeneration;
 
 import com.seibel.lod.core.a7.level.ILevel;
-import com.seibel.lod.core.util.gridList.ArrayGridList;
 import com.seibel.lod.core.wrapperInterfaces.chunk.IChunkWrapper;
 
 import java.util.concurrent.CompletableFuture;
+import java.util.function.Consumer;
 
 public abstract class AbstractBatchGenerationEnvionmentWrapper {
 	public enum Steps {
@@ -41,5 +41,5 @@ public abstract class AbstractBatchGenerationEnvionmentWrapper {
 
 	public abstract void stop(boolean blocking);
 
-	public abstract CompletableFuture<ArrayGridList<IChunkWrapper>> generateChunks(int minX, int minZ, int genSize, Steps targetStep, double runTimeRatio);
+	public abstract CompletableFuture<Void> generateChunks(int minX, int minZ, int genSize, Steps targetStep, double runTimeRatio, Consumer<IChunkWrapper> resultConsumer);
 }