wisplite/distant-horizons-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

now compressing using proper algorithm

Browse Source
This commit is contained in:
cola98765
2021-10-18 11:00:58 +02:00
parent d0f93a2c89
commit 96c19620cb
4 changed files with 35 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.gradle
+1
View File
@@ -131,6 +131,7 @@ dependencies {
// that the dep is a ForgeGradle 'patcher' dependency. And it's patches will be applied.
// The userdev artifact is a special name and will get all sorts of transformations applied to it.
minecraft 'net.minecraftforge:forge:1.16.5-36.1.0'
compile group: 'org.tukaani', name: 'xz', version: '1.9'
// these aren't needed right now
//implementation ('com.github.KaptainWutax:TerrainUtils:1.0.0'){
src/main/java/com/seibel/lod/handlers/LodDimensionFileHandler.java
+6 -3
View File
@@ -27,6 +27,9 @@ import com.seibel.lod.objects.VerticalLevelContainer;
import com.seibel.lod.proxy.ClientProxy;
import com.seibel.lod.util.LodThreadFactory;
import com.seibel.lod.util.LodUtil;
import com.seibel.lod.util.ThreadMapUtil;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import java.io.*;
import java.nio.file.Files;
@@ -156,7 +159,7 @@ public class LodDimensionFileHandler
dataSize -= 1;
if (dataSize > 0)
{
try (InputStream inputStream = new BufferedInputStream(new FileInputStream(file)))
try (XZCompressorInputStream inputStream = new XZCompressorInputStream(new FileInputStream(file)))
{
int fileVersion;
fileVersion = inputStream.read();
@@ -192,7 +195,7 @@ public class LodDimensionFileHandler
// this file is a readable version,
// read the file
byte[] data = new byte[(int) dataSize];
byte[] data = ThreadMapUtil.getSaveContainer(tempDetailLevel);
inputStream.read(data);
inputStream.close();
@@ -324,7 +327,7 @@ public class LodDimensionFileHandler
// the old file is good, now create a new temporary save file
File newFile = new File(fileName + TMP_FILE_EXTENSION);
try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(newFile)))
try (XZCompressorOutputStream outputStream = new XZCompressorOutputStream(new FileOutputStream(newFile), 3))
{
// add the version of this file
outputStream.write(LOD_SAVE_FILE_VERSION);
src/main/java/com/seibel/lod/objects/VerticalLevelContainer.java
+9 -70
View File
@@ -115,44 +115,10 @@ public class VerticalLevelContainer implements LevelContainer
for (int i = 0; i < x; i++)
{
newData = 0;
if (counter > -1)
{
dataContainer[i] = last;
if (last == 3)
{ //skip rest of void chunk
for (tempIndex = 1; tempIndex < maxVerticalData; tempIndex++)
{
dataContainer[i + tempIndex] = 0;
}
i += maxVerticalData - 1;
}
counter--;
}
else if ((inputData[index] & 0x3) == 0 || (inputData[index] & 0x3) == 3)
{
last = (byte) (inputData[index] & 0x3);
//recover counter
counter = (inputData[index] & 0x7c) >>> 2;
tempIndex = 0;
while ((inputData[index] & 0x80) == 0x80)
{ //overflow bit is on
index++;
counter += (inputData[index] & 0x7f) << (5 + 7 * tempIndex);
tempIndex++;
}
index++;
//since loop expects from us to put some data in, we just make it rerun it with new counter;
i--;
}
else if (index + 7 >= inputData.length)
break;
else
{
for (tempIndex = 0; tempIndex < 8; tempIndex++)
newData += (((long) inputData[index + tempIndex]) & 0xff) << (8 * tempIndex);
index = index + 8;
dataContainer[i] = newData;
}
for (tempIndex = 0; tempIndex < 8; tempIndex++)
newData += (((long) inputData[index + tempIndex]) & 0xff) << (8 * tempIndex);
index += 8;
dataContainer[i] = newData;
}
}
@@ -205,7 +171,7 @@ public class VerticalLevelContainer implements LevelContainer
int tempIndex;
long current;
byte[] tempData = ThreadMapUtil.getSaveContainer(2 + (x * 8));
byte[] tempData = ThreadMapUtil.getSaveContainer(detailLevel);
tempData[index] = detailLevel;
index++;
@@ -215,38 +181,11 @@ public class VerticalLevelContainer implements LevelContainer
for (int i = 0; i < x; i++)
{
current = dataContainer[i];
if ((current & 0b11) == 0 || (current & 0b11) == 3)
{
current &= 0b11; //clean any garbage data after those two bits
last = (byte) current;
if (current == 3) //skip rest of void chunk
i += maxVerticalData - 1;
counter++;
}
else
{
for (tempIndex = 0; tempIndex < 8; tempIndex++)
tempData[index + tempIndex] = (byte) (current >>> (8 * tempIndex));
index += 8;
}
if (last != -1 && (i == x - 1 || last != ((dataContainer[i + 1]) & 0b11)))
{ //save compressed data if next is different or if we reached onf of the data
tempData[index] = (byte) (0x7f & ((counter << 2) + last)); //save 5 bits of counter and compressed block
tempIndex = 0;
while ((counter >>> (5 + 7 * tempIndex)) != 0) //there is more of that counter
{
tempData[index] = (byte) (tempData[index] | 0x80); //set overflow bit to true
index++; // after setting overflow bit w can actually index++
tempData[index] = (byte) (0x7f & (counter >>> (5 + 7 * tempIndex))); // save 7 bits of counter
tempIndex++;
}
index++;
last = -1;
counter = -1;
}
for (tempIndex = 0; tempIndex < 8; tempIndex++)
tempData[index + tempIndex] = (byte) (current >>> (8 * tempIndex));
index += 8;
}
return Arrays.copyOfRange(tempData, 0, index);
return tempData;
}
@Override
src/main/java/com/seibel/lod/util/ThreadMapUtil.java
+19 -19
View File
@@ -25,7 +25,7 @@ public class ThreadMapUtil
public static final ConcurrentMap<String, long[][]> threadBuilderArrayMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, long[][]> threadBuilderVerticalArrayMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, long[]> threadVerticalAddDataMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, byte[]> saveContainer = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, byte[][]> saveContainer = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, short[]> projectionArrayMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, short[]> heightAndDepthMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, long[]> singleDataToMergeMap = new ConcurrentHashMap<>();
@@ -117,6 +117,24 @@ public class ThreadMapUtil
return threadBuilderVerticalArrayMap.get(Thread.currentThread().getName())[detailLevel];
}
/** returns the array filled with 0's */
public static byte[] getSaveContainer(int detailLevel)
{
if (!saveContainer.containsKey(Thread.currentThread().getName()) || (saveContainer.get(Thread.currentThread().getName()) == null))
{
byte[][] array = new byte[DETAIL_OPTIONS][];
int size = 1;
for (int i = DETAIL_OPTIONS - 1; i >= 0; i--)
{
array[i] = new byte[2 + 8 * size * size * DetailDistanceUtil.getMaxVerticalData(i)];
size = size << 1;
}
saveContainer.put(Thread.currentThread().getName(), array);
}
//Arrays.fill(threadBuilderVerticalArrayMap.get(Thread.currentThread().getName())[detailLevel], 0);
return saveContainer.get(Thread.currentThread().getName())[detailLevel];
}
/** returns the array filled with 0's */
public static long[] getVerticalDataArray(int arrayLength)
@@ -145,24 +163,6 @@ public class ThreadMapUtil
}
/** returns the array filled with 0's */
public static byte[] getSaveContainer(int arrayLength)
{
if (!saveContainer.containsKey(Thread.currentThread().getName()) || (saveContainer.get(Thread.currentThread().getName()) == null))
{
saveContainer.put(Thread.currentThread().getName(), new byte[arrayLength]);
}
else if (saveContainer.get(Thread.currentThread().getName()).length != arrayLength)
{
saveContainer.replace(Thread.currentThread().getName(), new byte[arrayLength]);
}
else
{
Arrays.fill(saveContainer.get(Thread.currentThread().getName()), (byte) 0);
}
return saveContainer.get(Thread.currentThread().getName());
}
/** returns the array filled with 0's */
public static long[] getVerticalUpdateArray(int detailLevel)
{