wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save/Load: Optimize Memory usage on save/load

Browse Source 
It now directly read/write to file stream instead of via a temp byte[]
buffer.
This commit is contained in:
tom lee
2021-12-29 18:17:44 +08:00
parent 1034360b88
commit fcab0d3b20
4 changed files with 173 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/java/com/seibel/lod/core/handlers/LodDimensionFileHandler.java
+30 -91
View File
@@ -192,8 +192,8 @@ public class LodDimensionFileHandler
+ " version found: " + fileVersion
+ ", version requested: " + LOD_SAVE_FILE_VERSION
+ ". File was been deleted.");
break;
// This should not break, but be continue to see whether other versions can be loaded or updated
continue;
}
else if (fileVersion > LOD_SAVE_FILE_VERSION)
{
@@ -205,26 +205,21 @@ public class LodDimensionFileHandler
+ " version found: " + fileVersion
+ ", version requested: " + LOD_SAVE_FILE_VERSION
+ " this region will not be written to in order to protect the newer file.");
break;
// This should not break, but be continue to see whether other versions can be loaded or updated
continue;
}
else if (fileVersion < LOD_SAVE_FILE_VERSION)
{
//this is old, but readable version
byte[] data = ThreadMapUtil.getSaveContainer(tempDetailLevel);
inputStream.read(data);
// this is old, but readable version
// read and add the data to our region
region.addLevelContainer(new VerticalLevelContainer(new DataInputStream(inputStream), fileVersion));
inputStream.close();
// add the data to our region
region.addLevelContainer(new VerticalLevelContainer(data, fileVersion));
} else
{
// this file is a readable version,
// read the file
byte[] data = ThreadMapUtil.getSaveContainer(tempDetailLevel);
inputStream.read(data);
// read and add the data to our region
region.addLevelContainer(new VerticalLevelContainer(new DataInputStream(inputStream), LOD_SAVE_FILE_VERSION));
inputStream.close();
// add the data to our region
region.addLevelContainer(new VerticalLevelContainer(data, LOD_SAVE_FILE_VERSION));
}
}
catch (IOException ioEx)
@@ -305,10 +300,11 @@ public class LodDimensionFileHandler
}
File oldFile = new File(fileName);
//ClientProxy.LOGGER.info("saving region [" + region.regionPosX + ", " + region.regionPosZ + "] to file.");
byte[] temp = region.getLevel(detailLevel).toDataString();
//byte[] temp = region.getLevel(detailLevel).toDataString();
try
{
boolean isFileFullyGened = false;
// make sure the file and folder exists
if (!oldFile.exists())
{
@@ -325,12 +321,11 @@ public class LodDimensionFileHandler
// (to make sure we don't overwrite a newer
// version file if it exists)
int fileVersion = LOD_SAVE_FILE_VERSION;
int isFull = 0;
try (XZCompressorInputStream inputStream = new XZCompressorInputStream(new FileInputStream(oldFile)))
{
fileVersion = inputStream.read();
inputStream.skip(1);
isFull = inputStream.read() & 0b10000000;
isFileFullyGened = (inputStream.read() & 0b10000000) != 0;
inputStream.close();
}
catch (IOException ex)
@@ -344,15 +339,7 @@ public class LodDimensionFileHandler
// the file we are reading is a newer version,
// don't write anything, we don't want to accidentally
// delete anything the user may want.
return;
}
if ((temp[1] & 0b10000000) != 0b10000000 && isFull == 0b10000000)
{
// existing file is complete while new one is only partially generate
// this can happen is for some reason loading failed
// this doesn't fix the bug, but at least protects old data
ClientApi.LOGGER.error("LOD file write error. Attempted to overwrite complete region with incomplete one [" + fileName + "]");
return;
continue;
}
// if we got this far then we are good
// to overwrite the old file
@@ -365,9 +352,23 @@ public class LodDimensionFileHandler
outputStream.write(LOD_SAVE_FILE_VERSION);
// add each LodChunk to the file
outputStream.write(temp);
boolean isLocalFullyGened = region.getLevel(detailLevel).writeData(new DataOutputStream(outputStream));
outputStream.close();
if (!isLocalFullyGened && isFileFullyGened)
{
// existing file is complete while new one is only partially generate
// this can happen is for some reason loading failed
// this doesn't fix the bug, but at least protects old data
ClientApi.LOGGER.error("LOD file write error. Attempted to overwrite complete region with incomplete one [" + fileName + "]");
try {
newFile.delete();
} catch (SecurityException e) {
// Failed to delete temp file... just continue.
}
continue;
}
// overwrite the old file with the new one
Files.move(newFile.toPath(), oldFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
}
@@ -384,73 +385,10 @@ public class LodDimensionFileHandler
}
}
public void saveRegionFile (byte[] regionFile, RegionPos regionPos, DistanceGenerationMode generationMode, byte detailLevel, VerticalQuality verticalQuality)
{
int regionX = regionPos.x;
int regionZ = regionPos.z;
String fileName = getFileNameAndPathForRegion(regionX, regionZ, generationMode, detailLevel, verticalQuality);
if (fileName != null)
{
File oldFile = new File(fileName);
File newFile = new File(fileName + TMP_FILE_EXTENSION);
try (OutputStream os = new FileOutputStream(newFile))
{
os.write(regionFile);
Files.move(newFile.toPath(), oldFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
os.close();
}
catch (IOException ioEx)
{
ClientApi.LOGGER.error("LOD file write error. Unable to write to [" + fileName + "] error [" + ioEx.getMessage() + "]: ");
ioEx.printStackTrace();
}
}
}
public byte[] getRegionFile (RegionPos regionPos, DistanceGenerationMode generationMode, byte detailLevel, VerticalQuality verticalQuality)
{
int regionX = regionPos.x;
int regionZ = regionPos.z;
String fileName = getFileNameAndPathForRegion(regionX, regionZ, generationMode, detailLevel, verticalQuality);
if (fileName != null)
{
File file = new File(fileName);
try (InputStream is = new FileInputStream(file))
{
byte[] data = ThreadMapUtil.getSaveContainer(detailLevel);
is.read(data);
is.close();
return Arrays.copyOf(data, (int) file.length());
}
catch (IOException ioEx)
{
ClientApi.LOGGER.error("LOD file read error. Unable to read to [" + fileName + "] error [" + ioEx.getMessage() + "]: ");
ioEx.printStackTrace();
}
}
return new byte[0];
}
//================//
// helper methods //
//================//
public int getHashFromFile(RegionPos regionPos, DistanceGenerationMode generationMode, byte detailLevel, VerticalQuality verticalQuality)
{
int regionX = regionPos.x;
int regionZ = regionPos.z;
String fileName = getFileNameAndPathForRegion(regionX, regionZ, generationMode, detailLevel, verticalQuality);
if (fileName == null)
return 0;
File file = new File(fileName);
return file.hashCode();
}
/**
* Return the name of the file that should contain the
* region at the given x and z. <br>
@@ -476,7 +414,8 @@ public class LodDimensionFileHandler
}
catch (IOException | SecurityException e)
{
ClientApi.LOGGER.warn("Unable to get the filename for the region [" + regionX + ", " + regionZ + "], error: [" + e.getMessage() + "], stacktrace: ");
ClientApi.LOGGER.warn("Unable to get the filename for the region [" + regionX + ", " + regionZ + "], error: [" + e.getMessage() + "].\n"
+ "One possible cause is that the process failed to read the current path location due to security config. Stacktrace: ");
e.printStackTrace();
return null;
}
src/main/java/com/seibel/lod/core/objects/lod/LevelContainer.java
+11 -5
View File
@@ -19,6 +19,10 @@
package com.seibel.lod.core.objects.lod;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
/**
* A level container is a quad tree level
*/
@@ -99,13 +103,15 @@ public interface LevelContainer
* @param posZ z position in the detail level to update
*/
void updateData(LevelContainer lowerLevelContainer, int posX, int posZ);
/**
* This will give the data to save in the file
* @return data as a String
* This will write the raw data without metadata to the output stream
* @return isAllGenerated whether the data is all generated
* @throws IOException
*/
byte[] toDataString();
default boolean writeData(DataOutputStream output) throws IOException {
throw new UnsupportedOperationException();
}
/**
* This will give the data to save in the file
src/main/java/com/seibel/lod/core/objects/lod/VerticalLevelContainer.java
+132 -26
View File
@@ -19,6 +19,12 @@
package com.seibel.lod.core.objects.lod;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import com.seibel.lod.core.dataFormat.*;
import com.seibel.lod.core.enums.config.DistanceGenerationMode;
import com.seibel.lod.core.util.*;
@@ -165,6 +171,122 @@ public class VerticalLevelContainer implements LevelContainer
return DataPointUtil.doesItExist(getSingleData(posX, posZ));
}
private long[] readDataVersion6(DataInputStream inputData, int tempMaxVerticalData) throws IOException {
int x = size * size * tempMaxVerticalData;
long[] tempDataContainer = new long[x];
for (int i = 0; i < tempDataContainer.length; i++)
{
long newData = Long.reverseBytes(inputData.readLong());
newData = DataPointUtil.createDataPoint(
DataPointUtil.getAlpha(newData),
DataPointUtil.getRed(newData),
DataPointUtil.getGreen(newData),
DataPointUtil.getBlue(newData),
DataPointUtil.getHeight(newData) - minHeight,
DataPointUtil.getDepth(newData) - minHeight,
DataPointUtil.getLightSky(newData),
DataPointUtil.getLightBlock(newData),
DataPointUtil.getGenerationMode(newData),
DataPointUtil.getFlag(newData));
tempDataContainer[i] = newData;
}
return tempDataContainer;
}
private long[] readDataVersion7(DataInputStream inputData, int tempMaxVerticalData) throws IOException {
int x = size * size * tempMaxVerticalData;
long[] tempDataContainer = new long[x];
for (int i = 0; i < tempDataContainer.length; i++)
{
long newData = Long.reverseBytes(inputData.readLong());
newData = DataPointUtil.createDataPoint(
DataPointUtil.getAlpha(newData),
DataPointUtil.getRed(newData),
DataPointUtil.getGreen(newData),
DataPointUtil.getBlue(newData),
DataPointUtil.getHeight(newData) - 64 - minHeight,
DataPointUtil.getDepth(newData) - 64 - minHeight,
DataPointUtil.getLightSky(newData),
DataPointUtil.getLightBlock(newData),
DataPointUtil.getGenerationMode(newData),
DataPointUtil.getFlag(newData));
tempDataContainer[i] = newData;
}
return tempDataContainer;
}
private long[] readDataVersion8(DataInputStream inputData, int tempMaxVerticalData) throws IOException {
int x = size * size * tempMaxVerticalData;
long[] tempDataContainer = new long[x];
short tempMinHeight = Short.reverseBytes(inputData.readShort());
if (tempMinHeight != minHeight) {
for (int i = 0; i < tempDataContainer.length; i++) {
long newData = Long.reverseBytes(inputData.readLong());
newData = DataPointUtil.createDataPoint(
DataPointUtil.getAlpha(newData),
DataPointUtil.getRed(newData),
DataPointUtil.getGreen(newData),
DataPointUtil.getBlue(newData),
DataPointUtil.getHeight(newData) + tempMinHeight - minHeight,
DataPointUtil.getDepth(newData) + tempMinHeight - minHeight,
DataPointUtil.getLightSky(newData),
DataPointUtil.getLightBlock(newData),
DataPointUtil.getGenerationMode(newData),
DataPointUtil.getFlag(newData));
tempDataContainer[i] = newData;
}
} else {
for (int i = 0; i < tempDataContainer.length; i++) {
tempDataContainer[i] = Long.reverseBytes(inputData.readLong());
}
}
return tempDataContainer;
}
public VerticalLevelContainer(DataInputStream inputData, int version) throws IOException {
minHeight = SingletonHandler.get(IMinecraftWrapper.class).getWrappedClientWorld().getMinHeight();
detailLevel = inputData.readByte();
size = 1 << (LodUtil.REGION_DETAIL_LEVEL - detailLevel);
int fileMaxVerticalData = inputData.readByte() & 0b01111111;
long[] tempDataContainer = null;
switch (version) {
case 6:
tempDataContainer = readDataVersion6(inputData, fileMaxVerticalData);
break;
case 7:
tempDataContainer = readDataVersion7(inputData, fileMaxVerticalData);
break;
case 8:
tempDataContainer = readDataVersion8(inputData, fileMaxVerticalData);
break;
default:
assert false;
}
int targetMaxVerticalData = DetailDistanceUtil.getMaxVerticalData(detailLevel);
if (fileMaxVerticalData > targetMaxVerticalData)
{
long[] dataToMerge = new long[fileMaxVerticalData];
long[] tempDataContainer2 = new long[size * size * targetMaxVerticalData];
for (int i = 0; i < size * size; i++)
{
System.arraycopy(tempDataContainer, i * fileMaxVerticalData, dataToMerge, 0, fileMaxVerticalData);
dataToMerge = DataPointUtil.mergeMultiData(dataToMerge, fileMaxVerticalData, targetMaxVerticalData);
System.arraycopy(dataToMerge, 0, tempDataContainer2, i * targetMaxVerticalData, targetMaxVerticalData);
}
verticalSize = targetMaxVerticalData;
this.dataContainer = tempDataContainer2;
}
else
{
verticalSize = fileMaxVerticalData;
this.dataContainer = tempDataContainer;
}
}
// Deprecated. Please use the DataInputStream version.
@Deprecated
public VerticalLevelContainer(byte[] inputData, int version)
{
minHeight = SingletonHandler.get(IMinecraftWrapper.class).getWrappedClientWorld().getMinHeight();
@@ -1062,40 +1184,24 @@ public class VerticalLevelContainer implements LevelContainer
}
@Override
public byte[] toDataString()
{
int index = 0;
int x = size * size;
int tempIndex;
long current;
public boolean writeData(DataOutputStream output) throws IOException {
output.writeByte(detailLevel);
output.writeByte((byte) verticalSize);
output.writeByte((byte) (minHeight & 0xFF));
output.writeByte((byte) ((minHeight >> 8) & 0xFF));
boolean allGenerated = true;
byte[] tempData = ThreadMapUtil.getSaveContainer(detailLevel);
tempData[index] = detailLevel;
index++;
tempData[index] = (byte) verticalSize;
index++;
tempData[index] = (byte) (minHeight & 0xFF);
index++;
tempData[index] = (byte) ((minHeight >> 8) & 0xFF);
index++;
int j;
int x = size * size;
for (int i = 0; i < x; i++)
{
for (j = 0; j < verticalSize; j++)
for (int j = 0; j < verticalSize; j++)
{
current = dataContainer[i * verticalSize + j];
for (tempIndex = 0; tempIndex < 8; tempIndex++)
tempData[index + tempIndex] = (byte) (current >>> (8 * tempIndex));
index += 8;
long current = dataContainer[i * verticalSize + j];
output.writeLong(Long.reverseBytes(current));
}
if (!DataPointUtil.doesItExist(dataContainer[i]))
allGenerated = false;
}
if (allGenerated)
tempData[1] |= 0b10000000;
return tempData;
return allGenerated;
}
@Override
src/main/java/com/seibel/lod/core/util/ThreadMapUtil.java
-21
View File
@@ -42,7 +42,6 @@ public class ThreadMapUtil
public static final ConcurrentMap<String, long[][]> threadBuilderArrayMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, long[][]> threadBuilderVerticalArrayMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, long[]> threadVerticalAddDataMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, byte[][]> saveContainer = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, short[]> projectionArrayMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, short[]> heightAndDepthMap = new ConcurrentHashMap<>();
public static final ConcurrentMap<String, long[]> singleDataToMergeMap = new ConcurrentHashMap<>();
@@ -136,25 +135,6 @@ public class ThreadMapUtil
return threadBuilderVerticalArrayMap.get(Thread.currentThread().getName())[detailLevel];
}
/** returns the array NOT cleared every time */
public static byte[] getSaveContainer(int detailLevel)
{
if (!saveContainer.containsKey(Thread.currentThread().getName()) || (saveContainer.get(Thread.currentThread().getName()) == null))
{
byte[][] array = new byte[LodUtil.DETAIL_OPTIONS][];
int size = 1;
for (int i = LodUtil.DETAIL_OPTIONS - 1; i >= 0; i--)
{
array[i] = new byte[4 + 8 * size * size * DetailDistanceUtil.getMaxVerticalData(i)];
size = size << 1;
}
saveContainer.put(Thread.currentThread().getName(), array);
}
//Arrays.fill(threadBuilderVerticalArrayMap.get(Thread.currentThread().getName())[detailLevel], 0);
return saveContainer.get(Thread.currentThread().getName())[detailLevel];
}
/** returns the array filled with 0's */
public static long[] getVerticalDataArray(int arrayLength)
{
@@ -209,7 +189,6 @@ public class ThreadMapUtil
threadBuilderArrayMap.clear();
threadBuilderVerticalArrayMap.clear();
threadVerticalAddDataMap.clear();
saveContainer.clear();
projectionArrayMap.clear();
heightAndDepthMap.clear();
singleDataToMergeMap.clear();