wisplite/distant-horizons-core-sharded
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move data source meta data from the compressed binary into DB columns

Browse Source
This commit is contained in:
James Seibel
2023-10-07 16:10:16 -05:00
parent 444090ed39
commit 526f483bfb
6 changed files with 122 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/src/main/java/com/seibel/distanthorizons/core/file/renderfile/RenderDataMetaFile.java
+2 -20
View File
@@ -259,25 +259,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
private InputStream getInputStream() throws IOException
{
MetaDataDto dto = this.renderDataSourceProvider.getRepo().getByPrimaryKey(this.pos.serialize());
ByteArrayInputStream inputStream = new ByteArrayInputStream(dto.dataArray);
// skip the meta-data bytes
int bytesToSkip = AbstractMetaDataContainerFile.METADATA_SIZE_IN_BYTES;
while (bytesToSkip > 0)
{
long skippedByteCount = inputStream.skip(bytesToSkip);
if (skippedByteCount == 0)
{
throw new IOException("Invalid file: Failed to skip metadata.");
}
bytesToSkip -= skippedByteCount;
}
if (bytesToSkip != 0)
{
throw new IOException("File IO Error: Failed to skip metadata.");
}
return inputStream;
return new ByteArrayInputStream(dto.dataArray);
}
@@ -419,7 +401,7 @@ public class RenderDataMetaFile extends AbstractMetaDataContainerFile implements
super.writeData((dhDataOutputStream) -> renderSource.writeData(dhDataOutputStream), byteArrayOutputStream);
this.doesDtoExist = true;
MetaDataDto dto = new MetaDataDto(this.pos, byteArrayOutputStream.toByteArray());
MetaDataDto dto = new MetaDataDto(this.baseMetaData, byteArrayOutputStream.toByteArray());
this.renderDataSourceProvider.getRepo().save(dto);
}
catch (IOException e)
core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractMetaDataRepo.java
+53 -10
View File
@@ -19,6 +19,8 @@
package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.coreapi.util.StringUtil;
@@ -44,9 +46,24 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
String posString = (String) objectMap.get("DhSectionPos");
DhSectionPos pos = DhSectionPos.deserialize(posString);
// meta data
int checksum = (Integer) objectMap.get("Checksum");
long dataVersion = (Long) objectMap.get("DataVersion");
byte dataDetailLevel = (Byte) objectMap.get("DataDetailLevel");
String worldGenStepString = (String) objectMap.get("WorldGenStep");
EDhApiWorldGenerationStep worldGenStep = EDhApiWorldGenerationStep.fromName(worldGenStepString);
String dataType = (String) objectMap.get("DataType");
byte binaryDataFormatVersion = (Byte) objectMap.get("BinaryDataFormatVersion");
BaseMetaData baseMetaData = new BaseMetaData(pos,
checksum, dataDetailLevel, worldGenStep,
dataType, binaryDataFormatVersion, dataVersion);
// binary data
byte[] dataByteArray = (byte[]) objectMap.get("Data");
MetaDataDto metaFile = new MetaDataDto(pos, dataByteArray);
MetaDataDto metaFile = new MetaDataDto(baseMetaData, dataByteArray);
return metaFile;
}
@@ -56,22 +73,48 @@ public abstract class AbstractMetaDataRepo extends AbstractDhRepo<MetaDataDto>
@Override
public String createInsertSql(MetaDataDto dto)
{
String pos = dto.pos.serialize();
String dataString = createDataHexString(dto);
String posString = dto.baseMetaData.pos.serialize();
String dataHexString = createDataHexString(dto);
return
"INSERT INTO "+this.getTableName()+" (DhSectionPos, Data) " +
"VALUES('"+pos+"',"+dataString+");";
"INSERT INTO "+this.getTableName() + "\n" +
" (DhSectionPos, \n" +
"Checksum, DataVersion, DataDetailLevel, WorldGenStep, DataType, BinaryDataFormatVersion, \n" +
"Data) \n" +
" VALUES( \n" +
" '"+posString+"' \n" +
" ,"+dto.baseMetaData.checksum + "\n" +
" ,"+dto.baseMetaData.dataVersion + "\n" +
" ,"+dto.baseMetaData.dataDetailLevel + "\n" +
" ,'"+dto.baseMetaData.worldGenStep.name + "' \n" +
" ,'"+dto.baseMetaData.dataType + "' \n" +
" ,"+dto.baseMetaData.binaryDataFormatVersion + "\n" +
" ,"+dataHexString + "\n" +
// created/lastModified are automatically set by Sqlite
");";
}
@Override
public String createUpdateSql(MetaDataDto dto)
{
String pos = dto.pos.serialize();
String dataString = createDataHexString(dto);
String posString = dto.baseMetaData.pos.serialize();
String dataHexString = createDataHexString(dto);
return
"UPDATE "+this.getTableName()+" " +
"SET Data = "+dataString +
"WHERE DhSectionPos = '"+pos+"'";
"UPDATE "+this.getTableName()+" \n" +
"SET \n" +
" Checksum = "+dto.baseMetaData.checksum + "\n" +
" ,DataVersion = "+dto.baseMetaData.dataVersion + "\n" +
" ,DataDetailLevel = "+dto.baseMetaData.dataDetailLevel + "\n" +
" ,WorldGenStep = '"+dto.baseMetaData.worldGenStep.name + "' \n" +
" ,DataType = '"+dto.baseMetaData.dataType + "' \n" +
" ,BinaryDataFormatVersion = "+dto.baseMetaData.binaryDataFormatVersion + "\n" +
" ,Data = "+dataHexString + "\n" +
" ,LastModifiedDateTime = CURRENT_TIMESTAMP \n" +
"WHERE DhSectionPos = '"+posString+"'";
}
core/src/main/java/com/seibel/distanthorizons/core/sql/IBaseDTO.java
+4
View File
@@ -19,6 +19,10 @@
package com.seibel.distanthorizons.core.sql;
/**
* DTO = DataTable Object <br>
* Any object that's stored in the database should extend this object.
*/
public interface IBaseDTO
{
String getPrimaryKeyString();
core/src/main/java/com/seibel/distanthorizons/core/sql/MetaDataDto.java
+9 -6
View File
@@ -19,22 +19,25 @@
package com.seibel.distanthorizons.core.sql;
import com.seibel.distanthorizons.core.pos.DhSectionPos;
import com.seibel.distanthorizons.core.dataObjects.fullData.sources.interfaces.IFullDataSource;
import com.seibel.distanthorizons.core.dataObjects.render.ColumnRenderSource;
import com.seibel.distanthorizons.core.file.metaData.BaseMetaData;
/** handles storing both {@link IFullDataSource}'s and {@link ColumnRenderSource}'s in the database. */
public class MetaDataDto implements IBaseDTO
{
public DhSectionPos pos;
public byte[] dataArray;
public final BaseMetaData baseMetaData;
public final byte[] dataArray;
public MetaDataDto(DhSectionPos pos, byte[] dataArray)
public MetaDataDto(BaseMetaData baseMetaData, byte[] dataArray)
{
this.pos = pos;
this.baseMetaData = baseMetaData;
this.dataArray = dataArray;
}
@Override
public String getPrimaryKeyString() { return this.pos.serialize(); }
public String getPrimaryKeyString() { return this.baseMetaData.pos.serialize(); }
}
core/src/main/resources/sqlScripts/0010-sqlite-createInitialDataTables.sql
+18
View File
@@ -1,10 +1,19 @@
CREATE TABLE DhFullData(
DhSectionPos TEXT NOT NULL PRIMARY KEY
-- meta data
,DataDetailLevel TINYINT NULL
,Checksum INT NULL
,DataVersion BIGINT NULL
,WorldGenStep NVARCHAR(32) NULL
,DataType NVARCHAR(48) NULL
,BinaryDataFormatVersion TINYINT NULL
,Data BLOB NULL
,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
,LastModifiedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
);
-- Note: each statement must be separated by the following batch comment line otherwise Java won't run anything after the first query
@@ -13,7 +22,16 @@ CREATE TABLE DhFullData(
CREATE TABLE DhRenderData(
DhSectionPos TEXT NOT NULL PRIMARY KEY
-- meta data
,DataDetailLevel TINYINT NULL
,Checksum INT NULL
,DataVersion BIGINT NULL
,WorldGenStep NVARCHAR(32) NULL
,DataType NVARCHAR(48) NULL
,BinaryDataFormatVersion TINYINT NULL
,Data BLOB NULL
,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
,LastModifiedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
);