1.12.2 working in dev

2026-03-11 09:20:32 +01:00
parent 1d0d67d215
commit c5adc3f72a
128 changed files with 5096 additions and 502 deletions
@@ -0,0 +1,37 @@
+
+CREATE TABLE DhFullData(
+	 DhSectionPos TEXT NOT NULL PRIMARY KEY
+	
+	-- meta data
+	,DataDetailLevel TINYINT NULL
+	,Checksum INT NULL
+	,DataVersion BIGINT NULL
+	,WorldGenStep NVARCHAR(32) NULL
+	,DataType NVARCHAR(48) NULL
+	,BinaryDataFormatVersion TINYINT NULL
+	 
+    ,Data BLOB NULL
+    
+    ,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
+    ,LastModifiedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
+);
+
+-- Note: each statement must be separated by the following batch comment line otherwise Java won't run anything after the first query
+--batch--
+
+CREATE TABLE DhRenderData(
+	 DhSectionPos TEXT NOT NULL PRIMARY KEY
+	 
+	 -- meta data 
+    ,DataDetailLevel TINYINT NULL
+    ,Checksum INT NULL
+    ,DataVersion BIGINT NULL
+    ,WorldGenStep NVARCHAR(32) NULL
+    ,DataType NVARCHAR(48) NULL
+    ,BinaryDataFormatVersion TINYINT NULL
+	 
+    ,Data BLOB NULL
+    
+    ,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
+    ,LastModifiedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC
+);
@@ -0,0 +1,33 @@
+
+ALTER TABLE DhFullData RENAME TO Legacy_FullData_V1;
+
+--batch--
+
+ALTER TABLE Legacy_FullData_V1 ADD COLUMN MigrationFailed BIT NOT NULL DEFAULT 0;
+
+--batch--
+
+CREATE TABLE FullData ( 
+    -- compound primary key
+     DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8
+    ,PosX INT NOT NULL
+    ,PosZ INT NOT NULL
+    
+    ,MinY INT NOT NULL
+    ,DataChecksum INT NOT NULL
+    
+    ,Data BLOB NULL
+    ,ColumnGenerationStep BLOB NULL
+    ,ColumnWorldCompressionMode BLOB NULL
+    ,Mapping BLOB NULL
+    
+    ,DataFormatVersion TINYINT NULL
+    ,CompressionMode TINYINT NULL
+    
+    ,ApplyToParent BIT NULL
+    
+    ,LastModifiedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    ,CreatedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    
+    ,PRIMARY KEY (DetailLevel, PosX, PosZ)
+);
@@ -0,0 +1,9 @@
+
+-- this PRAGMA will automatically commit, so we have to disable
+-- DH's automatic transactions, otherwise the connection will throw an error
+
+--No Transactions--
+
+-- James ran into some issues where Windows had trouble deleting the Journal file, 
+-- using TRUNCATE should fix that issue
+PRAGMA journal_mode = TRUNCATE;
@@ -0,0 +1,8 @@
+
+-- these PRAGMA's will automatically commit, so we have to disable
+-- DH's automatic transactions, otherwise the connection will throw an error
+
+--No Transactions--
+
+pragma journal_mode = WAL;
+pragma synchronous = NORMAL;
@@ -0,0 +1,4 @@
+
+-- The render cache was discovered to not speed up LOD loading,
+-- so to reduce DB file size it was removed.
+drop table DhRenderData;
@@ -0,0 +1,3 @@
+
+-- significantly speeds up parent update handling
+create index FullDataUpdatedIndex on FullData (ApplyToParent) where ApplyToParent = 1
@@ -0,0 +1,13 @@
+
+CREATE TABLE ChunkHash(
+    -- compound primary key
+     ChunkPosX INT NOT NULL
+    ,ChunkPosZ INT NOT NULL
+    
+    ,ChunkHash INT NOT NULL
+    
+    ,LastModifiedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    ,CreatedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    
+    ,PRIMARY KEY (ChunkPosX, ChunkPosZ)
+);
@@ -0,0 +1,16 @@
+
+CREATE TABLE BeaconBeam(
+    -- compound primary key
+     BlockPosX INT NOT NULL
+    ,BlockPosY INT NOT NULL
+    ,BlockPosZ INT NOT NULL
+    
+    ,ColorR INT NOT NULL
+    ,ColorG INT NOT NULL
+    ,ColorB INT NOT NULL
+    
+    ,LastModifiedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    ,CreatedUnixDateTime BIGINT NOT NULL -- in GMT 0
+    
+    ,PRIMARY KEY (BlockPosX, BlockPosY, BlockPosZ)
+);
@@ -0,0 +1,9 @@
+
+-- Applying to children is needed to fix a bug with N-sized generation.
+-- If we don't fill the whole tree with data, it's possible to render empty/incomplete LODs, which looks bad.
+alter table FullData add column ApplyToChildren BIT NULL;
+
+--batch--
+
+-- significantly speeds up update handling
+create index FullDataApplyToChildrenIndex on FullData (ApplyToChildren) where ApplyToChildren = 1;
@@ -0,0 +1,12 @@
+
+-- storing adjacent data (IE a single line of data on the +X/-X/+Z/-Z axis)
+-- allows for significantly reduced render loading times since we only have to
+-- handle part of the adjacent data source vs all of it
+
+alter table FullData add column NorthAdjData BLOB NULL;
+--batch--
+alter table FullData add column SouthAdjData BLOB NULL;
+--batch--
+alter table FullData add column EastAdjData BLOB NULL;
+--batch--
+alter table FullData add column WestAdjData BLOB NULL;
@@ -0,0 +1,13 @@
+
+-- This is done to fix a bug where a lot of unnecessary
+-- ID mapping data is saved, which significantly reduces
+-- loading/deserializing/decompression time
+
+
+-- delete all data above 0 (max detail)
+-- so it can be re-created
+delete from FullData where DetailLevel > 0;
+--batch--
+
+-- re-downsample all LOD data
+update FullData set ApplyToParent = 1;
@@ -0,0 +1,51 @@
+
+### All Sql scripts should be run exactly once per database and old scripts shouldn't be changed. Any necessary schema changes should be done by creating new scripts that modify the existing database.
+
+This system is roughly based on the DbUp library from .NET, for information about DbUp and it's general philosophy please refer to the following doc:
+https://dbup.readthedocs.io/en/latest/philosophy-behind-dbup/
+
+<br>
+
+### Adding New Scripts:
+New scripts must be added to the "scriptList.txt" file, otherwise they will not be run. <br>
+(If anyone has a good way to automatically pull all resource files ending in `.sql` instead, please let us know.)
+ 
+<br>
+
+### File Naming:
+- The first 3 numbers are major scripts.
+- The 4th number is for minor/related scripts or if a bug fix needs to be applied between scripts.
+- flavor of database the script is for (for now this is just sqlite)
+- description of the script
+
+<br>
+
+### Mutli-query Scripts: 
+When creating a script with multiple queries the queries must be separated with the SQL comment `--batch--` otherwise only the first query will be executed.
+
+Example:
+```roomsql
+CREATE TABLE TableOne(
+     DhSectionPos TEXT NOT NULL PRIMARY KEY 
+    ,Data BLOB NULL
+);
+
+--batch--
+
+CREATE TABLE TableTwo(
+     DhSectionPos TEXT NOT NULL PRIMARY KEY 
+    ,Data BLOB NULL
+);
+```
+
+### PRAGMA Auto Commits
+
+Certain queries will auto commit after running, specifically certain `PRAGMA` commands. In that case we have to disable DH's automatic transactions by putting `--No Transactions--` somewhere in the file. Otherwise, when the system attempts to commit, it will fail due to the PRAGMA having already committed itself.
+
+Due to how these commands work it's best to only have a single command in the file to prevent confusion and potential database corruption.
+
+```roomsql
+--No Transactions--
+
+PRAGMA journal_mode = TRUNCATE;
+```
@@ -0,0 +1,12 @@
+
+0010-sqlite-createInitialDataTables.sql
+0020-sqlite-createFullDataSourceV2Tables.sql
+0030-sqlite-changeTableJournaling.sql
+0031-sqlite-useSqliteWalJournaling.sql
+0040-sqlite-removeRenderCache.sql
+0050-sqlite-addApplyToParentIndex.sql
+0060-sqlite-createChunkHashTable.sql
+0070-sqlite-createBeaconBeamTable.sql
+0080-sqlite-addApplyToChildrenColumn.sql
+0090-sqlite-addAdjacentFullDataColumns.sql
+0100-sqlite-deleteLowDetailDataForRegen.sql