From 1ec06fa94e823da77a597241df44e38dede5c674 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Mon, 2 Oct 2023 19:54:43 -0500 Subject: [PATCH] Add database auto updating --- .../core/sql/AbstractDhRepo.java | 67 +++++++------- .../core/sql/DatabaseUpdater.java | 89 +++++++++++++++++++ .../0010-sqlite-createFullDataTable.sql | 8 ++ .../0010-sqlite-createInitialTables.sql | 9 -- .../0011-sqlite-createInitialTables.sql | 9 -- .../0020-sqlite-createRenderDataTable.sql | 8 ++ core/src/main/resources/sqlScripts/readme.md | 15 ++++ .../test/java/testItems/sql/TestDataRepo.java | 3 + .../src/test/java/tests/DhRepoSqliteTest.java | 25 ++++-- 9 files changed, 177 insertions(+), 56 deletions(-) create mode 100644 core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java create mode 100644 core/src/main/resources/sqlScripts/0010-sqlite-createFullDataTable.sql delete mode 100644 core/src/main/resources/sqlScripts/0010-sqlite-createInitialTables.sql delete mode 100644 core/src/main/resources/sqlScripts/0011-sqlite-createInitialTables.sql create mode 100644 core/src/main/resources/sqlScripts/0020-sqlite-createRenderDataTable.sql create mode 100644 core/src/main/resources/sqlScripts/readme.md diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java index 9a17f7d69..4e449443f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/AbstractDhRepo.java @@ -20,15 +20,11 @@ package com.seibel.distanthorizons.core.sql; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; -import com.seibel.distanthorizons.core.util.ResourceUtil; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.junit.Assert; -import java.io.IOException; -import java.net.URISyntaxException; import java.sql.*; -import java.util.*; /** * Handles interfacing with SQL databases. @@ -62,27 +58,7 @@ public abstract class AbstractDhRepo this.connection = DriverManager.getConnection(this.databaseType+":"+this.databaseLocation); - this.runFirstTimeSetup(); - } - private void runFirstTimeSetup() - { - // get the resource scripts - ArrayList sqlScripts; - try - { - sqlScripts = ResourceUtil.getFilesInFolder("sqlScripts", ".sql"); - } - catch (URISyntaxException | IOException e) - { - throw new RuntimeException(e); - } - - // attempt to run them - for (ResourceUtil.ResourceFile file : sqlScripts) - { - LOGGER.info("Running automatic SQL script: ["+file.name +"]"); - this.queryNoResult(file.content); - } + DatabaseUpdater.runAutoUpdateScripts(this); } @@ -133,14 +109,16 @@ public abstract class AbstractDhRepo public void queryNoResult(String sql) { this.query(sql, false); } public ResultSet query(String sql) { return this.query(sql, true); } + /** note: this can only handle 1 command at a time */ @Nullable - private ResultSet query(String sql, boolean returnResultSet) + private ResultSet query(String sql, boolean returnResultSet) throws RuntimeException { try { - Statement statement = this.connection .createStatement(); + Statement statement = this.connection.createStatement(); statement.setQueryTimeout(TIMEOUT_SECONDS); + // Note: this can only handle 1 command at a time boolean resultSetPresent = statement.execute(sql); if (resultSetPresent) { @@ -162,14 +140,29 @@ public abstract class AbstractDhRepo } catch(SQLException e) { - // if the error message is "out of memory", - // it probably means no database file is found - Assert.fail("Unexpected error for query ["+sql+"]: " + e.getMessage()); + // SQL exceptions generally only happen when something is wrong with + // the database or the query and should cause the system to blow up to notify the developer + throw new RuntimeException(e); } - - return null; } + public PreparedStatement createPreparedStatement(String sql) + { + try + { + PreparedStatement statement = this.connection.prepareStatement(sql); + statement.setQueryTimeout(TIMEOUT_SECONDS); + return statement; + } + catch(SQLException e) + { + // SQL exceptions generally only happen when something is wrong with + // the database or the query and should cause the system to blow up to notify the developer + throw new RuntimeException(e); + } + } + + public void close() { @@ -189,11 +182,21 @@ public abstract class AbstractDhRepo + //================// + // helper methods // + //================// + + public String createWherePrimaryKeyStatement(TDTO dto) { return this.createWherePrimaryKeyStatement(dto.getPrimaryKeyString()); } + public String createWherePrimaryKeyStatement(String primaryKeyValue) { return "WHERE "+this.getPrimaryKeyName()+" = '"+primaryKeyValue+"'"; } + + + //==================// // abstract methods // //==================// public abstract String getTableName(); + public abstract String getPrimaryKeyName(); @Nullable public abstract TDTO convertResultSetToDto(ResultSet resultSet) throws SQLException; diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java new file mode 100644 index 000000000..678a33346 --- /dev/null +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/DatabaseUpdater.java @@ -0,0 +1,89 @@ +/* + * This file is part of the Distant Horizons mod + * licensed under the GNU LGPL v3 License. + * + * Copyright (C) 2020-2023 James Seibel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +package com.seibel.distanthorizons.core.sql; + +import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; +import com.seibel.distanthorizons.core.util.ResourceUtil; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; + +public class DatabaseUpdater +{ + private static final Logger LOGGER = DhLoggerBuilder.getLogger(); + + public static final String SCHEMA_TABLE_NAME = "Schema"; + + + + /** Handles both initial setup and */ + public static void runAutoUpdateScripts(AbstractDhRepo repo) throws SQLException + { + // get the resource scripts + ArrayList sqlScripts; + try + { + sqlScripts = ResourceUtil.getFilesInFolder("sqlScripts", ".sql"); + } + catch (URISyntaxException | IOException e) + { + // shouldn't normally happen, but just incase + throw new RuntimeException(e); + } + + + // create the base update table if necessary + ResultSet schemaTableExistsResult = repo.query("SELECT COUNT(name) FROM sqlite_master WHERE type='table' AND name='Schema';"); + if (schemaTableExistsResult.next()) + { + boolean schemaTableMissing = schemaTableExistsResult.getInt(1) == 0; + if (schemaTableMissing) + { + // Note: if this table ever needs to be modified, that should be done via an auto update script to prevent issues with updating old databases + String createBaseSchemaTable = + "CREATE TABLE "+SCHEMA_TABLE_NAME+"( \n" + + " FileName TEXT NOT NULL PRIMARY KEY \n" + + " ,AppliedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP --in UTC 0 timezone \n" + + ");"; + repo.queryNoResult(createBaseSchemaTable); + } + } + + + // attempt to run any un-run update scripts + for (ResourceUtil.ResourceFile file : sqlScripts) + { + ResultSet scriptAlreadyRunResult = repo.query("SELECT EXISTS(SELECT 1 FROM Schema WHERE FileName='"+file.name+"');"); + if (!scriptAlreadyRunResult.next() || !scriptAlreadyRunResult.getBoolean(1)) + { + LOGGER.info("Running SQL update script: ["+file.name+"], for repo: ["+repo.databaseLocation+"]"); + repo.queryNoResult(file.content); + + // record the successfully run script + repo.queryNoResult("INSERT INTO Schema (FileName) VALUES('"+file.name+"');"); + } + } + } + +} diff --git a/core/src/main/resources/sqlScripts/0010-sqlite-createFullDataTable.sql b/core/src/main/resources/sqlScripts/0010-sqlite-createFullDataTable.sql new file mode 100644 index 000000000..2d359c228 --- /dev/null +++ b/core/src/main/resources/sqlScripts/0010-sqlite-createFullDataTable.sql @@ -0,0 +1,8 @@ + +CREATE TABLE DhFullData( + DhSectionPos TEXT NOT NULL PRIMARY KEY + + ,Data BLOB NULL + + ,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC +); diff --git a/core/src/main/resources/sqlScripts/0010-sqlite-createInitialTables.sql b/core/src/main/resources/sqlScripts/0010-sqlite-createInitialTables.sql deleted file mode 100644 index dac04a703..000000000 --- a/core/src/main/resources/sqlScripts/0010-sqlite-createInitialTables.sql +++ /dev/null @@ -1,9 +0,0 @@ - --- TODO should track ran scripts -CREATE TABLE IF NOT EXISTS DhFullData( - DhSectionPos TEXT NOT NULL PRIMARY KEY - - ,Data BLOB NULL - - --,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC -); diff --git a/core/src/main/resources/sqlScripts/0011-sqlite-createInitialTables.sql b/core/src/main/resources/sqlScripts/0011-sqlite-createInitialTables.sql deleted file mode 100644 index ab76adbe2..000000000 --- a/core/src/main/resources/sqlScripts/0011-sqlite-createInitialTables.sql +++ /dev/null @@ -1,9 +0,0 @@ - --- TODO should track ran scripts -CREATE TABLE IF NOT EXISTS DhRenderData( - DhSectionPos TEXT NOT NULL PRIMARY KEY - - ,Data BLOB NULL - - --,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC -); diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createRenderDataTable.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createRenderDataTable.sql new file mode 100644 index 000000000..41a9d9049 --- /dev/null +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createRenderDataTable.sql @@ -0,0 +1,8 @@ + +CREATE TABLE DhRenderData( + DhSectionPos TEXT NOT NULL PRIMARY KEY + + ,Data BLOB NULL + + ,CreatedDateTime DATETIME NOT NULL default CURRENT_TIMESTAMP -- in UTC +); diff --git a/core/src/main/resources/sqlScripts/readme.md b/core/src/main/resources/sqlScripts/readme.md new file mode 100644 index 000000000..ad4660402 --- /dev/null +++ b/core/src/main/resources/sqlScripts/readme.md @@ -0,0 +1,15 @@ + +All Sql scripts should be run exactly once per database and old scripts shouldn't be changed. Any necessary schema changes should be done by creating new scripts that modify the existing database. + +This system is roughly based on the DbUp library from .NET, for information about DbUp and it's general philosophy please refer to the following doc: +https://dbup.readthedocs.io/en/latest/philosophy-behind-dbup/ + + +File naming scheme: +- The first 3 numbers are major scripts. +- The 4th number is for minor/related scripts or if a bug fix needs to be applied between scripts. +- database type the script is for (for now this will just be sqlite) +- description of the script + + +Note: Currently the repo can only handle 1 statement per script so each file can only contain 1 statement, any subsequent statements are ignored. diff --git a/core/src/test/java/testItems/sql/TestDataRepo.java b/core/src/test/java/testItems/sql/TestDataRepo.java index 2b8741358..ee0f101c8 100644 --- a/core/src/test/java/testItems/sql/TestDataRepo.java +++ b/core/src/test/java/testItems/sql/TestDataRepo.java @@ -50,6 +50,9 @@ public class TestDataRepo extends AbstractDhRepo @Override public String getTableName() { return "Test"; } + @Override + public String getPrimaryKeyName() { return "Id"; } + @Override public TestDto convertResultSetToDto(ResultSet resultSet) throws SQLException diff --git a/core/src/test/java/tests/DhRepoSqliteTest.java b/core/src/test/java/tests/DhRepoSqliteTest.java index bf3ee22e1..9b3308068 100644 --- a/core/src/test/java/tests/DhRepoSqliteTest.java +++ b/core/src/test/java/tests/DhRepoSqliteTest.java @@ -19,12 +19,14 @@ package tests; +import com.seibel.distanthorizons.core.sql.DatabaseUpdater; import org.junit.Assert; import org.junit.Test; import testItems.sql.TestDataRepo; import testItems.sql.TestDto; import java.io.File; +import java.sql.ResultSet; import java.sql.SQLException; /** @@ -57,6 +59,23 @@ public class DhRepoSqliteTest Assert.assertTrue("dbFile not created", dbFile.exists()); + + //==========================// + // Auto update script tests // + //==========================// + + ResultSet autoUpdateTablePresentResult = testDataRepo.query("SELECT name FROM sqlite_master WHERE type='table' AND name='"+DatabaseUpdater.SCHEMA_TABLE_NAME+"';"); + if (!autoUpdateTablePresentResult.next() || autoUpdateTablePresentResult.getString(1) == null) + { + Assert.fail("Auto DB update table missing."); + } + + + + //===========// + // DTO tests // + //===========// + // insert TestDto insertDto = new TestDto(0, "a"); testDataRepo.save(insertDto); @@ -97,12 +116,6 @@ public class DhRepoSqliteTest { testDataRepo.close(); } - - dbFile = new File(dbFileName); - if (dbFile.exists()) - { - Assert.assertTrue("unable to delete test DB File.", dbFile.delete()); - } } }