From 94304eb055969e1262feba280fcceea96e3bcdd5 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 12:12:50 -0500 Subject: [PATCH] speed up initial loading when DB migration is necessary --- .../FullDataSourceProviderV2.java | 34 ++++++++++++- .../core/sql/repo/FullDataSourceV1Repo.java | 48 +++++++++++++++++++ ...20-sqlite-createFullDataSourceV2Tables.sql | 6 --- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index 7a8a7cf57..7844eb84f 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -317,10 +317,42 @@ public class FullDataSourceProviderV2 String dimensionName = this.level.getLevelWrapper().getDimensionType().getDimensionName(); LOGGER.info("Attempting to migrate data sources for: ["+dimensionName+"]-["+this.saveDir+"]..."); + + + //============================// + // delete unused data sources // + //============================// + + // this could be done all at once via SQL, + // but doing it in chunks prevents locking the database for long periods of time + long unusedCount = 0; + long totalUnusedCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); + if (totalUnusedCount != 0) + { + LOGGER.info("deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources..."); + + ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); + while (unusedDataPosList.size() != 0) + { + this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); + unusedCount += unusedDataPosList.size(); + unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); + + LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalUnusedCount + "]..."); + } + + LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources."); + } + + + + //=========// + // migrate // + //=========// + int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); LOGGER.info("Found ["+totalCount+"] data sources that need migration."); - ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index 521163695..30dfa21c2 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -22,6 +22,7 @@ package com.seibel.distanthorizons.core.sql.repo; import com.seibel.distanthorizons.api.enums.worldGeneration.EDhApiWorldGenerationStep; import com.seibel.distanthorizons.core.pos.DhSectionPos; import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV1DTO; +import com.seibel.distanthorizons.coreapi.util.StringUtil; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -234,4 +235,51 @@ public class FullDataSourceV1Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( + "select Count(*) as unusedCount from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource'"); + + if (resultMap != null) + { + // Number cast is necessary because the returned number can be an int or long + Number resultNumber = (Number) resultMap.get("unusedCount"); + long count = resultNumber.longValue(); + return count; + } + else + { + return 0; + } + } + + /** Returns single quote surrounded {@link DhSectionPos} serailzed values */ + public ArrayList getUnusedDataSourcePositionStringList(int deleteCount) + { + List> deletePosResultMapList = this.queryDictionary( + "select DhSectionPos from "+this.getTableName()+" where DataDetailLevel <> 0 and DataType <> 'CompleteFullDataSource' limit "+deleteCount); + + ArrayList deletePosList = new ArrayList<>(); + for (Map deletePosMap : deletePosResultMapList) + { + String posString = (String) deletePosMap.get("DhSectionPos"); + deletePosList.add("'"+posString+"'"); + } + + return deletePosList; + } + + /** Expects positions to already be surrounded in single quotes */ + public void deleteUnusedLegacyData(ArrayList deletePosList) + { + String sectionPosCsv = StringUtil.join(",", deletePosList); + this.queryDictionaryFirst("delete from " + this.getTableName() + " where DhSectionPos in (" + sectionPosCsv + ")"); + } + } diff --git a/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql index 007fa7b5b..150b788cf 100644 --- a/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql +++ b/core/src/main/resources/sqlScripts/0020-sqlite-createFullDataSourceV2Tables.sql @@ -7,12 +7,6 @@ ALTER TABLE Legacy_FullData_V1 ADD COLUMN MigrationFailed BIT NOT NULL DEFAULT 0 --batch-- --- we only want to convert the level 0 LOD data, the rest can be generated later -delete from Legacy_FullData_V1 -where DataType <> 'CompleteFullDataSource' or DataDetailLevel <> 0; - ---batch-- - CREATE TABLE FullData ( -- compound primary key DetailLevel TINYINT NOT NULL -- LOD detail level, not section detail level IE 0, 1, 2 not 6, 7, 8