From 63371e8111a7426d1c78f057c285e0f802d28349 Mon Sep 17 00:00:00 2001 From: James Seibel Date: Sat, 13 Apr 2024 16:58:52 -0500 Subject: [PATCH] improve migration deletion logic and add F3 logging --- .../FullDataSourceProviderV1.java | 2 +- .../FullDataSourceProviderV2.java | 77 ++++++++++++++----- .../core/level/ClientLevelModule.java | 8 +- .../core/sql/repo/FullDataSourceV1Repo.java | 5 +- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java index 9fb3bd9fc..fde42a6a7 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV1.java @@ -142,7 +142,7 @@ public class FullDataSourceProviderV1 // migration // //===========// - public int getDataSourceMigrationCount() { return this.repo.getMigrationCount(); } + public long getDataSourceMigrationCount() { return this.repo.getMigrationCount(); } public ArrayList getDataSourcesToMigrate(int limit) { diff --git a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java index da63e4857..d1658064b 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/file/fullDatafile/FullDataSourceProviderV2.java @@ -79,9 +79,6 @@ public class FullDataSourceProviderV2 private static final int MIGRATION_MAX_UPDATE_TIMEOUT_IN_MS = 5 * 60 * 1_000; - private static boolean migrationMessageShown = false; - - protected final ThreadPoolExecutor migrationThreadPool; /** * Interrupting the migration thread pool doesn't work well and may corrupt the database @@ -90,6 +87,9 @@ public class FullDataSourceProviderV2 protected final AtomicBoolean migrationThreadRunning = new AtomicBoolean(true); protected final FullDataSourceProviderV1 legacyFileHandler; + protected long legacyDeletionCount = -1; + protected long migrationCount = -1; + /** * Tracks which positions are currently being updated * to prevent duplicate concurrent updates. @@ -120,8 +120,6 @@ public class FullDataSourceProviderV2 String dimensionName = level.getLevelWrapper().getDimensionType().getDimensionName(); // start migrating any legacy data sources present in the background - int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); - LOGGER.info("Found ["+totalCount+"] data sources that need migration."); this.migrationThreadPool = ThreadUtil.makeRateLimitedThreadPool(1, MIGRATION_THREAD_NAME_PREFIX +"["+dimensionName+"]", Config.Client.Advanced.MultiThreading.runTimeRatioForUpdatePropagatorThreads.get(), Thread.MIN_PRIORITY, (Semaphore)null); this.migrationThreadPool.execute(() -> this.convertLegacyDataSources()); @@ -333,43 +331,70 @@ public class FullDataSourceProviderV2 // this could be done all at once via SQL, // but doing it in chunks prevents locking the database for long periods of time long unusedCount = 0; - long totalUnusedCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); - if (totalUnusedCount != 0) + long totalDeleteCount = this.legacyFileHandler.repo.getUnusedDataSourceCount(); + if (totalDeleteCount != 0) { - LOGGER.info("deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources..."); + // this should only be shown once per session but should be shown during + // either when the deletion or migration phases start + ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + + + LOGGER.info("deleting [" + dimensionName + "] - ["+totalDeleteCount+"] unused data sources..."); + this.legacyDeletionCount = totalDeleteCount; - ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); + ArrayList unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50); while (unusedDataPosList.size() != 0) { - this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); unusedCount += unusedDataPosList.size(); - unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(100); - - LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalUnusedCount + "]..."); + this.legacyDeletionCount -= unusedDataPosList.size(); + + + long startTime = System.currentTimeMillis(); + + // delete batch and get next batch + this.legacyFileHandler.repo.deleteUnusedLegacyData(unusedDataPosList); + unusedDataPosList = this.legacyFileHandler.repo.getUnusedDataSourcePositionStringList(50); + + long endStart = System.currentTimeMillis(); + long deleteTime = endStart - startTime; + LOGGER.info("Deleting [" + dimensionName + "] - [" + unusedCount + "/" + totalDeleteCount + "] in ["+deleteTime+"]ms ..."); + + + // a slight delay is added to prevent accidentally locking the database when deleting a lot of rows + // (that shouldn't be the case since we're using WAL journaling, but just in case) + try + { + // use the delete time so we don't make powerful computers wait super long + // and weak computers wait no time at all + Thread.sleep(deleteTime / 2); + } + catch (InterruptedException ignore){} } - LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalUnusedCount+"] unused data sources."); + LOGGER.info("Done deleting [" + dimensionName + "] - ["+totalDeleteCount+"] unused data sources."); } - //=========// - // migrate // - //=========// + //===========// + // migration // + //===========// - int totalCount = this.legacyFileHandler.getDataSourceMigrationCount(); - LOGGER.info("Found ["+totalCount+"] data sources that need migration."); + long totalMigrationCount = this.legacyFileHandler.getDataSourceMigrationCount(); + this.migrationCount = totalMigrationCount; + LOGGER.info("Found ["+totalMigrationCount+"] data sources that need migration."); ArrayList legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); if (!legacyDataSourceList.isEmpty()) { ClientApi.INSTANCE.showMigrationMessageOnNextFrame(); + // keep going until every data source has been migrated int progressCount = 0; while (!legacyDataSourceList.isEmpty() && this.migrationThreadRunning.get()) { - LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalCount + "]..."); + LOGGER.info("Migrating [" + dimensionName + "] - [" + progressCount + "/" + totalMigrationCount + "]..."); ArrayList> updateFutureList = new ArrayList<>(); for (int i = 0; i < legacyDataSourceList.size() && this.migrationThreadRunning.get(); i++) @@ -391,6 +416,12 @@ public class FullDataSourceProviderV2 { // after the update finishes the legacy data source can be safely deleted this.legacyFileHandler.repo.deleteWithKey(legacyDataSource.getPos()); + + try + { + newDataSource.close(); + } + catch (Exception ignore){ } }); } catch (Exception e) @@ -418,13 +449,16 @@ public class FullDataSourceProviderV2 } legacyDataSourceList = this.legacyFileHandler.getDataSourcesToMigrate(MIGRATION_BATCH_COUNT); + progressCount += legacyDataSourceList.size(); + this.migrationCount -= legacyDataSourceList.size(); } if (this.migrationThreadRunning.get()) { LOGGER.info("migration complete for: ["+dimensionName+"]-["+this.saveDir+"]."); + this.migrationCount = 0; } else { @@ -439,7 +473,8 @@ public class FullDataSourceProviderV2 this.migrationThreadRunning.set(false); } - public int getMigrationCount() { return this.legacyFileHandler.getDataSourceMigrationCount(); } + public long getLegacyDeletionCount() { return this.legacyDeletionCount; } + public long getTotalMigrationCount() { return this.migrationCount; } diff --git a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java index 90accbcde..b10692021 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/level/ClientLevelModule.java @@ -26,7 +26,6 @@ import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSour import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector; import com.seibel.distanthorizons.core.file.AbstractNewDataSourceHandler; import com.seibel.distanthorizons.core.file.fullDatafile.FullDataSourceProviderV2; -import com.seibel.distanthorizons.core.file.structure.AbstractSaveStructure; import com.seibel.distanthorizons.core.logging.DhLoggerBuilder; import com.seibel.distanthorizons.core.logging.f3.F3Screen; import com.seibel.distanthorizons.core.pos.DhBlockPos2D; @@ -272,7 +271,8 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle String updateCompletedTaskSize = (updateExecutor != null) ? updateExecutor.getCompletedTaskCount()+"" : "-"; int unsavedDataSourceCount = this.fullDataSourceProvider.getUnsavedDataSourceCount(); - int migrationCount = this.fullDataSourceProvider.getMigrationCount(); + long legacyDeletionCount = this.fullDataSourceProvider.getLegacyDeletionCount(); + long migrationCount = this.fullDataSourceProvider.getTotalMigrationCount(); @@ -284,6 +284,10 @@ public class ClientLevelModule implements Closeable, AbstractNewDataSourceHandle { lines.add("File Handler [" + dimName + "]"); lines.add(" File thread pool tasks: " + fileQueueSize + " (completed: " + fileCompletedTaskSize + ")"); + if (legacyDeletionCount > 0) + { + lines.add(" Legacy Deletion #: " + legacyDeletionCount); + } if (migrationCount > 0) { lines.add(" Legacy Migration #: " + migrationCount); diff --git a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java index d40f2889c..03a9f232c 100644 --- a/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java +++ b/core/src/main/java/com/seibel/distanthorizons/core/sql/repo/FullDataSourceV1Repo.java @@ -187,7 +187,7 @@ public class FullDataSourceV1Repo extends AbstractDhRepo resultMap = this.queryDictionaryFirst( "select COUNT(*) as itemCount from "+this.getTableName()+" where MigrationFailed <> 1"); @@ -198,7 +198,8 @@ public class FullDataSourceV1Repo extends AbstractDhRepo