Remove files and dirs under no media dir

lib/use_case/compat/v37.dart

@@ -0,0 +1,206 @@
+import 'package:collection/collection.dart';
+import 'package:idb_shim/idb_client.dart';
+import 'package:logging/logging.dart';
+import 'package:nc_photos/app_db.dart';
+import 'package:nc_photos/entity/file_util.dart' as file_util;
+import 'package:nc_photos/iterable_extension.dart';
+import 'package:nc_photos/object_extension.dart';
+import 'package:path/path.dart' as path_lib;
+
+/// Compatibility helper for v37
+class CompatV37 {
+  static Future<void> setAppDbMigrationFlag(AppDb appDb) async {
+    _log.info("[setAppDbMigrationFlag] Set db flag");
+    try {
+      await appDb.use((db) async {
+        final transaction =
+            db.transaction(AppDb.metaStoreName, idbModeReadWrite);
+        final metaStore = transaction.objectStore(AppDb.metaStoreName);
+        await metaStore
+            .put(const AppDbMetaEntryCompatV37(false).toEntry().toJson());
+        await transaction.completed;
+      });
+    } catch (e, stackTrace) {
+      _log.shout(
+          "[setAppDbMigrationFlag] Failed while setting db flag, drop db instead",
+          e,
+          stackTrace);
+      await appDb.delete();
+    }
+  }
+
+  static Future<bool> isAppDbNeedMigration(AppDb appDb) async {
+    final dbItem = await appDb.use((db) async {
+      final transaction = db.transaction(AppDb.metaStoreName, idbModeReadOnly);
+      final metaStore = transaction.objectStore(AppDb.metaStoreName);
+      return await metaStore.getObject(AppDbMetaEntryCompatV37.key) as Map?;
+    });
+    if (dbItem == null) {
+      return false;
+    }
+    try {
+      final dbEntry = AppDbMetaEntry.fromJson(dbItem.cast<String, dynamic>());
+      final compatV37 = AppDbMetaEntryCompatV37.fromJson(dbEntry.obj);
+      return !compatV37.isMigrated;
+    } catch (e, stackTrace) {
+      _log.shout("[isAppDbNeedMigration] Failed", e, stackTrace);
+      return true;
+    }
+  }
+
+  static Future<void> migrateAppDb(AppDb appDb) async {
+    _log.info("[migrateAppDb] Migrate AppDb");
+    try {
+      await appDb.use((db) async {
+        final transaction = db.transaction(
+            [AppDb.file2StoreName, AppDb.dirStoreName, AppDb.metaStoreName],
+            idbModeReadWrite);
+        final noMediaFiles = <_NoMediaFile>[];
+        try {
+          final fileStore = transaction.objectStore(AppDb.file2StoreName);
+          final dirStore = transaction.objectStore(AppDb.dirStoreName);
+          // scan the db to see which dirs contain a no media marker
+          await for (final c in fileStore.openCursor()) {
+            final item = c.value as Map;
+            final strippedPath = item["strippedPath"] as String;
+            if (file_util.isNoMediaMarkerPath(strippedPath)) {
+              noMediaFiles.add(_NoMediaFile(
+                item["server"],
+                item["userId"],
+                path_lib
+                    .dirname(item["strippedPath"])
+                    .run((p) => p == "." ? "" : p),
+                item["file"]["fileId"],
+              ));
+            }
+            c.next();
+          }
+          // sort to make sure parent dirs are always in front of sub dirs
+          noMediaFiles
+              .sort((a, b) => a.strippedDirPath.compareTo(b.strippedDirPath));
+          _log.info(
+              "[migrateAppDb] nomedia dirs: ${noMediaFiles.toReadableString()}");
+
+          if (noMediaFiles.isNotEmpty) {
+            await _migrateAppDbFileStore(appDb, noMediaFiles,
+                fileStore: fileStore);
+            await _migrateAppDbDirStore(appDb, noMediaFiles,
+                dirStore: dirStore);
+          }
+
+          final metaStore = transaction.objectStore(AppDb.metaStoreName);
+          await metaStore
+              .put(const AppDbMetaEntryCompatV37(true).toEntry().toJson());
+        } catch (_) {
+          transaction.abort();
+          rethrow;
+        }
+      });
+    } catch (e, stackTrace) {
+      _log.shout("[migrateAppDb] Failed while migrating, drop db instead", e,
+          stackTrace);
+      await appDb.delete();
+      rethrow;
+    }
+  }
+
+  /// Remove files under no media dirs
+  static Future<void> _migrateAppDbFileStore(
+    AppDb appDb,
+    List<_NoMediaFile> noMediaFiles, {
+    required ObjectStore fileStore,
+  }) async {
+    await for (final c in fileStore.openCursor()) {
+      final item = c.value as Map;
+      final under = noMediaFiles.firstWhereOrNull((e) {
+        if (e.server != item["server"] || e.userId != item["userId"]) {
+          return false;
+        }
+        final prefix = e.strippedDirPath.isEmpty ? "" : "${e.strippedDirPath}/";
+        final itemDir = path_lib
+            .dirname(item["strippedPath"])
+            .run((p) => p == "." ? "" : p);
+        // check isNotEmpty to prevent user root being removed when the
+        // marker is placed in root
+        return item["strippedPath"].isNotEmpty &&
+            item["strippedPath"].startsWith(prefix) &&
+            // keep no media marker in top-most dir
+            !(itemDir == e.strippedDirPath &&
+                file_util.isNoMediaMarkerPath(item["strippedPath"]));
+      });
+      if (under != null) {
+        _log.fine("[_migrateAppDbFileStore] Remove db entry: ${c.primaryKey}");
+        await c.delete();
+      }
+      c.next();
+    }
+  }
+
+  /// Remove dirs under no media dirs
+  static Future<void> _migrateAppDbDirStore(
+    AppDb appDb,
+    List<_NoMediaFile> noMediaFiles, {
+    required ObjectStore dirStore,
+  }) async {
+    await for (final c in dirStore.openCursor()) {
+      final item = c.value as Map;
+      final under = noMediaFiles.firstWhereOrNull((e) {
+        if (e.server != item["server"] || e.userId != item["userId"]) {
+          return false;
+        }
+        final prefix = e.strippedDirPath.isEmpty ? "" : "${e.strippedDirPath}/";
+        return item["strippedPath"].startsWith(prefix) ||
+            e.strippedDirPath == item["strippedPath"];
+      });
+      if (under != null) {
+        if (under.strippedDirPath == item["strippedPath"]) {
+          // this dir contains the no media marker
+          // remove all children, keep only the marker
+          final newChildren = (item["children"] as List)
+              .where((childId) => childId == under.fileId)
+              .toList();
+          if (newChildren.isEmpty) {
+            // ???
+            _log.severe(
+                "[_migrateAppDbDirStore] Marker not found in dir: ${item["strippedPath"]}");
+            // drop this dir
+            await c.delete();
+          }
+          _log.fine(
+              "[_migrateAppDbDirStore] Migrate db entry: ${c.primaryKey}");
+          await c.update(Map.of(item).apply((obj) {
+            obj["children"] = newChildren;
+          }));
+        } else {
+          // this dir is a sub dir
+          // drop this dir
+          _log.fine("[_migrateAppDbDirStore] Remove db entry: ${c.primaryKey}");
+          await c.delete();
+        }
+      }
+      c.next();
+    }
+  }
+
+  static final _log = Logger("use_case.compat.v37.CompatV37");
+}
+
+class _NoMediaFile {
+  const _NoMediaFile(
+      this.server, this.userId, this.strippedDirPath, this.fileId);
+
+  @override
+  toString() => "$runtimeType {"
+      "server: $server, "
+      "userId: $userId, "
+      "strippedDirPath: $strippedDirPath, "
+      "fileId: $fileId, "
+      "}";
+
+  final String server;
+  // no need to use CiString as all strings are stored with the same casing in
+  // db
+  final String userId;
+  final String strippedDirPath;
+  final int fileId;
+}

lib/widget/splash.dart

@@ -11,6 +11,7 @@ import 'package:nc_photos/pref.dart';
 import 'package:nc_photos/snack_bar_manager.dart';
 import 'package:nc_photos/theme.dart';
 import 'package:nc_photos/use_case/compat/v29.dart';
+import 'package:nc_photos/use_case/compat/v37.dart';
 import 'package:nc_photos/use_case/db_compat/v5.dart';
 import 'package:nc_photos/widget/home.dart';
 import 'package:nc_photos/widget/processing_dialog.dart';
@@ -148,6 +149,10 @@ class _SplashState extends State<Splash> {
       showUpdateDialog();
       await _upgrade29(lastVersion);
     }
+    if (lastVersion < 370) {
+      showUpdateDialog();
+      await _upgrade37(lastVersion);
+    }
     if (isShowDialog) {
       Navigator.of(context).pop();
     }
@@ -163,6 +168,11 @@ class _SplashState extends State<Splash> {
     }
   }
 
+  Future<void> _upgrade37(int lastVersion) async {
+    final c = KiwiContainer().resolve<DiContainer>();
+    return CompatV37.setAppDbMigrationFlag(c.appDb);
+  }
+
   Future<void> _migrateDb() async {
     bool isShowDialog = false;
     void showUpdateDialog() {
@@ -189,6 +199,17 @@ class _SplashState extends State<Splash> {
         ));
       }
     }
+    if (await CompatV37.isAppDbNeedMigration(c.appDb)) {
+      showUpdateDialog();
+      try {
+        await CompatV37.migrateAppDb(c.appDb);
+      } catch (_) {
+        SnackBarManager().showSnackBar(SnackBar(
+          content: Text(L10n.global().migrateDatabaseFailureNotification),
+          duration: k.snackBarDurationNormal,
+        ));
+      }
+    }
     if (isShowDialog) {
       Navigator.of(context).pop();
     }

test/use_case/compat/v37_test.dart

@@ -0,0 +1,234 @@
+import 'package:idb_shim/idb_client.dart';
+import 'package:nc_photos/app_db.dart';
+import 'package:nc_photos/list_extension.dart';
+import 'package:nc_photos/use_case/compat/v37.dart';
+import 'package:test/test.dart';
+
+import '../../mock_type.dart';
+import '../../test_util.dart' as util;
+
+void main() {
+  group("CompatV37", () {
+    group("isAppDbNeedMigration", () {
+      test("w/ meta entry == false", _isAppDbNeedMigrationEntryFalse);
+      test("w/ meta entry == true", _isAppDbNeedMigrationEntryTrue);
+      test("w/o meta entry", _isAppDbNeedMigrationWithoutEntry);
+    });
+    group("migrateAppDb", () {
+      test("w/o nomedia", _migrateAppDbWithoutNomedia);
+      test("w/ nomedia", _migrateAppDb);
+      test("w/ nomedia nested dir", _migrateAppDbNestedDir);
+      test("w/ nomedia nested no media marker", _migrateAppDbNestedMarker);
+      test("w/ nomedia root", _migrateAppDbRoot);
+    });
+  });
+}
+
+/// Check if migration is necessary with isMigrated flag = false
+///
+/// Expect: true
+Future<void> _isAppDbNeedMigrationEntryFalse() async {
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    final transaction = db.transaction(AppDb.metaStoreName, idbModeReadWrite);
+    final metaStore = transaction.objectStore(AppDb.metaStoreName);
+    const entry = AppDbMetaEntryCompatV37(false);
+    await metaStore.put(entry.toEntry().toJson());
+  });
+
+  expect(await CompatV37.isAppDbNeedMigration(appDb), true);
+}
+
+/// Check if migration is necessary with isMigrated flag = true
+///
+/// Expect: false
+Future<void> _isAppDbNeedMigrationEntryTrue() async {
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    final transaction = db.transaction(AppDb.metaStoreName, idbModeReadWrite);
+    final metaStore = transaction.objectStore(AppDb.metaStoreName);
+    const entry = AppDbMetaEntryCompatV37(true);
+    await metaStore.put(entry.toEntry().toJson());
+  });
+
+  expect(await CompatV37.isAppDbNeedMigration(appDb), false);
+}
+
+/// Check if migration is necessary with isMigrated flag missing
+///
+/// Expect: false
+Future<void> _isAppDbNeedMigrationWithoutEntry() async {
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    final transaction = db.transaction(AppDb.metaStoreName, idbModeReadWrite);
+    final metaStore = transaction.objectStore(AppDb.metaStoreName);
+    const entry = AppDbMetaEntryCompatV37(true);
+    await metaStore.put(entry.toEntry().toJson());
+  });
+
+  expect(await CompatV37.isAppDbNeedMigration(appDb), false);
+}
+
+/// Migrate db without nomedia file
+///
+/// Expect: all files remain
+Future<void> _migrateAppDbWithoutNomedia() async {
+  final account = util.buildAccount();
+  final files = (util.FilesBuilder()
+        ..addDir("admin")
+        ..addJpeg("admin/test1.jpg")
+        ..addDir("admin/dir1")
+        ..addJpeg("admin/dir1/test2.jpg"))
+      .build();
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    await util.fillAppDb(appDb, account, files);
+    await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3));
+    await util.fillAppDbDir(appDb, account, files[2], [files[3]]);
+  });
+  await CompatV37.migrateAppDb(appDb);
+
+  final fileObjs = await util.listAppDb(
+      appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file);
+  expect(fileObjs, files);
+  final dirEntries = await util.listAppDb(
+      appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e));
+  expect(dirEntries, [
+    AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)),
+    AppDbDirEntry.fromFiles(account, files[2], [files[3]]),
+  ]);
+}
+
+/// Migrate db with nomedia file
+///
+/// nomedia: admin/dir1/.nomedia
+/// Expect: files (except .nomedia) under admin/dir1 removed
+Future<void> _migrateAppDb() async {
+  final account = util.buildAccount();
+  final files = (util.FilesBuilder()
+        ..addDir("admin")
+        ..addJpeg("admin/test1.jpg")
+        ..addDir("admin/dir1")
+        ..addGenericFile("admin/dir1/.nomedia", "text/plain")
+        ..addJpeg("admin/dir1/test2.jpg"))
+      .build();
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    await util.fillAppDb(appDb, account, files);
+    await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3));
+    await util.fillAppDbDir(appDb, account, files[2], files.slice(3, 5));
+  });
+  await CompatV37.migrateAppDb(appDb);
+
+  final fileObjs = await util.listAppDb(
+      appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file);
+  expect(fileObjs, files.slice(0, 4));
+  final dirEntries = await util.listAppDb(
+      appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e));
+  expect(dirEntries, [
+    AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)),
+    AppDbDirEntry.fromFiles(account, files[2], [files[3]]),
+  ]);
+}
+
+/// Migrate db with nomedia file
+///
+/// nomedia: admin/dir1/.nomedia
+/// Expect: files (except .nomedia) under admin/dir1 removed
+Future<void> _migrateAppDbNestedDir() async {
+  final account = util.buildAccount();
+  final files = (util.FilesBuilder()
+        ..addDir("admin")
+        ..addJpeg("admin/test1.jpg")
+        ..addDir("admin/dir1")
+        ..addGenericFile("admin/dir1/.nomedia", "text/plain")
+        ..addJpeg("admin/dir1/test2.jpg")
+        ..addDir("admin/dir1/dir1-1")
+        ..addJpeg("admin/dir1/dir1-1/test3.jpg"))
+      .build();
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    await util.fillAppDb(appDb, account, files);
+    await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3));
+    await util.fillAppDbDir(appDb, account, files[2], files.slice(3, 6));
+    await util.fillAppDbDir(appDb, account, files[5], [files[6]]);
+  });
+  await CompatV37.migrateAppDb(appDb);
+
+  final fileObjs = await util.listAppDb(
+      appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file);
+  expect(fileObjs, files.slice(0, 4));
+  final dirEntries = await util.listAppDb(
+      appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e));
+  expect(dirEntries, [
+    AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)),
+    AppDbDirEntry.fromFiles(account, files[2], [files[3]]),
+  ]);
+}
+
+/// Migrate db with nomedia file
+///
+/// nomedia: admin/dir1/.nomedia, admin/dir1/dir1-1/.nomedia
+/// Expect: files (except admin/dir1/.nomedia) under admin/dir1 removed
+Future<void> _migrateAppDbNestedMarker() async {
+  final account = util.buildAccount();
+  final files = (util.FilesBuilder()
+        ..addDir("admin")
+        ..addJpeg("admin/test1.jpg")
+        ..addDir("admin/dir1")
+        ..addGenericFile("admin/dir1/.nomedia", "text/plain")
+        ..addJpeg("admin/dir1/test2.jpg")
+        ..addDir("admin/dir1/dir1-1")
+        ..addGenericFile("admin/dir1/dir1-1/.nomedia", "text/plain")
+        ..addJpeg("admin/dir1/dir1-1/test3.jpg"))
+      .build();
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    await util.fillAppDb(appDb, account, files);
+    await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3));
+    await util.fillAppDbDir(appDb, account, files[2], files.slice(3, 6));
+    await util.fillAppDbDir(appDb, account, files[5], files.slice(6, 8));
+  });
+  await CompatV37.migrateAppDb(appDb);
+
+  final fileObjs = await util.listAppDb(
+      appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file);
+  expect(fileObjs, files.slice(0, 4));
+  final dirEntries = await util.listAppDb(
+      appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e));
+  expect(dirEntries, [
+    AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)),
+    AppDbDirEntry.fromFiles(account, files[2], [files[3]]),
+  ]);
+}
+
+/// Migrate db with nomedia file
+///
+/// nomedia: admin/.nomedia
+/// Expect: files (except .nomedia) under admin removed
+Future<void> _migrateAppDbRoot() async {
+  final account = util.buildAccount();
+  final files = (util.FilesBuilder()
+        ..addDir("admin")
+        ..addGenericFile("admin/.nomedia", "text/plain")
+        ..addJpeg("admin/test1.jpg")
+        ..addDir("admin/dir1")
+        ..addJpeg("admin/dir1/test2.jpg"))
+      .build();
+  final appDb = MockAppDb();
+  await appDb.use((db) async {
+    await util.fillAppDb(appDb, account, files);
+    await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 4));
+    await util.fillAppDbDir(appDb, account, files[3], [files[4]]);
+  });
+  await CompatV37.migrateAppDb(appDb);
+
+  final objs = await util.listAppDb(
+      appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file);
+  expect(objs, files.slice(0, 2));
+  final dirEntries = await util.listAppDb(
+      appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e));
+  expect(dirEntries, [
+    AppDbDirEntry.fromFiles(account, files[0], [files[1]]),
+  ]);
+}