From 16fa9af03037f1bfbd70756deda315ebe33db707 Mon Sep 17 00:00:00 2001 From: Ming Ming Date: Tue, 8 Feb 2022 03:22:41 +0800 Subject: [PATCH 1/4] No longer return other files in a no media dir --- lib/app_db.dart | 15 +++++++++++++ lib/entity/file/data_source.dart | 37 ++++++++++++++++++++++++-------- lib/entity/file_util.dart | 7 ++++-- lib/iterable_extension.dart | 9 ++++++++ 4 files changed, 57 insertions(+), 11 deletions(-) diff --git a/lib/app_db.dart b/lib/app_db.dart index aacc4362..afcca233 100644 --- a/lib/app_db.dart +++ b/lib/app_db.dart @@ -451,6 +451,21 @@ class AppDbMetaEntryDbCompatV5 { final bool isMigrated; } +class AppDbMetaEntryCompatV37 { + static const key = "compatV37"; + + const AppDbMetaEntryCompatV37(this.isMigrated); + + factory AppDbMetaEntryCompatV37.fromJson(JsonObj json) => + AppDbMetaEntryCompatV37(json["isMigrated"]); + + AppDbMetaEntry toEntry() => AppDbMetaEntry(key, { + "isMigrated": isMigrated, + }); + + final bool isMigrated; +} + class _DummyVersionChangeEvent implements VersionChangeEvent { const _DummyVersionChangeEvent(this.oldVersion, this.newVersion, this.transaction, this.target, this.currentTarget, this.database); diff --git a/lib/entity/file/data_source.dart b/lib/entity/file/data_source.dart index 05cd8f3b..c775cdba 100644 --- a/lib/entity/file/data_source.dart +++ b/lib/entity/file/data_source.dart @@ -9,6 +9,7 @@ import 'package:nc_photos/api/api.dart'; import 'package:nc_photos/app_db.dart'; import 'package:nc_photos/debug_util.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file_util.dart' as file_util; import 'package:nc_photos/entity/webdav_response_parser.dart'; import 'package:nc_photos/exception.dart'; import 'package:nc_photos/iterable_extension.dart'; @@ -64,17 +65,35 @@ class FileWebdavDataSource implements FileDataSource { final xml = XmlDocument.parse(response.body); var files = WebdavResponseParser().parseFiles(xml); // _log.fine("[list] Parsed files: [$files]"); - files = files.where((element) => _validateFile(element)).map((e) { - if (e.metadata == null || e.metadata!.fileEtag == e.etag) { - return e; - } else { - _log.info("[list] Ignore outdated metadata for ${e.path}"); - return e.copyWith(metadata: OrNull(null)); - } - }).toList(); + bool hasNoMediaMarker = false; + files = files + .forEachLazy((f) { + if (file_util.isNoMediaMarker(f)) { + hasNoMediaMarker = true; + } + }) + .where((f) => _validateFile(f)) + .map((e) { + if (e.metadata == null || e.metadata!.fileEtag == e.etag) { + return e; + } else { + _log.info("[list] Ignore outdated metadata for ${e.path}"); + return e.copyWith(metadata: OrNull(null)); + } + }) + .toList(); await _compatUpgrade(account, files); - return files; + + if (hasNoMediaMarker) { + // return only the marker and the dir itself + return files + .where((f) => + dir.compareServerIdentity(f) || file_util.isNoMediaMarker(f)) + .toList(); + } else { + return files; + } } @override diff --git a/lib/entity/file_util.dart b/lib/entity/file_util.dart index 8eee8d69..68b27b65 100644 --- a/lib/entity/file_util.dart +++ b/lib/entity/file_util.dart @@ -72,8 +72,11 @@ String renameConflict(String filename, int conflictCount) { /// /// A no media marker marks the parent dir and its sub dirs as not containing /// media files of interest -bool isNoMediaMarker(File file) { - final filename = file.filename; +bool isNoMediaMarker(File file) => isNoMediaMarkerPath(file.path); + +/// See [isNoMediaMarker] +bool isNoMediaMarkerPath(String path) { + final filename = path_lib.basename(path); return filename == ".nomedia" || filename == ".noimage"; } diff --git a/lib/iterable_extension.dart b/lib/iterable_extension.dart index 898e0450..03ab3368 100644 --- a/lib/iterable_extension.dart +++ b/lib/iterable_extension.dart @@ -56,4 +56,13 @@ extension IterableExtension on Iterable { return where((element) => s.add(OverrideComparator(element, equalFn, hashCodeFn))).toList(); } + + /// Invokes [action] on each element of this iterable in iteration order + /// lazily + Iterable forEachLazy(void Function(T element) action) sync* { + for (final e in this) { + action(e); + yield e; + } + } } From 8d7b5080845ceaeb0ff48c381602cfb760a234c0 Mon Sep 17 00:00:00 2001 From: Ming Ming Date: Tue, 8 Feb 2022 04:32:39 +0800 Subject: [PATCH 2/4] Remove files and dirs under no media dir --- lib/use_case/compat/v37.dart | 206 +++++++++++++++++++++++++ lib/widget/splash.dart | 21 +++ test/use_case/compat/v37_test.dart | 234 +++++++++++++++++++++++++++++ 3 files changed, 461 insertions(+) create mode 100644 lib/use_case/compat/v37.dart create mode 100644 test/use_case/compat/v37_test.dart diff --git a/lib/use_case/compat/v37.dart b/lib/use_case/compat/v37.dart new file mode 100644 index 00000000..d35bd926 --- /dev/null +++ b/lib/use_case/compat/v37.dart @@ -0,0 +1,206 @@ +import 'package:collection/collection.dart'; +import 'package:idb_shim/idb_client.dart'; +import 'package:logging/logging.dart'; +import 'package:nc_photos/app_db.dart'; +import 'package:nc_photos/entity/file_util.dart' as file_util; +import 'package:nc_photos/iterable_extension.dart'; +import 'package:nc_photos/object_extension.dart'; +import 'package:path/path.dart' as path_lib; + +/// Compatibility helper for v37 +class CompatV37 { + static Future setAppDbMigrationFlag(AppDb appDb) async { + _log.info("[setAppDbMigrationFlag] Set db flag"); + try { + await appDb.use((db) async { + final transaction = + db.transaction(AppDb.metaStoreName, idbModeReadWrite); + final metaStore = transaction.objectStore(AppDb.metaStoreName); + await metaStore + .put(const AppDbMetaEntryCompatV37(false).toEntry().toJson()); + await transaction.completed; + }); + } catch (e, stackTrace) { + _log.shout( + "[setAppDbMigrationFlag] Failed while setting db flag, drop db instead", + e, + stackTrace); + await appDb.delete(); + } + } + + static Future isAppDbNeedMigration(AppDb appDb) async { + final dbItem = await appDb.use((db) async { + final transaction = db.transaction(AppDb.metaStoreName, idbModeReadOnly); + final metaStore = transaction.objectStore(AppDb.metaStoreName); + return await metaStore.getObject(AppDbMetaEntryCompatV37.key) as Map?; + }); + if (dbItem == null) { + return false; + } + try { + final dbEntry = AppDbMetaEntry.fromJson(dbItem.cast()); + final compatV37 = AppDbMetaEntryCompatV37.fromJson(dbEntry.obj); + return !compatV37.isMigrated; + } catch (e, stackTrace) { + _log.shout("[isAppDbNeedMigration] Failed", e, stackTrace); + return true; + } + } + + static Future migrateAppDb(AppDb appDb) async { + _log.info("[migrateAppDb] Migrate AppDb"); + try { + await appDb.use((db) async { + final transaction = db.transaction( + [AppDb.file2StoreName, AppDb.dirStoreName, AppDb.metaStoreName], + idbModeReadWrite); + final noMediaFiles = <_NoMediaFile>[]; + try { + final fileStore = transaction.objectStore(AppDb.file2StoreName); + final dirStore = transaction.objectStore(AppDb.dirStoreName); + // scan the db to see which dirs contain a no media marker + await for (final c in fileStore.openCursor()) { + final item = c.value as Map; + final strippedPath = item["strippedPath"] as String; + if (file_util.isNoMediaMarkerPath(strippedPath)) { + noMediaFiles.add(_NoMediaFile( + item["server"], + item["userId"], + path_lib + .dirname(item["strippedPath"]) + .run((p) => p == "." ? "" : p), + item["file"]["fileId"], + )); + } + c.next(); + } + // sort to make sure parent dirs are always in front of sub dirs + noMediaFiles + .sort((a, b) => a.strippedDirPath.compareTo(b.strippedDirPath)); + _log.info( + "[migrateAppDb] nomedia dirs: ${noMediaFiles.toReadableString()}"); + + if (noMediaFiles.isNotEmpty) { + await _migrateAppDbFileStore(appDb, noMediaFiles, + fileStore: fileStore); + await _migrateAppDbDirStore(appDb, noMediaFiles, + dirStore: dirStore); + } + + final metaStore = transaction.objectStore(AppDb.metaStoreName); + await metaStore + .put(const AppDbMetaEntryCompatV37(true).toEntry().toJson()); + } catch (_) { + transaction.abort(); + rethrow; + } + }); + } catch (e, stackTrace) { + _log.shout("[migrateAppDb] Failed while migrating, drop db instead", e, + stackTrace); + await appDb.delete(); + rethrow; + } + } + + /// Remove files under no media dirs + static Future _migrateAppDbFileStore( + AppDb appDb, + List<_NoMediaFile> noMediaFiles, { + required ObjectStore fileStore, + }) async { + await for (final c in fileStore.openCursor()) { + final item = c.value as Map; + final under = noMediaFiles.firstWhereOrNull((e) { + if (e.server != item["server"] || e.userId != item["userId"]) { + return false; + } + final prefix = e.strippedDirPath.isEmpty ? "" : "${e.strippedDirPath}/"; + final itemDir = path_lib + .dirname(item["strippedPath"]) + .run((p) => p == "." ? "" : p); + // check isNotEmpty to prevent user root being removed when the + // marker is placed in root + return item["strippedPath"].isNotEmpty && + item["strippedPath"].startsWith(prefix) && + // keep no media marker in top-most dir + !(itemDir == e.strippedDirPath && + file_util.isNoMediaMarkerPath(item["strippedPath"])); + }); + if (under != null) { + _log.fine("[_migrateAppDbFileStore] Remove db entry: ${c.primaryKey}"); + await c.delete(); + } + c.next(); + } + } + + /// Remove dirs under no media dirs + static Future _migrateAppDbDirStore( + AppDb appDb, + List<_NoMediaFile> noMediaFiles, { + required ObjectStore dirStore, + }) async { + await for (final c in dirStore.openCursor()) { + final item = c.value as Map; + final under = noMediaFiles.firstWhereOrNull((e) { + if (e.server != item["server"] || e.userId != item["userId"]) { + return false; + } + final prefix = e.strippedDirPath.isEmpty ? "" : "${e.strippedDirPath}/"; + return item["strippedPath"].startsWith(prefix) || + e.strippedDirPath == item["strippedPath"]; + }); + if (under != null) { + if (under.strippedDirPath == item["strippedPath"]) { + // this dir contains the no media marker + // remove all children, keep only the marker + final newChildren = (item["children"] as List) + .where((childId) => childId == under.fileId) + .toList(); + if (newChildren.isEmpty) { + // ??? + _log.severe( + "[_migrateAppDbDirStore] Marker not found in dir: ${item["strippedPath"]}"); + // drop this dir + await c.delete(); + } + _log.fine( + "[_migrateAppDbDirStore] Migrate db entry: ${c.primaryKey}"); + await c.update(Map.of(item).apply((obj) { + obj["children"] = newChildren; + })); + } else { + // this dir is a sub dir + // drop this dir + _log.fine("[_migrateAppDbDirStore] Remove db entry: ${c.primaryKey}"); + await c.delete(); + } + } + c.next(); + } + } + + static final _log = Logger("use_case.compat.v37.CompatV37"); +} + +class _NoMediaFile { + const _NoMediaFile( + this.server, this.userId, this.strippedDirPath, this.fileId); + + @override + toString() => "$runtimeType {" + "server: $server, " + "userId: $userId, " + "strippedDirPath: $strippedDirPath, " + "fileId: $fileId, " + "}"; + + final String server; + // no need to use CiString as all strings are stored with the same casing in + // db + final String userId; + final String strippedDirPath; + final int fileId; +} diff --git a/lib/widget/splash.dart b/lib/widget/splash.dart index e36f322e..076bb208 100644 --- a/lib/widget/splash.dart +++ b/lib/widget/splash.dart @@ -11,6 +11,7 @@ import 'package:nc_photos/pref.dart'; import 'package:nc_photos/snack_bar_manager.dart'; import 'package:nc_photos/theme.dart'; import 'package:nc_photos/use_case/compat/v29.dart'; +import 'package:nc_photos/use_case/compat/v37.dart'; import 'package:nc_photos/use_case/db_compat/v5.dart'; import 'package:nc_photos/widget/home.dart'; import 'package:nc_photos/widget/processing_dialog.dart'; @@ -148,6 +149,10 @@ class _SplashState extends State { showUpdateDialog(); await _upgrade29(lastVersion); } + if (lastVersion < 370) { + showUpdateDialog(); + await _upgrade37(lastVersion); + } if (isShowDialog) { Navigator.of(context).pop(); } @@ -163,6 +168,11 @@ class _SplashState extends State { } } + Future _upgrade37(int lastVersion) async { + final c = KiwiContainer().resolve(); + return CompatV37.setAppDbMigrationFlag(c.appDb); + } + Future _migrateDb() async { bool isShowDialog = false; void showUpdateDialog() { @@ -189,6 +199,17 @@ class _SplashState extends State { )); } } + if (await CompatV37.isAppDbNeedMigration(c.appDb)) { + showUpdateDialog(); + try { + await CompatV37.migrateAppDb(c.appDb); + } catch (_) { + SnackBarManager().showSnackBar(SnackBar( + content: Text(L10n.global().migrateDatabaseFailureNotification), + duration: k.snackBarDurationNormal, + )); + } + } if (isShowDialog) { Navigator.of(context).pop(); } diff --git a/test/use_case/compat/v37_test.dart b/test/use_case/compat/v37_test.dart new file mode 100644 index 00000000..4691d4b9 --- /dev/null +++ b/test/use_case/compat/v37_test.dart @@ -0,0 +1,234 @@ +import 'package:idb_shim/idb_client.dart'; +import 'package:nc_photos/app_db.dart'; +import 'package:nc_photos/list_extension.dart'; +import 'package:nc_photos/use_case/compat/v37.dart'; +import 'package:test/test.dart'; + +import '../../mock_type.dart'; +import '../../test_util.dart' as util; + +void main() { + group("CompatV37", () { + group("isAppDbNeedMigration", () { + test("w/ meta entry == false", _isAppDbNeedMigrationEntryFalse); + test("w/ meta entry == true", _isAppDbNeedMigrationEntryTrue); + test("w/o meta entry", _isAppDbNeedMigrationWithoutEntry); + }); + group("migrateAppDb", () { + test("w/o nomedia", _migrateAppDbWithoutNomedia); + test("w/ nomedia", _migrateAppDb); + test("w/ nomedia nested dir", _migrateAppDbNestedDir); + test("w/ nomedia nested no media marker", _migrateAppDbNestedMarker); + test("w/ nomedia root", _migrateAppDbRoot); + }); + }); +} + +/// Check if migration is necessary with isMigrated flag = false +/// +/// Expect: true +Future _isAppDbNeedMigrationEntryFalse() async { + final appDb = MockAppDb(); + await appDb.use((db) async { + final transaction = db.transaction(AppDb.metaStoreName, idbModeReadWrite); + final metaStore = transaction.objectStore(AppDb.metaStoreName); + const entry = AppDbMetaEntryCompatV37(false); + await metaStore.put(entry.toEntry().toJson()); + }); + + expect(await CompatV37.isAppDbNeedMigration(appDb), true); +} + +/// Check if migration is necessary with isMigrated flag = true +/// +/// Expect: false +Future _isAppDbNeedMigrationEntryTrue() async { + final appDb = MockAppDb(); + await appDb.use((db) async { + final transaction = db.transaction(AppDb.metaStoreName, idbModeReadWrite); + final metaStore = transaction.objectStore(AppDb.metaStoreName); + const entry = AppDbMetaEntryCompatV37(true); + await metaStore.put(entry.toEntry().toJson()); + }); + + expect(await CompatV37.isAppDbNeedMigration(appDb), false); +} + +/// Check if migration is necessary with isMigrated flag missing +/// +/// Expect: false +Future _isAppDbNeedMigrationWithoutEntry() async { + final appDb = MockAppDb(); + await appDb.use((db) async { + final transaction = db.transaction(AppDb.metaStoreName, idbModeReadWrite); + final metaStore = transaction.objectStore(AppDb.metaStoreName); + const entry = AppDbMetaEntryCompatV37(true); + await metaStore.put(entry.toEntry().toJson()); + }); + + expect(await CompatV37.isAppDbNeedMigration(appDb), false); +} + +/// Migrate db without nomedia file +/// +/// Expect: all files remain +Future _migrateAppDbWithoutNomedia() async { + final account = util.buildAccount(); + final files = (util.FilesBuilder() + ..addDir("admin") + ..addJpeg("admin/test1.jpg") + ..addDir("admin/dir1") + ..addJpeg("admin/dir1/test2.jpg")) + .build(); + final appDb = MockAppDb(); + await appDb.use((db) async { + await util.fillAppDb(appDb, account, files); + await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3)); + await util.fillAppDbDir(appDb, account, files[2], [files[3]]); + }); + await CompatV37.migrateAppDb(appDb); + + final fileObjs = await util.listAppDb( + appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file); + expect(fileObjs, files); + final dirEntries = await util.listAppDb( + appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e)); + expect(dirEntries, [ + AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)), + AppDbDirEntry.fromFiles(account, files[2], [files[3]]), + ]); +} + +/// Migrate db with nomedia file +/// +/// nomedia: admin/dir1/.nomedia +/// Expect: files (except .nomedia) under admin/dir1 removed +Future _migrateAppDb() async { + final account = util.buildAccount(); + final files = (util.FilesBuilder() + ..addDir("admin") + ..addJpeg("admin/test1.jpg") + ..addDir("admin/dir1") + ..addGenericFile("admin/dir1/.nomedia", "text/plain") + ..addJpeg("admin/dir1/test2.jpg")) + .build(); + final appDb = MockAppDb(); + await appDb.use((db) async { + await util.fillAppDb(appDb, account, files); + await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3)); + await util.fillAppDbDir(appDb, account, files[2], files.slice(3, 5)); + }); + await CompatV37.migrateAppDb(appDb); + + final fileObjs = await util.listAppDb( + appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file); + expect(fileObjs, files.slice(0, 4)); + final dirEntries = await util.listAppDb( + appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e)); + expect(dirEntries, [ + AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)), + AppDbDirEntry.fromFiles(account, files[2], [files[3]]), + ]); +} + +/// Migrate db with nomedia file +/// +/// nomedia: admin/dir1/.nomedia +/// Expect: files (except .nomedia) under admin/dir1 removed +Future _migrateAppDbNestedDir() async { + final account = util.buildAccount(); + final files = (util.FilesBuilder() + ..addDir("admin") + ..addJpeg("admin/test1.jpg") + ..addDir("admin/dir1") + ..addGenericFile("admin/dir1/.nomedia", "text/plain") + ..addJpeg("admin/dir1/test2.jpg") + ..addDir("admin/dir1/dir1-1") + ..addJpeg("admin/dir1/dir1-1/test3.jpg")) + .build(); + final appDb = MockAppDb(); + await appDb.use((db) async { + await util.fillAppDb(appDb, account, files); + await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3)); + await util.fillAppDbDir(appDb, account, files[2], files.slice(3, 6)); + await util.fillAppDbDir(appDb, account, files[5], [files[6]]); + }); + await CompatV37.migrateAppDb(appDb); + + final fileObjs = await util.listAppDb( + appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file); + expect(fileObjs, files.slice(0, 4)); + final dirEntries = await util.listAppDb( + appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e)); + expect(dirEntries, [ + AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)), + AppDbDirEntry.fromFiles(account, files[2], [files[3]]), + ]); +} + +/// Migrate db with nomedia file +/// +/// nomedia: admin/dir1/.nomedia, admin/dir1/dir1-1/.nomedia +/// Expect: files (except admin/dir1/.nomedia) under admin/dir1 removed +Future _migrateAppDbNestedMarker() async { + final account = util.buildAccount(); + final files = (util.FilesBuilder() + ..addDir("admin") + ..addJpeg("admin/test1.jpg") + ..addDir("admin/dir1") + ..addGenericFile("admin/dir1/.nomedia", "text/plain") + ..addJpeg("admin/dir1/test2.jpg") + ..addDir("admin/dir1/dir1-1") + ..addGenericFile("admin/dir1/dir1-1/.nomedia", "text/plain") + ..addJpeg("admin/dir1/dir1-1/test3.jpg")) + .build(); + final appDb = MockAppDb(); + await appDb.use((db) async { + await util.fillAppDb(appDb, account, files); + await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 3)); + await util.fillAppDbDir(appDb, account, files[2], files.slice(3, 6)); + await util.fillAppDbDir(appDb, account, files[5], files.slice(6, 8)); + }); + await CompatV37.migrateAppDb(appDb); + + final fileObjs = await util.listAppDb( + appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file); + expect(fileObjs, files.slice(0, 4)); + final dirEntries = await util.listAppDb( + appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e)); + expect(dirEntries, [ + AppDbDirEntry.fromFiles(account, files[0], files.slice(1, 3)), + AppDbDirEntry.fromFiles(account, files[2], [files[3]]), + ]); +} + +/// Migrate db with nomedia file +/// +/// nomedia: admin/.nomedia +/// Expect: files (except .nomedia) under admin removed +Future _migrateAppDbRoot() async { + final account = util.buildAccount(); + final files = (util.FilesBuilder() + ..addDir("admin") + ..addGenericFile("admin/.nomedia", "text/plain") + ..addJpeg("admin/test1.jpg") + ..addDir("admin/dir1") + ..addJpeg("admin/dir1/test2.jpg")) + .build(); + final appDb = MockAppDb(); + await appDb.use((db) async { + await util.fillAppDb(appDb, account, files); + await util.fillAppDbDir(appDb, account, files[0], files.slice(1, 4)); + await util.fillAppDbDir(appDb, account, files[3], [files[4]]); + }); + await CompatV37.migrateAppDb(appDb); + + final objs = await util.listAppDb( + appDb, AppDb.file2StoreName, (e) => AppDbFile2Entry.fromJson(e).file); + expect(objs, files.slice(0, 2)); + final dirEntries = await util.listAppDb( + appDb, AppDb.dirStoreName, (e) => AppDbDirEntry.fromJson(e)); + expect(dirEntries, [ + AppDbDirEntry.fromFiles(account, files[0], [files[1]]), + ]); +} From 49f592a1c94c7d5514f4eb5c047fa994d719721b Mon Sep 17 00:00:00 2001 From: Ming Ming Date: Tue, 8 Feb 2022 13:50:03 +0800 Subject: [PATCH 3/4] Remove no media marker handling in use cases --- lib/use_case/scan_dir.dart | 11 ------- lib/use_case/scan_dir_offline.dart | 35 ++++------------------- lib/use_case/scan_missing_metadata.dart | 3 +- lib/use_case/update_missing_metadata.dart | 4 +-- 4 files changed, 8 insertions(+), 45 deletions(-) diff --git a/lib/use_case/scan_dir.dart b/lib/use_case/scan_dir.dart index 0c076386..eb8c5137 100644 --- a/lib/use_case/scan_dir.dart +++ b/lib/use_case/scan_dir.dart @@ -12,15 +12,9 @@ class ScanDir { ScanDir(this.fileRepo); /// List all files under a dir recursively - /// - /// Dirs with a .nomedia/.noimage file will be ignored. The returned stream - /// would emit either List or ExceptionEvent Stream call(Account account, File root) async* { try { final items = await Ls(fileRepo)(account, root); - if (_shouldScanIgnoreDir(items)) { - return; - } yield items .where( (f) => f.isCollection != true && file_util.isSupportedFormat(f)) @@ -43,11 +37,6 @@ class ScanDir { } } - /// Return if this dir should be ignored in a scan op based on files under - /// this dir - static bool _shouldScanIgnoreDir(Iterable files) => - files.any((f) => file_util.isNoMediaMarker(f)); - final FileRepo fileRepo; static final _log = Logger("use_case.scan_dir.ScanDir"); diff --git a/lib/use_case/scan_dir_offline.dart b/lib/use_case/scan_dir_offline.dart index afc32ab6..f94e11a2 100644 --- a/lib/use_case/scan_dir_offline.dart +++ b/lib/use_case/scan_dir_offline.dart @@ -1,12 +1,9 @@ import 'package:idb_shim/idb_client.dart'; -import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/app_db.dart'; import 'package:nc_photos/di_container.dart'; import 'package:nc_photos/entity/file.dart'; import 'package:nc_photos/entity/file_util.dart' as file_util; -import 'package:nc_photos/iterable_extension.dart'; -import 'package:path/path.dart' as path_lib; class ScanDirOffline { ScanDirOffline(this._c) : assert(require(_c)); @@ -14,11 +11,8 @@ class ScanDirOffline { static bool require(DiContainer c) => DiContainer.has(c, DiType.appDb); /// List all files under a dir recursively from the local DB - /// - /// Dirs with a .nomedia/.noimage file will be ignored Future> call(Account account, File root) async { - final skipDirs = []; - final files = await _c.appDb.use((db) async { + return await _c.appDb.use((db) async { final transaction = db.transaction(AppDb.file2StoreName, idbModeReadOnly); final store = transaction.objectStore(AppDb.file2StoreName); final index = store.index(AppDbFile2Entry.strippedPathIndexName); @@ -26,34 +20,15 @@ class ScanDirOffline { AppDbFile2Entry.toStrippedPathIndexLowerKeyForDir(account, root), AppDbFile2Entry.toStrippedPathIndexUpperKeyForDir(account, root), ); - final files = []; - await for (final f in index + return await index .openCursor(range: range, autoAdvance: true) .map((c) => c.value) .cast() - .map((e) => - AppDbFile2Entry.fromJson(e.cast()).file)) { - if (file_util.isNoMediaMarker(f)) { - skipDirs.add(File(path: path_lib.dirname(f.path))); - } else if (file_util.isSupportedFormat(f)) { - files.add(f); - } - } - return files; - }); - - _log.info( - "[call] Skip dirs: ${skipDirs.map((d) => d.strippedPath).toReadableString()}"); - if (skipDirs.isEmpty) { - return files; - } else { - return files - .where((f) => !skipDirs.any((d) => file_util.isUnderDir(f, d))) + .map((e) => AppDbFile2Entry.fromJson(e.cast()).file) + .where((f) => file_util.isSupportedFormat(f)) .toList(); - } + }); } final DiContainer _c; - - static final _log = Logger("use_case.scan_dir_offline.ScanDirOffline"); } diff --git a/lib/use_case/scan_missing_metadata.dart b/lib/use_case/scan_missing_metadata.dart index 0e64d5f7..00d34688 100644 --- a/lib/use_case/scan_missing_metadata.dart +++ b/lib/use_case/scan_missing_metadata.dart @@ -10,8 +10,7 @@ class ScanMissingMetadata { /// List all files that support metadata but yet having one under a dir /// - /// Dirs with a .nomedia/.noimage file will be ignored. The returned stream - /// would emit either File data or ExceptionEvent + /// The returned stream would emit either File data or ExceptionEvent /// /// If [isRecursive] is true, [root] and its sub dirs will be listed, /// otherwise only [root] will be listed. Default to true diff --git a/lib/use_case/update_missing_metadata.dart b/lib/use_case/update_missing_metadata.dart index e321ee08..3c1364bf 100644 --- a/lib/use_case/update_missing_metadata.dart +++ b/lib/use_case/update_missing_metadata.dart @@ -19,8 +19,8 @@ class UpdateMissingMetadata { /// Update metadata for all files that support one under a dir /// - /// Dirs with a .nomedia/.noimage file will be ignored. The returned stream - /// would emit either File data (for each updated files) or ExceptionEvent + /// The returned stream would emit either File data (for each updated files) + /// or ExceptionEvent /// /// If [isRecursive] is true, [root] and its sub dirs will be scanned, /// otherwise only [root] will be scanned. Default to true From 4f35b02ce7defec623f8526d76fca12be0e75eb5 Mon Sep 17 00:00:00 2001 From: Ming Ming Date: Tue, 8 Feb 2022 13:50:08 +0800 Subject: [PATCH 4/4] Update tests --- test/use_case/scan_dir_offline_test.dart | 27 ------------------------ 1 file changed, 27 deletions(-) diff --git a/test/use_case/scan_dir_offline_test.dart b/test/use_case/scan_dir_offline_test.dart index 31625ec2..9adf7685 100644 --- a/test/use_case/scan_dir_offline_test.dart +++ b/test/use_case/scan_dir_offline_test.dart @@ -14,7 +14,6 @@ void main() { test("root", _root); test("subdir", _subDir); test("unsupported file", _unsupportedFile); - test("nomedia", _noMediaDir); }); group("multiple account", () { test("root", _multiAccountRoot); @@ -99,32 +98,6 @@ Future _unsupportedFile() async { ); } -/// Scan nomedia dir -/// -/// Files: admin/test1.jpg, admin/test/test2.jpg, admin/test/.nomedia -/// Scan: admin -/// Expect: admin/test1.jpg -Future _noMediaDir() async { - final account = util.buildAccount(); - final files = (util.FilesBuilder() - ..addJpeg("admin/test1.jpg") - ..addJpeg("admin/test/test2.jpg") - ..addGenericFile("admin/test/.nomedia", "application/octet-stream")) - .build(); - final c = DiContainer( - appDb: await MockAppDb().applyFuture((obj) async { - await util.fillAppDb(obj, account, files); - }), - ); - - expect( - (await ScanDirOffline(c)( - account, File(path: file_util.unstripPath(account, ".")))) - .toSet(), - {files[0]}, - ); -} - /// Scan root dir with multiple accounts /// /// Files: admin/test1.jpg, admin/test/test2.jpg, user1/test1.jpg,