Store each files in db

lib/app_db.dart

@@ -12,10 +12,13 @@ import 'package:synchronized/synchronized.dart';
 
 class AppDb {
   static const dbName = "app.db";
-  static const dbVersion = 2;
+  static const dbVersion = 3;
   static const fileStoreName = "files";
   static const albumStoreName = "albums";
 
+  /// this is a stupid name but 'files' is already being used so...
+  static const fileDbStoreName = "filesDb";
+
   /// Run [fn] with an opened database instance
   ///
   /// This function guarantees that:
@@ -41,7 +44,7 @@ class AppDb {
       _log.info("[_open] Upgrade database: ${event.oldVersion} -> $dbVersion");
 
       final db = event.database;
-      ObjectStore fileStore, albumStore;
+      ObjectStore fileStore, albumStore, fileDbStore;
       if (event.oldVersion < 1) {
         fileStore = db.createObjectStore(fileStoreName);
         albumStore = db.createObjectStore(albumStoreName);
@@ -57,6 +60,20 @@ class AppDb {
         albumStore.createIndex(
             AppDbAlbumEntry.indexName, AppDbAlbumEntry.keyPath);
       }
+      if (event.oldVersion < 3) {
+        // new object store in v3
+        fileDbStore = db.createObjectStore(fileDbStoreName);
+        fileDbStore.createIndex(
+            AppDbFileDbEntry.indexName, AppDbFileDbEntry.keyPath,
+            unique: false);
+
+        // drop files
+        // ObjectStore.clear is bugged when there's index created on Android
+        final cursor = fileStore.openKeyCursor(autoAdvance: true);
+        await for (final k in cursor) {
+          await fileStore.delete(k.primaryKey);
+        }
+      }
     });
   }
 
@@ -140,3 +157,38 @@ class AppDbAlbumEntry {
   // properties other than Album.items is undefined when index > 0
   final Album album;
 }
+
+class AppDbFileDbEntry {
+  static const indexName = "fileDbStore_namespacedFileId";
+  static const keyPath = "namespacedFileId";
+
+  AppDbFileDbEntry(this.namespacedFileId, this.file);
+
+  factory AppDbFileDbEntry.fromFile(Account account, File file) {
+    return AppDbFileDbEntry(toNamespacedFileId(account, file), file);
+  }
+
+  Map<String, dynamic> toJson() {
+    return {
+      "namespacedFileId": namespacedFileId,
+      "file": file.toJson(),
+    };
+  }
+
+  factory AppDbFileDbEntry.fromJson(Map<String, dynamic> json) {
+    return AppDbFileDbEntry(
+      json["namespacedFileId"],
+      File.fromJson(json["file"].cast<String, dynamic>()),
+    );
+  }
+
+  /// File ID namespaced by the server URL
+  final String namespacedFileId;
+  final File file;
+
+  static String toPrimaryKey(Account account, File file) =>
+      "${account.url}/${file.path}";
+
+  static String toNamespacedFileId(Account account, File file) =>
+      "${account.url}/${file.fileId}";
+}

lib/entity/file/data_source.dart

@@ -254,10 +254,13 @@ class FileAppDbDataSource implements FileDataSource {
   }) {
     _log.info("[updateProperty] ${f.path}");
     return AppDb.use((db) async {
-      final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
+      final transaction = db.transaction(
-      final store = transaction.objectStore(AppDb.fileStoreName);
+          [AppDb.fileStoreName, AppDb.fileDbStoreName], idbModeReadWrite);
+
+      // update file store
+      final fileStore = transaction.objectStore(AppDb.fileStoreName);
       final parentDir = File(path: path.dirname(f.path));
-      final parentList = await _doList(store, account, parentDir);
+      final parentList = await _doList(fileStore, account, parentDir);
       final jsonList = parentList.map((e) {
         if (e.path == f.path) {
           return e.copyWith(
@@ -268,7 +271,17 @@ class FileAppDbDataSource implements FileDataSource {
           return e;
         }
       });
-      await _cacheListResults(store, account, parentDir, jsonList);
+      await _cacheListResults(fileStore, account, parentDir, jsonList);
+
+      // update file db store
+      final fileDbStore = transaction.objectStore(AppDb.fileDbStoreName);
+      final newFile = f.copyWith(
+        metadata: metadata,
+        isArchived: isArchived,
+      );
+      await fileDbStore.put(
+          AppDbFileDbEntry.fromFile(account, newFile).toJson(),
+          AppDbFileDbEntry.toPrimaryKey(account, newFile));
     });
   }
 
@@ -351,12 +364,21 @@ class FileCachedDataSource implements FileDataSource {
       }
 
       if (cache != null) {
-        try {
+        _syncCacheWithRemote(account, remote, cache);
-          await _cleanUpCachedDir(account, remote, cache);
+      } else {
-        } catch (e, stacktrace) {
+        AppDb.use((db) async {
-          _log.shout("[list] Failed while _cleanUpCachedList", e, stacktrace);
+          final transaction =
-          // ignore error
+              db.transaction(AppDb.fileDbStoreName, idbModeReadWrite);
-        }
+          final fileDbStore = transaction.objectStore(AppDb.fileDbStoreName);
+          for (final f in remote) {
+            try {
+              await _upsertFileDbStoreCache(account, f, fileDbStore);
+            } catch (e, stacktrace) {
+              _log.shout(
+                  "[list] Failed while _upsertFileDbStoreCache", e, stacktrace);
+            }
+          }
+        });
       }
       return remote;
     } on ApiException catch (e) {
@@ -453,47 +475,103 @@ class FileCachedDataSource implements FileDataSource {
     });
   }
 
-  /// Remove dangling dir entries in the file object store
+  /// Sync the remote result and local cache
-  Future<void> _cleanUpCachedDir(
+  void _syncCacheWithRemote(
-      Account account, List<File> remoteResults, List<File> cachedResults) {
+      Account account, List<File> remote, List<File> cache) async {
-    final removed = cachedResults
+    final removed =
-        .where((cache) =>
+        cache.where((c) => !remote.any((r) => r.path == c.path)).toList();
-            !remoteResults.any((remote) => remote.path == cache.path))
+    _log.info(
-        .toList();
+        "[_syncCacheWithRemote] Removed: ${removed.map((f) => f.path).toReadableString()}");
-    if (removed.isEmpty) {
-      return Future.delayed(Duration.zero);
-    }
-    return AppDb.use((db) async {
-      final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
-      final store = transaction.objectStore(AppDb.fileStoreName);
-      final index = store.index(AppDbFileEntry.indexName);
-      for (final r in removed) {
-        final path = AppDbFileEntry.toPath(account, r);
-        final keys = [];
-        // delete the dir itself
-        final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]);
-        // delete with KeyRange is not supported in idb_shim/idb_sqflite
-        // await store.delete(dirRange);
-        keys.addAll(await index
-            .openKeyCursor(range: dirRange, autoAdvance: true)
-            .map((cursor) => cursor.primaryKey)
-            .toList());
-        // then its children
-        final childrenRange =
-            KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]);
-        keys.addAll(await index
-            .openKeyCursor(range: childrenRange, autoAdvance: true)
-            .map((cursor) => cursor.primaryKey)
-            .toList());
 
-        for (final k in keys) {
+    AppDb.use((db) async {
-          _log.fine("[_cleanUpCachedDir] Removing DB entry: $k");
+      final transaction = db.transaction(
-          await store.delete(k);
+          [AppDb.fileStoreName, AppDb.fileDbStoreName], idbModeReadWrite);
+      final fileStore = transaction.objectStore(AppDb.fileStoreName);
+      final fileStoreIndex = fileStore.index(AppDbFileEntry.indexName);
+      final fileDbStore = transaction.objectStore(AppDb.fileDbStoreName);
+      for (final f in removed) {
+        try {
+          await _removeFileDbStoreCache(account, f, fileDbStore);
+        } catch (e, stacktrace) {
+          _log.shout(
+              "[_syncCacheWithRemote] Failed while _removeFileDbStoreCache",
+              e,
+              stacktrace);
+        }
+        try {
+          await _removeFileStoreCache(account, f, fileStore, fileStoreIndex);
+        } catch (e, stacktrace) {
+          _log.shout(
+              "[_syncCacheWithRemote] Failed while _removeFileStoreCache",
+              e,
+              stacktrace);
+        }
+      }
+      for (final f in remote) {
+        try {
+          await _upsertFileDbStoreCache(account, f, fileDbStore);
+        } catch (e, stacktrace) {
+          _log.shout(
+              "[_syncCacheWithRemote] Failed while _upsertFileDbStoreCache",
+              e,
+              stacktrace);
         }
       }
     });
   }
 
+  Future<void> _removeFileDbStoreCache(
+      Account account, File file, ObjectStore objStore) async {
+    if (file.isCollection == true) {
+      final fullPath = AppDbFileDbEntry.toPrimaryKey(account, file);
+      final range = KeyRange.bound("$fullPath/", "$fullPath/\uffff");
+      await for (final k
+          in objStore.openKeyCursor(range: range, autoAdvance: true)) {
+        _log.fine(
+            "[_removeFileDbStoreCache] Removing DB entry: ${k.primaryKey}");
+        objStore.delete(k.primaryKey);
+      }
+    } else {
+      await objStore.delete(AppDbFileDbEntry.toPrimaryKey(account, file));
+    }
+  }
+
+  Future<void> _upsertFileDbStoreCache(
+      Account account, File file, ObjectStore objStore) async {
+    if (file.isCollection == true) {
+      return;
+    }
+    await objStore.put(AppDbFileDbEntry.fromFile(account, file).toJson(),
+        AppDbFileDbEntry.toPrimaryKey(account, file));
+  }
+
+  /// Remove dangling dir entries in the file object store
+  Future<void> _removeFileStoreCache(
+      Account account, File file, ObjectStore objStore, Index index) async {
+    if (file.isCollection != true) {
+      return;
+    }
+
+    final path = AppDbFileEntry.toPath(account, file);
+    // delete the dir itself
+    final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]);
+    // delete with KeyRange is not supported in idb_shim/idb_sqflite
+    // await store.delete(dirRange);
+    await for (final k
+        in index.openKeyCursor(range: dirRange, autoAdvance: true)) {
+      _log.fine("[_removeFileStoreCache] Removing DB entry: ${k.primaryKey}");
+      objStore.delete(k.primaryKey);
+    }
+    // then its children
+    final childrenRange =
+        KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]);
+    await for (final k
+        in index.openKeyCursor(range: childrenRange, autoAdvance: true)) {
+      _log.fine("[_removeFileStoreCache] Removing DB entry: ${k.primaryKey}");
+      objStore.delete(k.primaryKey);
+    }
+  }
+
   final bool shouldCheckCache;
 
   final _remoteSrc = FileWebdavDataSource();