Use more cache during the inital query

app/lib/bloc/scan_account_dir.dart

@@ -1,6 +1,7 @@
 import 'dart:async';
 
 import 'package:bloc/bloc.dart';
+import 'package:collection/collection.dart';
 import 'package:kiwi/kiwi.dart';
 import 'package:logging/logging.dart';
 import 'package:nc_photos/account.dart';
@@ -177,20 +178,19 @@ class ScanAccountDirBloc
   Stream<ScanAccountDirBlocState> _onEventQuery(
       ScanAccountDirBlocQueryBase ev) async* {
     yield ScanAccountDirBlocLoading(state.files);
-    bool hasContent = state.files.isNotEmpty;
+    final hasContent = state.files.isNotEmpty;
 
-    List<File> cacheFiles = [];
+    final stopwatch = Stopwatch()..start();
+    final cacheFiles = await _queryOffline(ev);
+    _log.info(
+        "[_onEventQuery] Elapsed time (_queryOffline): ${stopwatch.elapsedMilliseconds}ms");
     if (!hasContent) {
       // show something instantly on first load
-      final stopwatch = Stopwatch()..start();
-      cacheFiles = await _queryOffline(ev);
-      _log.info(
-          "[_onEventQuery] Elapsed time (_queryOffline): ${stopwatch.elapsedMilliseconds}ms");
-      yield ScanAccountDirBlocLoading(cacheFiles);
-      hasContent = cacheFiles.isNotEmpty;
+      yield ScanAccountDirBlocLoading(
+          cacheFiles.where((f) => file_util.isSupportedFormat(f)).toList());
     }
 
-    yield* _queryOnline(ev, hasContent ? cacheFiles : null);
+    yield* _queryOnline(ev, cacheFiles);
   }
 
   Stream<ScanAccountDirBlocState> _onExternalEvent(
@@ -323,7 +323,8 @@ class ScanAccountDirBloc
     for (final r in account.roots) {
       try {
         final dir = File(path: file_util.unstripPath(account, r));
-        files.addAll(await ScanDirOffline(c)(account, dir));
+        files.addAll(await ScanDirOffline(c)(account, dir,
+            isOnlySupportedFormat: false));
       } catch (e, stackTrace) {
         _log.shout(
             "[_queryOffline] Failed while ScanDirOffline: ${logFilename(r)}",
@@ -335,24 +336,29 @@ class ScanAccountDirBloc
   }
 
   Stream<ScanAccountDirBlocState> _queryOnline(
-      ScanAccountDirBlocQueryBase ev, List<File>? cache) async* {
+      ScanAccountDirBlocQueryBase ev, List<File> cache) async* {
     // 1st pass: scan for new files
     var files = <File>[];
+    final cacheMap = FileForwardCacheManager.prepareFileMap(cache);
     {
       final stopwatch = Stopwatch()..start();
       final fileRepo = FileRepo(FileCachedDataSource(AppDb(),
-          forwardCacheManager: FileForwardCacheManager(AppDb())));
+          forwardCacheManager: FileForwardCacheManager(AppDb(), cacheMap)));
       final fileRepoNoCache = FileRepo(FileCachedDataSource(AppDb()));
       await for (final event in _queryWithFileRepo(fileRepo, ev,
           fileRepoForShareDir: fileRepoNoCache)) {
         if (event is ExceptionEvent) {
           _log.shout("[_queryOnline] Exception while request (1st pass)",
               event.error, event.stackTrace);
-          yield ScanAccountDirBlocFailure(cache ?? files, event.error);
+          yield ScanAccountDirBlocFailure(
+              cache.isEmpty
+                  ? files
+                  : cache.where((f) => file_util.isSupportedFormat(f)).toList(),
+              event.error);
           return;
         }
         files.addAll(event);
-        if (cache == null) {
+        if (cache.isEmpty) {
           // only emit partial results if there's no cache
           yield ScanAccountDirBlocLoading(files);
         }
@@ -367,13 +373,13 @@ class ScanAccountDirBloc
         _shouldCheckCache = false;
 
         // announce the result of the 1st pass
-        // if cache == null, we have already emitted the results in the loop
-        if (cache != null || files.isEmpty) {
+        // if cache is empty, we have already emitted the results in the loop
+        if (cache.isNotEmpty || files.isEmpty) {
           // emit results from remote
           yield ScanAccountDirBlocLoading(files);
         }
 
-        files = await _queryOnlinePass2(ev, files);
+        files = await _queryOnlinePass2(ev, cacheMap, files);
       }
     } catch (e, stackTrace) {
       _log.shout(
@@ -382,12 +388,16 @@ class ScanAccountDirBloc
     yield ScanAccountDirBlocSuccess(files);
   }
 
-  Future<List<File>> _queryOnlinePass2(
-      ScanAccountDirBlocQueryBase ev, List<File> pass1Files) async {
+  Future<List<File>> _queryOnlinePass2(ScanAccountDirBlocQueryBase ev,
+      Map<int, File> cacheMap, List<File> pass1Files) async {
     const touchTokenManager = TouchTokenManager();
+    // combine the file maps because [pass1Files] doesn't contain non-supported
+    // files
+    final pass2CacheMap = CombinedMapView(
+        [FileForwardCacheManager.prepareFileMap(pass1Files), cacheMap]);
     final fileRepo = FileRepo(FileCachedDataSource(AppDb(),
         shouldCheckCache: true,
-        forwardCacheManager: FileForwardCacheManager(AppDb())));
+        forwardCacheManager: FileForwardCacheManager(AppDb(), pass2CacheMap)));
     final remoteTouchEtag =
         await touchTokenManager.getRemoteRootEtag(fileRepo, account);
     if (remoteTouchEtag == null) {

app/lib/entity/file/data_source.dart

@@ -14,6 +14,7 @@ import 'package:nc_photos/entity/webdav_response_parser.dart';
 import 'package:nc_photos/exception.dart';
 import 'package:nc_photos/iterable_extension.dart';
 import 'package:nc_photos/k.dart' as k;
+import 'package:nc_photos/object_extension.dart';
 import 'package:nc_photos/or_null.dart';
 import 'package:nc_photos/touch_token_manager.dart';
 import 'package:nc_photos/use_case/compat/v32.dart';
@@ -565,7 +566,14 @@ class FileCachedDataSource implements FileDataSource {
 /// passed to us in one transaction. For this reason, this should only be used
 /// when it's necessary to query everything
 class FileForwardCacheManager {
-  FileForwardCacheManager(this.appDb);
+  FileForwardCacheManager(this.appDb, this.knownFiles);
+
+  /// Transform a list of files to a map suitable to be passed as the
+  /// [knownFiles] argument
+  static Map<int, File> prepareFileMap(List<File> knownFiles) => knownFiles
+      .where((f) => f.fileId != null)
+      .map((e) => MapEntry(e.fileId!, e))
+      .run((obj) => Map.fromEntries(obj));
 
   Future<List<File>> list(Account account, File dir) async {
     // check cache
@@ -618,20 +626,43 @@ class FileForwardCacheManager {
     // cache files
     final fileIds = dirs.map((e) => e.children).fold<List<int>>(
         [], (previousValue, element) => previousValue + element);
-    final fileItems = await appDb.use(
-      (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
-      (transaction) async {
-        final store = transaction.objectStore(AppDb.file2StoreName);
-        return await Future.wait(fileIds.map((id) =>
-            store.getObject(AppDbFile2Entry.toPrimaryKey(account, id))));
-      },
-    );
-    final files = fileItems
-        .cast<Map?>()
-        .whereType<Map>()
-        .map((i) => AppDbFile2Entry.fromJson(i.cast<String, dynamic>()))
-        .toList();
-    _fileCache.addEntries(files.map((e) => MapEntry(e.file.fileId!, e.file)));
+
+    final needQuery = <int>[];
+    final files = <File>[];
+    // check files already known to us
+    if (knownFiles.isNotEmpty) {
+      for (final id in fileIds) {
+        final f = knownFiles[id];
+        if (f != null) {
+          files.add(f);
+        } else {
+          needQuery.add(id);
+        }
+      }
+    } else {
+      needQuery.addAll(fileIds);
+    }
+    _log.info(
+        "[_cacheDir] ${files.length} files known, ${needQuery.length} files need querying");
+
+    // query other files
+    if (needQuery.isNotEmpty) {
+      final fileItems = await appDb.use(
+        (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
+        (transaction) async {
+          final store = transaction.objectStore(AppDb.file2StoreName);
+          return await needQuery
+              .mapStream(
+                  (id) => store
+                      .getObject(AppDbFile2Entry.toPrimaryKey(account, id)),
+                  k.simultaneousQuery)
+              .toList();
+        },
+      );
+      files.addAll(fileItems.cast<Map?>().whereType<Map>().map(
+          (i) => AppDbFile2Entry.fromJson(i.cast<String, dynamic>()).file));
+    }
+    _fileCache.addEntries(files.map((f) => MapEntry(f.fileId!, f)));
     _log.info(
         "[_cacheDir] Cached ${files.length} files under ${logFilename(dir.path)}");
   }
@@ -650,6 +681,7 @@ class FileForwardCacheManager {
   }
 
   final AppDb appDb;
+  final Map<int, File> knownFiles;
   final _dirCache = <String, AppDbDirEntry>{};
   final _fileCache = <int, File>{};
 

app/lib/use_case/scan_dir_offline.dart

@@ -11,7 +11,11 @@ class ScanDirOffline {
   static bool require(DiContainer c) => DiContainer.has(c, DiType.appDb);
 
   /// List all files under a dir recursively from the local DB
-  Future<List<File>> call(Account account, File root) async {
+  Future<List<File>> call(
+    Account account,
+    File root, {
+    bool isOnlySupportedFormat = true,
+  }) async {
     return await _c.appDb.use(
       (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
       (transaction) async {
@@ -26,7 +30,7 @@ class ScanDirOffline {
             in index.openCursor(range: range, autoAdvance: false)) {
           final e = AppDbFile2Entry.fromJson(
               (c.value as Map).cast<String, dynamic>());
-          if (file_util.isSupportedFormat(e.file)) {
+          if (!isOnlySupportedFormat || file_util.isSupportedFormat(e.file)) {
             product.add(e.file);
           }
           c.next();