Parse json in isolate

app/lib/entity/file/data_source.dart

@@ -13,12 +13,12 @@ import 'package:nc_photos/entity/file_util.dart' as file_util;
 import 'package:nc_photos/entity/webdav_response_parser.dart';
 import 'package:nc_photos/exception.dart';
 import 'package:nc_photos/iterable_extension.dart';
-import 'package:nc_photos/k.dart' as k;
 import 'package:nc_photos/object_extension.dart';
 import 'package:nc_photos/or_null.dart';
 import 'package:nc_photos/touch_token_manager.dart';
 import 'package:nc_photos/use_case/compat/v32.dart';
 import 'package:path/path.dart' as path_lib;
+import 'package:tuple/tuple.dart';
 import 'package:uuid/uuid.dart';
 import 'package:xml/xml.dart';
 
@@ -269,9 +269,9 @@ class FileAppDbDataSource implements FileDataSource {
   const FileAppDbDataSource(this.appDb);
 
   @override
-  list(Account account, File dir) {
+  list(Account account, File dir) async {
     _log.info("[list] ${dir.path}");
-    return appDb.use(
+    final dbItems = await appDb.use(
       (db) => db.transaction(
           [AppDb.dirStoreName, AppDb.file2StoreName], idbModeReadOnly),
       (transaction) async {
@@ -284,21 +284,31 @@ class FileAppDbDataSource implements FileDataSource {
         }
         final dirEntry =
             AppDbDirEntry.fromJson(dirItem.cast<String, dynamic>());
-        final entries = await dirEntry.children.mapStream((c) async {
+        return Tuple2(
+          dirEntry.dir,
+          await Future.wait(
+            dirEntry.children.map((c) async {
               final fileItem = await fileStore
                   .getObject(AppDbFile2Entry.toPrimaryKey(account, c)) as Map?;
               if (fileItem == null) {
                 _log.warning(
                     "[list] Missing file ($c) in db for dir: ${logFilename(dir.path)}");
                 throw CacheNotFoundException("No entry for dir child: $c");
+              } else {
+                return fileItem;
               }
-          return AppDbFile2Entry.fromJson(fileItem.cast<String, dynamic>());
+            }),
-        }, k.simultaneousQuery).toList();
+            eagerError: true,
-        // we need to add dir to match the remote query
+          ),
-        return [dirEntry.dir] +
+        );
-            entries.map((e) => e.file).where((f) => _validateFile(f)).toList();
       },
     );
+    // we need to add dir to match the remote query
+    return [
+      dbItems.item1,
+      ...(await dbItems.item2.computeAll(_covertAppDbFile2Entry))
+          .where((f) => _validateFile(f))
+    ];
   }
 
   @override
@@ -647,20 +657,16 @@ class FileForwardCacheManager {
 
     // query other files
     if (needQuery.isNotEmpty) {
-      final fileItems = await appDb.use(
+      final dbItems = await appDb.use(
         (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
         (transaction) async {
           final store = transaction.objectStore(AppDb.file2StoreName);
-          return await needQuery
+          return await Future.wait(needQuery.map((id) =>
-              .mapStream(
+              store.getObject(AppDbFile2Entry.toPrimaryKey(account, id))));
-                  (id) => store
-                      .getObject(AppDbFile2Entry.toPrimaryKey(account, id)),
-                  k.simultaneousQuery)
-              .toList();
         },
       );
-      files.addAll(fileItems.cast<Map?>().whereType<Map>().map(
+      files.addAll(
-          (i) => AppDbFile2Entry.fromJson(i.cast<String, dynamic>()).file));
+          await dbItems.whereType<Map>().computeAll(_covertAppDbFile2Entry));
     }
     _fileCache.addEntries(files.map((f) => MapEntry(f.fileId!, f)));
     _log.info(
@@ -692,3 +698,6 @@ bool _validateFile(File f) {
   // See: https://gitlab.com/nkming2/nc-photos/-/issues/9
   return f.lastModified != null;
 }
+
+File _covertAppDbFile2Entry(Map json) =>
+    AppDbFile2Entry.fromJson(json.cast<String, dynamic>()).file;

app/lib/iterable_extension.dart

@@ -114,4 +114,34 @@ extension IterableExtension<T> on Iterable<T> {
       yield e;
     }
   }
+
+  Future<List<U>> computeAll<U>(ComputeCallback<T, U> callback) async {
+    return await compute(
+        _computeAllImpl<T, U>, _ComputeAllMessage(callback, asList()));
+  }
+
+  /// Return a list containing elements in this iterable
+  ///
+  /// If this Iterable is itself a list, this will be returned directly with no
+  /// copying
+  List<T> asList() {
+    if (this is List) {
+      return this as List<T>;
+    } else {
+      return toList();
+    }
+  }
+}
+
+class _ComputeAllMessage<T, U> {
+  const _ComputeAllMessage(this.callback, this.data);
+
+  final ComputeCallback<T, U> callback;
+  final List<T> data;
+}
+
+Future<List<U>> _computeAllImpl<T, U>(_ComputeAllMessage<T, U> message) async {
+  final result = await Future.wait(
+      message.data.map((e) async => await message.callback(e)));
+  return result;
 }

app/lib/use_case/find_file.dart

@@ -1,11 +1,9 @@
+import 'package:flutter/foundation.dart';
 import 'package:idb_shim/idb_client.dart';
 import 'package:nc_photos/account.dart';
 import 'package:nc_photos/app_db.dart';
 import 'package:nc_photos/di_container.dart';
 import 'package:nc_photos/entity/file.dart';
-import 'package:nc_photos/iterable_extension.dart';
-import 'package:nc_photos/k.dart' as k;
-import 'package:quiver/iterables.dart';
 
 class FindFile {
   FindFile(this._c) : assert(require(_c));
@@ -25,27 +23,22 @@ class FindFile {
       (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
       (transaction) async {
         final fileStore = transaction.objectStore(AppDb.file2StoreName);
-        return await fileIds
+        return await Future.wait(fileIds.map((id) =>
-            .mapStream(
+            fileStore.getObject(AppDbFile2Entry.toPrimaryKey(account, id))));
-                (id) => fileStore
-                    .getObject(AppDbFile2Entry.toPrimaryKey(account, id)),
-                k.simultaneousQuery)
-            .toList();
       },
     );
+    final fileMap = await compute(_covertFileMap, dbItems);
     final files = <File>[];
-    for (final pair in zip([fileIds, dbItems])) {
+    for (final id in fileIds) {
-      final dbItem = pair[1] as Map?;
+      final f = fileMap[id];
-      if (dbItem == null) {
+      if (f == null) {
         if (onFileNotFound == null) {
-          throw StateError("File ID not found: ${pair[0]}");
+          throw StateError("File ID not found: $id");
         } else {
-          onFileNotFound(pair[0] as int);
+          onFileNotFound(id);
         }
       } else {
-        final dbEntry =
+        files.add(f);
-            AppDbFile2Entry.fromJson(dbItem.cast<String, dynamic>());
-        files.add(dbEntry.file);
       }
     }
     return files;
@@ -53,3 +46,10 @@ class FindFile {
 
   final DiContainer _c;
 }
+
+Map<int, File> _covertFileMap(List<Object?> dbItems) {
+  return Map.fromEntries(dbItems
+      .whereType<Map>()
+      .map((j) => AppDbFile2Entry.fromJson(j.cast<String, dynamic>()).file)
+      .map((f) => MapEntry(f.fileId!, f)));
+}

app/lib/use_case/resync_album.dart

@@ -1,3 +1,4 @@
+import 'package:flutter/foundation.dart';
 import 'package:idb_shim/idb_client.dart';
 import 'package:logging/logging.dart';
 import 'package:nc_photos/account.dart';
@@ -6,8 +7,7 @@ import 'package:nc_photos/debug_util.dart';
 import 'package:nc_photos/entity/album.dart';
 import 'package:nc_photos/entity/album/item.dart';
 import 'package:nc_photos/entity/album/provider.dart';
-import 'package:nc_photos/iterable_extension.dart';
+import 'package:nc_photos/entity/file.dart';
-import 'package:nc_photos/k.dart' as k;
 
 /// Resync files inside an album with the file db
 class ResyncAlbum {
@@ -20,31 +20,21 @@ class ResyncAlbum {
           "Resync only make sense for static albums: ${album.name}");
     }
     final items = AlbumStaticProvider.of(album).items;
-    final fileIds =
+    final dbItems = await appDb.use(
-        items.whereType<AlbumFileItem>().map((i) => i.file.fileId!).toList();
-    final dbItems = Map.fromEntries(await appDb.use(
       (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
       (transaction) async {
         final store = transaction.objectStore(AppDb.file2StoreName);
-        return await fileIds
+        return await Future.wait(items.whereType<AlbumFileItem>().map((i) =>
-            .mapStream(
+            store.getObject(
-                (id) async => MapEntry(
+                AppDbFile2Entry.toPrimaryKey(account, i.file.fileId!))));
-                      id,
-                      await store.getObject(
-                          AppDbFile2Entry.toPrimaryKey(account, id)) as Map?,
-                    ),
-                k.simultaneousQuery)
-            .toList();
       },
-    ));
+    );
+    final fileMap = await compute(_covertFileMap, dbItems);
     return items.map((i) {
       if (i is AlbumFileItem) {
         try {
-          final dbItem = dbItems[i.file.fileId]!;
-          final dbEntry =
-              AppDbFile2Entry.fromJson(dbItem.cast<String, dynamic>());
           return i.copyWith(
-            file: dbEntry.file,
+            file: fileMap[i.file.fileId]!,
           );
         } catch (e, stackTrace) {
           _log.shout(
@@ -63,3 +53,10 @@ class ResyncAlbum {
 
   static final _log = Logger("use_case.resync_album.ResyncAlbum");
 }
+
+Map<int, File> _covertFileMap(List<Object?> dbItems) {
+  return Map.fromEntries(dbItems
+      .whereType<Map>()
+      .map((j) => AppDbFile2Entry.fromJson(j.cast<String, dynamic>()).file)
+      .map((f) => MapEntry(f.fileId!, f)));
+}

app/lib/use_case/scan_dir_offline.dart

@@ -4,6 +4,7 @@ import 'package:nc_photos/app_db.dart';
 import 'package:nc_photos/di_container.dart';
 import 'package:nc_photos/entity/file.dart';
 import 'package:nc_photos/entity/file_util.dart' as file_util;
+import 'package:nc_photos/iterable_extension.dart';
 
 class ScanDirOffline {
   ScanDirOffline(this._c) : assert(require(_c));
@@ -11,12 +12,12 @@ class ScanDirOffline {
   static bool require(DiContainer c) => DiContainer.has(c, DiType.appDb);
 
   /// List all files under a dir recursively from the local DB
-  Future<List<File>> call(
+  Future<Iterable<File>> call(
     Account account,
     File root, {
     bool isOnlySupportedFormat = true,
   }) async {
-    return await _c.appDb.use(
+    final dbItems = await _c.appDb.use(
       (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly),
       (transaction) async {
         final store = transaction.objectStore(AppDb.file2StoreName);
@@ -25,20 +26,25 @@ class ScanDirOffline {
           AppDbFile2Entry.toStrippedPathIndexLowerKeyForDir(account, root),
           AppDbFile2Entry.toStrippedPathIndexUpperKeyForDir(account, root),
         );
-        final product = <File>[];
+        return await index
-        await for (final c
+            .openCursor(range: range, autoAdvance: false)
-            in index.openCursor(range: range, autoAdvance: false)) {
+            .map((c) {
-          final e = AppDbFile2Entry.fromJson(
+          final v = c.value as Map;
-              (c.value as Map).cast<String, dynamic>());
-          if (!isOnlySupportedFormat || file_util.isSupportedFormat(e.file)) {
-            product.add(e.file);
-          }
           c.next();
-        }
+          return v;
-        return product;
+        }).toList();
       },
     );
+    final results = await dbItems.computeAll(_covertAppDbFile2Entry);
+    if (isOnlySupportedFormat) {
+      return results.where((f) => file_util.isSupportedFormat(f));
+    } else {
+      return results;
+    }
   }
 
   final DiContainer _c;
 }
+
+File _covertAppDbFile2Entry(Map json) =>
+    AppDbFile2Entry.fromJson(json.cast<String, dynamic>()).file;