From 763762d3850cfc3e5cf395a01edb549f8862d82a Mon Sep 17 00:00:00 2001 From: Ming Ming Date: Mon, 6 Jun 2022 18:02:54 +0800 Subject: [PATCH] Parse json in isolate --- app/lib/entity/file/data_source.dart | 59 +++++++++++++++----------- app/lib/iterable_extension.dart | 30 +++++++++++++ app/lib/use_case/find_file.dart | 34 +++++++-------- app/lib/use_case/resync_album.dart | 35 +++++++-------- app/lib/use_case/scan_dir_offline.dart | 30 +++++++------ 5 files changed, 115 insertions(+), 73 deletions(-) diff --git a/app/lib/entity/file/data_source.dart b/app/lib/entity/file/data_source.dart index 0b6a9bd1..759cf1dd 100644 --- a/app/lib/entity/file/data_source.dart +++ b/app/lib/entity/file/data_source.dart @@ -13,12 +13,12 @@ import 'package:nc_photos/entity/file_util.dart' as file_util; import 'package:nc_photos/entity/webdav_response_parser.dart'; import 'package:nc_photos/exception.dart'; import 'package:nc_photos/iterable_extension.dart'; -import 'package:nc_photos/k.dart' as k; import 'package:nc_photos/object_extension.dart'; import 'package:nc_photos/or_null.dart'; import 'package:nc_photos/touch_token_manager.dart'; import 'package:nc_photos/use_case/compat/v32.dart'; import 'package:path/path.dart' as path_lib; +import 'package:tuple/tuple.dart'; import 'package:uuid/uuid.dart'; import 'package:xml/xml.dart'; @@ -269,9 +269,9 @@ class FileAppDbDataSource implements FileDataSource { const FileAppDbDataSource(this.appDb); @override - list(Account account, File dir) { + list(Account account, File dir) async { _log.info("[list] ${dir.path}"); - return appDb.use( + final dbItems = await appDb.use( (db) => db.transaction( [AppDb.dirStoreName, AppDb.file2StoreName], idbModeReadOnly), (transaction) async { @@ -284,21 +284,31 @@ class FileAppDbDataSource implements FileDataSource { } final dirEntry = AppDbDirEntry.fromJson(dirItem.cast()); - final entries = await dirEntry.children.mapStream((c) async { - final fileItem = await fileStore - .getObject(AppDbFile2Entry.toPrimaryKey(account, c)) as Map?; - if (fileItem == null) { - _log.warning( - "[list] Missing file ($c) in db for dir: ${logFilename(dir.path)}"); - throw CacheNotFoundException("No entry for dir child: $c"); - } - return AppDbFile2Entry.fromJson(fileItem.cast()); - }, k.simultaneousQuery).toList(); - // we need to add dir to match the remote query - return [dirEntry.dir] + - entries.map((e) => e.file).where((f) => _validateFile(f)).toList(); + return Tuple2( + dirEntry.dir, + await Future.wait( + dirEntry.children.map((c) async { + final fileItem = await fileStore + .getObject(AppDbFile2Entry.toPrimaryKey(account, c)) as Map?; + if (fileItem == null) { + _log.warning( + "[list] Missing file ($c) in db for dir: ${logFilename(dir.path)}"); + throw CacheNotFoundException("No entry for dir child: $c"); + } else { + return fileItem; + } + }), + eagerError: true, + ), + ); }, ); + // we need to add dir to match the remote query + return [ + dbItems.item1, + ...(await dbItems.item2.computeAll(_covertAppDbFile2Entry)) + .where((f) => _validateFile(f)) + ]; } @override @@ -647,20 +657,16 @@ class FileForwardCacheManager { // query other files if (needQuery.isNotEmpty) { - final fileItems = await appDb.use( + final dbItems = await appDb.use( (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly), (transaction) async { final store = transaction.objectStore(AppDb.file2StoreName); - return await needQuery - .mapStream( - (id) => store - .getObject(AppDbFile2Entry.toPrimaryKey(account, id)), - k.simultaneousQuery) - .toList(); + return await Future.wait(needQuery.map((id) => + store.getObject(AppDbFile2Entry.toPrimaryKey(account, id)))); }, ); - files.addAll(fileItems.cast().whereType().map( - (i) => AppDbFile2Entry.fromJson(i.cast()).file)); + files.addAll( + await dbItems.whereType().computeAll(_covertAppDbFile2Entry)); } _fileCache.addEntries(files.map((f) => MapEntry(f.fileId!, f))); _log.info( @@ -692,3 +698,6 @@ bool _validateFile(File f) { // See: https://gitlab.com/nkming2/nc-photos/-/issues/9 return f.lastModified != null; } + +File _covertAppDbFile2Entry(Map json) => + AppDbFile2Entry.fromJson(json.cast()).file; diff --git a/app/lib/iterable_extension.dart b/app/lib/iterable_extension.dart index 38a1e059..0ea473d7 100644 --- a/app/lib/iterable_extension.dart +++ b/app/lib/iterable_extension.dart @@ -114,4 +114,34 @@ extension IterableExtension on Iterable { yield e; } } + + Future> computeAll(ComputeCallback callback) async { + return await compute( + _computeAllImpl, _ComputeAllMessage(callback, asList())); + } + + /// Return a list containing elements in this iterable + /// + /// If this Iterable is itself a list, this will be returned directly with no + /// copying + List asList() { + if (this is List) { + return this as List; + } else { + return toList(); + } + } +} + +class _ComputeAllMessage { + const _ComputeAllMessage(this.callback, this.data); + + final ComputeCallback callback; + final List data; +} + +Future> _computeAllImpl(_ComputeAllMessage message) async { + final result = await Future.wait( + message.data.map((e) async => await message.callback(e))); + return result; } diff --git a/app/lib/use_case/find_file.dart b/app/lib/use_case/find_file.dart index 1d024713..0120c30a 100644 --- a/app/lib/use_case/find_file.dart +++ b/app/lib/use_case/find_file.dart @@ -1,11 +1,9 @@ +import 'package:flutter/foundation.dart'; import 'package:idb_shim/idb_client.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/app_db.dart'; import 'package:nc_photos/di_container.dart'; import 'package:nc_photos/entity/file.dart'; -import 'package:nc_photos/iterable_extension.dart'; -import 'package:nc_photos/k.dart' as k; -import 'package:quiver/iterables.dart'; class FindFile { FindFile(this._c) : assert(require(_c)); @@ -25,27 +23,22 @@ class FindFile { (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly), (transaction) async { final fileStore = transaction.objectStore(AppDb.file2StoreName); - return await fileIds - .mapStream( - (id) => fileStore - .getObject(AppDbFile2Entry.toPrimaryKey(account, id)), - k.simultaneousQuery) - .toList(); + return await Future.wait(fileIds.map((id) => + fileStore.getObject(AppDbFile2Entry.toPrimaryKey(account, id)))); }, ); + final fileMap = await compute(_covertFileMap, dbItems); final files = []; - for (final pair in zip([fileIds, dbItems])) { - final dbItem = pair[1] as Map?; - if (dbItem == null) { + for (final id in fileIds) { + final f = fileMap[id]; + if (f == null) { if (onFileNotFound == null) { - throw StateError("File ID not found: ${pair[0]}"); + throw StateError("File ID not found: $id"); } else { - onFileNotFound(pair[0] as int); + onFileNotFound(id); } } else { - final dbEntry = - AppDbFile2Entry.fromJson(dbItem.cast()); - files.add(dbEntry.file); + files.add(f); } } return files; @@ -53,3 +46,10 @@ class FindFile { final DiContainer _c; } + +Map _covertFileMap(List dbItems) { + return Map.fromEntries(dbItems + .whereType() + .map((j) => AppDbFile2Entry.fromJson(j.cast()).file) + .map((f) => MapEntry(f.fileId!, f))); +} diff --git a/app/lib/use_case/resync_album.dart b/app/lib/use_case/resync_album.dart index 190dfb82..dca7bb3a 100644 --- a/app/lib/use_case/resync_album.dart +++ b/app/lib/use_case/resync_album.dart @@ -1,3 +1,4 @@ +import 'package:flutter/foundation.dart'; import 'package:idb_shim/idb_client.dart'; import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; @@ -6,8 +7,7 @@ import 'package:nc_photos/debug_util.dart'; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/album/item.dart'; import 'package:nc_photos/entity/album/provider.dart'; -import 'package:nc_photos/iterable_extension.dart'; -import 'package:nc_photos/k.dart' as k; +import 'package:nc_photos/entity/file.dart'; /// Resync files inside an album with the file db class ResyncAlbum { @@ -20,31 +20,21 @@ class ResyncAlbum { "Resync only make sense for static albums: ${album.name}"); } final items = AlbumStaticProvider.of(album).items; - final fileIds = - items.whereType().map((i) => i.file.fileId!).toList(); - final dbItems = Map.fromEntries(await appDb.use( + final dbItems = await appDb.use( (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly), (transaction) async { final store = transaction.objectStore(AppDb.file2StoreName); - return await fileIds - .mapStream( - (id) async => MapEntry( - id, - await store.getObject( - AppDbFile2Entry.toPrimaryKey(account, id)) as Map?, - ), - k.simultaneousQuery) - .toList(); + return await Future.wait(items.whereType().map((i) => + store.getObject( + AppDbFile2Entry.toPrimaryKey(account, i.file.fileId!)))); }, - )); + ); + final fileMap = await compute(_covertFileMap, dbItems); return items.map((i) { if (i is AlbumFileItem) { try { - final dbItem = dbItems[i.file.fileId]!; - final dbEntry = - AppDbFile2Entry.fromJson(dbItem.cast()); return i.copyWith( - file: dbEntry.file, + file: fileMap[i.file.fileId]!, ); } catch (e, stackTrace) { _log.shout( @@ -63,3 +53,10 @@ class ResyncAlbum { static final _log = Logger("use_case.resync_album.ResyncAlbum"); } + +Map _covertFileMap(List dbItems) { + return Map.fromEntries(dbItems + .whereType() + .map((j) => AppDbFile2Entry.fromJson(j.cast()).file) + .map((f) => MapEntry(f.fileId!, f))); +} diff --git a/app/lib/use_case/scan_dir_offline.dart b/app/lib/use_case/scan_dir_offline.dart index 9b7da003..dbc3655f 100644 --- a/app/lib/use_case/scan_dir_offline.dart +++ b/app/lib/use_case/scan_dir_offline.dart @@ -4,6 +4,7 @@ import 'package:nc_photos/app_db.dart'; import 'package:nc_photos/di_container.dart'; import 'package:nc_photos/entity/file.dart'; import 'package:nc_photos/entity/file_util.dart' as file_util; +import 'package:nc_photos/iterable_extension.dart'; class ScanDirOffline { ScanDirOffline(this._c) : assert(require(_c)); @@ -11,12 +12,12 @@ class ScanDirOffline { static bool require(DiContainer c) => DiContainer.has(c, DiType.appDb); /// List all files under a dir recursively from the local DB - Future> call( + Future> call( Account account, File root, { bool isOnlySupportedFormat = true, }) async { - return await _c.appDb.use( + final dbItems = await _c.appDb.use( (db) => db.transaction(AppDb.file2StoreName, idbModeReadOnly), (transaction) async { final store = transaction.objectStore(AppDb.file2StoreName); @@ -25,20 +26,25 @@ class ScanDirOffline { AppDbFile2Entry.toStrippedPathIndexLowerKeyForDir(account, root), AppDbFile2Entry.toStrippedPathIndexUpperKeyForDir(account, root), ); - final product = []; - await for (final c - in index.openCursor(range: range, autoAdvance: false)) { - final e = AppDbFile2Entry.fromJson( - (c.value as Map).cast()); - if (!isOnlySupportedFormat || file_util.isSupportedFormat(e.file)) { - product.add(e.file); - } + return await index + .openCursor(range: range, autoAdvance: false) + .map((c) { + final v = c.value as Map; c.next(); - } - return product; + return v; + }).toList(); }, ); + final results = await dbItems.computeAll(_covertAppDbFile2Entry); + if (isOnlySupportedFormat) { + return results.where((f) => file_util.isSupportedFormat(f)); + } else { + return results; + } } final DiContainer _c; } + +File _covertAppDbFile2Entry(Map json) => + AppDbFile2Entry.fromJson(json.cast()).file;