From 0e4ecf4f2b02de77a23424c596841ece1d968610 Mon Sep 17 00:00:00 2001 From: Ming Ming Date: Thu, 21 Nov 2024 01:34:24 +0800 Subject: [PATCH] Rewrite metadata service, add support for server side exif --- app/lib/controller/files_controller.dart | 2 +- app/lib/controller/metadata_controller.dart | 2 +- app/lib/entity/file/data_source.dart | 4 +- app/lib/entity/file/file_cache_manager.dart | 9 +- app/lib/event/event.dart | 21 +- app/lib/metadata_task_manager.dart | 134 ------- app/lib/metadata_task_manager.g.dart | 32 -- app/lib/service.dart | 358 ------------------ app/lib/service/config.dart | 15 + app/lib/service/l10n.dart | 43 +++ app/lib/service/service.dart | 192 ++++++++++ app/lib/{ => service}/service.g.dart | 18 +- app/lib/use_case/battery_ensurer.dart | 35 ++ app/lib/use_case/scan_missing_metadata.dart | 50 --- .../use_case/sync_metadata/sync_by_app.dart | 120 ++++++ .../sync_metadata/sync_by_server.dart | 91 +++++ .../use_case/sync_metadata/sync_metadata.dart | 109 ++++++ .../sync_metadata/sync_metadata.g.dart | 30 ++ app/lib/use_case/update_missing_metadata.dart | 173 --------- .../use_case/update_missing_metadata.g.dart | 15 - app/lib/use_case/update_property.dart | 57 +-- app/lib/use_case/wifi_ensurer.dart | 41 ++ app/lib/widget/home_photos/bloc.dart | 6 +- .../widget/settings/metadata_settings.dart | 2 +- app/lib/widget/viewer_detail_pane.dart | 2 +- .../entity/file/file_cache_manager_test.dart | 28 +- np_collection/lib/np_collection.dart | 2 + np_db/lib/src/api.dart | 20 + np_db/lib/src/api.g.dart | 32 ++ .../lib/src/database/file_extension.dart | 65 +++- np_db_sqlite/lib/src/sqlite_api.dart | 28 ++ 31 files changed, 865 insertions(+), 871 deletions(-) delete mode 100644 app/lib/metadata_task_manager.dart delete mode 100644 app/lib/metadata_task_manager.g.dart delete mode 100644 app/lib/service.dart create mode 100644 app/lib/service/config.dart create mode 100644 app/lib/service/l10n.dart create mode 100644 app/lib/service/service.dart rename app/lib/{ => service}/service.g.dart (71%) create mode 100644 app/lib/use_case/battery_ensurer.dart delete mode 100644 app/lib/use_case/scan_missing_metadata.dart create mode 100644 app/lib/use_case/sync_metadata/sync_by_app.dart create mode 100644 app/lib/use_case/sync_metadata/sync_by_server.dart create mode 100644 app/lib/use_case/sync_metadata/sync_metadata.dart create mode 100644 app/lib/use_case/sync_metadata/sync_metadata.g.dart delete mode 100644 app/lib/use_case/update_missing_metadata.dart delete mode 100644 app/lib/use_case/update_missing_metadata.g.dart create mode 100644 app/lib/use_case/wifi_ensurer.dart diff --git a/app/lib/controller/files_controller.dart b/app/lib/controller/files_controller.dart index 56b46aaf..c29867c6 100644 --- a/app/lib/controller/files_controller.dart +++ b/app/lib/controller/files_controller.dart @@ -253,7 +253,7 @@ class FilesController { final failures = []; for (final f in files) { try { - await UpdateProperty(_c)( + await UpdateProperty(fileRepo: _c.fileRepo2)( account, f, metadata: metadata, diff --git a/app/lib/controller/metadata_controller.dart b/app/lib/controller/metadata_controller.dart index 03d452b4..097c64ea 100644 --- a/app/lib/controller/metadata_controller.dart +++ b/app/lib/controller/metadata_controller.dart @@ -6,7 +6,7 @@ import 'package:nc_photos/controller/pref_controller.dart'; import 'package:nc_photos/db/entity_converter.dart'; import 'package:nc_photos/di_container.dart'; import 'package:nc_photos/entity/file_util.dart' as file_util; -import 'package:nc_photos/service.dart' as service; +import 'package:nc_photos/service/service.dart' as service; import 'package:np_codegen/np_codegen.dart'; part 'metadata_controller.g.dart'; diff --git a/app/lib/entity/file/data_source.dart b/app/lib/entity/file/data_source.dart index 08df13e6..5465d2bd 100644 --- a/app/lib/entity/file/data_source.dart +++ b/app/lib/entity/file/data_source.dart @@ -635,7 +635,7 @@ class FileCachedDataSource implements FileDataSource { return state.files; } - await FileSqliteCacheUpdater(_c)(state.account, state.dir, + await FileSqliteCacheUpdater(_c.npDb)(state.account, state.dir, remote: state.files); if (shouldCheckCache) { // update our local touch token to match the remote one @@ -657,7 +657,7 @@ class FileCachedDataSource implements FileDataSource { if (remote.isCollection != true) { // only update regular files _log.info("[listSingle] Cache single file: ${logFilename(f.path)}"); - await FileSqliteCacheUpdater(_c).updateSingle(account, remote); + await FileSqliteCacheUpdater(_c.npDb).updateSingle(account, remote); } return remote; } diff --git a/app/lib/entity/file/file_cache_manager.dart b/app/lib/entity/file/file_cache_manager.dart index 60475043..3e464faf 100644 --- a/app/lib/entity/file/file_cache_manager.dart +++ b/app/lib/entity/file/file_cache_manager.dart @@ -8,6 +8,7 @@ import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/entity/file_descriptor.dart'; import 'package:nc_photos/exception.dart'; import 'package:np_codegen/np_codegen.dart'; +import 'package:np_db/np_db.dart'; part 'file_cache_manager.g.dart'; @@ -81,7 +82,7 @@ class FileCacheLoader { @npLog class FileSqliteCacheUpdater { - const FileSqliteCacheUpdater(this._c); + const FileSqliteCacheUpdater(this.db); Future call( Account account, @@ -90,7 +91,7 @@ class FileSqliteCacheUpdater { }) async { final s = Stopwatch()..start(); try { - await _c.npDb.syncDirFiles( + await db.syncDirFiles( account: account.toDb(), dirFile: dir.toDbKey(), files: remote.map((e) => e.toDb()).toList(), @@ -101,13 +102,13 @@ class FileSqliteCacheUpdater { } Future updateSingle(Account account, File remoteFile) async { - await _c.npDb.syncFile( + await db.syncFile( account: account.toDb(), file: remoteFile.toDb(), ); } - final DiContainer _c; + final NpDb db; } class FileSqliteCacheEmptier { diff --git a/app/lib/event/event.dart b/app/lib/event/event.dart index 126bb1c7..be29eea8 100644 --- a/app/lib/event/event.dart +++ b/app/lib/event/event.dart @@ -46,6 +46,7 @@ class AccountPrefUpdatedEvent { final dynamic value; } +@Deprecated("not fired anymore, to be removed") class FilePropertyUpdatedEvent { FilePropertyUpdatedEvent(this.account, this.file, this.properties); @@ -97,26 +98,6 @@ class FavoriteResyncedEvent { final Account account; } -enum MetadataTaskState { - /// No work is being done - idle, - - /// Processing images - prcoessing, - - /// Paused on data network - waitingForWifi, - - /// Paused on low battery - lowBattery, -} - -class MetadataTaskStateChangedEvent { - const MetadataTaskStateChangedEvent(this.state); - - final MetadataTaskState state; -} - @Deprecated("not fired anymore, to be removed") class PrefUpdatedEvent { PrefUpdatedEvent(this.key, this.value); diff --git a/app/lib/metadata_task_manager.dart b/app/lib/metadata_task_manager.dart deleted file mode 100644 index ed87305f..00000000 --- a/app/lib/metadata_task_manager.dart +++ /dev/null @@ -1,134 +0,0 @@ -import 'dart:async'; - -import 'package:event_bus/event_bus.dart'; -import 'package:kiwi/kiwi.dart'; -import 'package:logging/logging.dart'; -import 'package:nc_photos/account.dart'; -import 'package:nc_photos/di_container.dart'; -import 'package:nc_photos/entity/file.dart'; -import 'package:nc_photos/entity/file_util.dart' as file_util; -import 'package:nc_photos/entity/pref.dart'; -import 'package:nc_photos/event/event.dart'; -import 'package:nc_photos/use_case/update_missing_metadata.dart'; -import 'package:np_codegen/np_codegen.dart'; -import 'package:np_geocoder/np_geocoder.dart'; -import 'package:to_string/to_string.dart'; - -part 'metadata_task_manager.g.dart'; - -/// Task to update metadata for missing files -@npLog -@ToString(ignorePrivate: true) -class MetadataTask { - MetadataTask(this._c, this.account, this.pref) : assert(require(_c)); - - static bool require(DiContainer c) => DiContainer.has(c, DiType.fileRepo); - - @override - String toString() => _$toString(); - - Future call() async { - try { - final shareFolder = - File(path: file_util.unstripPath(account, pref.getShareFolderOr())); - bool hasScanShareFolder = false; - final geocoder = ReverseGeocoder(); - await geocoder.init(); - for (final r in account.roots) { - final dir = File(path: file_util.unstripPath(account, r)); - hasScanShareFolder |= file_util.isOrUnderDir(shareFolder, dir); - final op = UpdateMissingMetadata( - _c, const _UpdateMissingMetadataConfigProvider(), geocoder); - await for (final _ in op(account, dir)) { - if (!Pref().isEnableExifOr()) { - _log.info("[call] EXIF disabled, task ending immaturely"); - op.stop(); - return; - } - } - } - if (!hasScanShareFolder) { - final op = UpdateMissingMetadata( - _c, const _UpdateMissingMetadataConfigProvider(), geocoder); - await for (final _ in op( - account, - shareFolder, - isRecursive: false, - filter: (f) => f.ownerId != account.userId, - )) { - if (!Pref().isEnableExifOr()) { - _log.info("[call] EXIF disabled, task ending immaturely"); - op.stop(); - return; - } - } - } - } finally { - KiwiContainer() - .resolve() - .fire(const MetadataTaskStateChangedEvent(MetadataTaskState.idle)); - } - } - - final DiContainer _c; - - final Account account; - @ignore - final AccountPref pref; -} - -/// Manage metadata tasks to run concurrently -@npLog -class MetadataTaskManager { - factory MetadataTaskManager() { - _inst ??= MetadataTaskManager._(); - return _inst!; - } - - MetadataTaskManager._() { - _stateChangedListener.begin(); - _handleStream(); - } - - /// Add a task to the queue - void addTask(MetadataTask task) { - _log.info("[addTask] New task added: $task"); - _streamController.add(task); - } - - MetadataTaskState get state => _currentState; - - void _onMetadataTaskStateChanged(MetadataTaskStateChangedEvent ev) { - if (ev.state != _currentState) { - _currentState = ev.state; - } - } - - Future _handleStream() async { - await for (final task in _streamController.stream) { - if (Pref().isEnableExifOr()) { - _log.info("[_doTask] Executing task: $task"); - await task(); - } else { - _log.info("[_doTask] Ignoring task: $task"); - } - } - } - - final _streamController = StreamController.broadcast(); - - var _currentState = MetadataTaskState.idle; - late final _stateChangedListener = - AppEventListener( - _onMetadataTaskStateChanged); - - static MetadataTaskManager? _inst; -} - -class _UpdateMissingMetadataConfigProvider - implements UpdateMissingMetadataConfigProvider { - const _UpdateMissingMetadataConfigProvider(); - - @override - isWifiOnly() async => Pref().shouldProcessExifWifiOnlyOr(); -} diff --git a/app/lib/metadata_task_manager.g.dart b/app/lib/metadata_task_manager.g.dart deleted file mode 100644 index 65fc62ed..00000000 --- a/app/lib/metadata_task_manager.g.dart +++ /dev/null @@ -1,32 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'metadata_task_manager.dart'; - -// ************************************************************************** -// NpLogGenerator -// ************************************************************************** - -extension _$MetadataTaskNpLog on MetadataTask { - // ignore: unused_element - Logger get _log => log; - - static final log = Logger("metadata_task_manager.MetadataTask"); -} - -extension _$MetadataTaskManagerNpLog on MetadataTaskManager { - // ignore: unused_element - Logger get _log => log; - - static final log = Logger("metadata_task_manager.MetadataTaskManager"); -} - -// ************************************************************************** -// ToStringGenerator -// ************************************************************************** - -extension _$MetadataTaskToString on MetadataTask { - String _$toString() { - // ignore: unnecessary_string_interpolations - return "MetadataTask {account: $account}"; - } -} diff --git a/app/lib/service.dart b/app/lib/service.dart deleted file mode 100644 index d19c03a1..00000000 --- a/app/lib/service.dart +++ /dev/null @@ -1,358 +0,0 @@ -import 'dart:async'; - -import 'package:devicelocale/devicelocale.dart'; -import 'package:flutter/widgets.dart'; -import 'package:flutter_background_service/flutter_background_service.dart'; -import 'package:flutter_gen/gen_l10n/app_localizations.dart'; -import 'package:flutter_gen/gen_l10n/app_localizations_en.dart'; -import 'package:kiwi/kiwi.dart'; -import 'package:logging/logging.dart'; -import 'package:nc_photos/account.dart'; -import 'package:nc_photos/app_init.dart' as app_init; -import 'package:nc_photos/app_localizations.dart'; -import 'package:nc_photos/controller/pref_controller.dart'; -import 'package:nc_photos/di_container.dart'; -import 'package:nc_photos/entity/file.dart'; -import 'package:nc_photos/entity/file/data_source.dart'; -import 'package:nc_photos/entity/file_descriptor.dart'; -import 'package:nc_photos/entity/file_util.dart' as file_util; -import 'package:nc_photos/entity/pref.dart'; -import 'package:nc_photos/event/event.dart'; -import 'package:nc_photos/event/native_event.dart'; -import 'package:nc_photos/language_util.dart' as language_util; -import 'package:nc_photos/use_case/update_missing_metadata.dart'; -import 'package:nc_photos_plugin/nc_photos_plugin.dart'; -import 'package:np_async/np_async.dart'; -import 'package:np_codegen/np_codegen.dart'; -import 'package:np_geocoder/np_geocoder.dart'; -import 'package:np_platform_message_relay/np_platform_message_relay.dart'; - -part 'service.g.dart'; - -/// Start the background service -Future startService() async { - _C.log.info("[startService] Starting service"); - final service = FlutterBackgroundService(); - await service.configure( - androidConfiguration: AndroidConfiguration( - onStart: serviceMain, - autoStart: false, - isForegroundMode: true, - foregroundServiceNotificationTitle: - L10n.global().metadataTaskProcessingNotification, - ), - iosConfiguration: IosConfiguration( - onForeground: () => throw UnimplementedError(), - onBackground: () => throw UnimplementedError(), - ), - ); - // sync settings - await ServiceConfig.setProcessExifWifiOnly( - Pref().shouldProcessExifWifiOnlyOr()); - await service.start(); -} - -/// Ask the background service to stop ASAP -void stopService() { - _C.log.info("[stopService] Stopping service"); - FlutterBackgroundService().sendData({ - _dataKeyEvent: _eventStop, - }); -} - -@visibleForTesting -@pragma("vm:entry-point") -Future serviceMain() async { - _Service._shouldRun.value = true; - WidgetsFlutterBinding.ensureInitialized(); - - await _Service()(); -} - -class ServiceConfig { - static Future setProcessExifWifiOnly(bool flag) async { - await Preference.setBool(_servicePref, _servicePrefProcessWifiOnly, flag); - } -} - -@npLog -class _Service { - Future call() async { - final service = FlutterBackgroundService(); - service.setForegroundMode(true); - - await app_init.init(app_init.InitIsolateType.flutterIsolate); - await _L10n().init(); - - _log.info("[call] Service started"); - final onCancelSubscription = service.onCancel.listen((_) { - _log.info("[call] User canceled"); - _stopSelf(); - }); - final onDataSubscription = - service.onDataReceived.listen((event) => _onReceiveData(event ?? {})); - - try { - await _doWork(); - } catch (e, stackTrace) { - _log.shout("[call] Uncaught exception", e, stackTrace); - } - await onCancelSubscription.cancel(); - await onDataSubscription.cancel(); - await KiwiContainer().resolve().npDb.dispose(); - service.stopBackgroundService(); - _log.info("[call] Service stopped"); - } - - Future _doWork() async { - final prefController = PrefController(Pref()); - final account = prefController.currentAccountValue; - if (account == null) { - _log.shout("[_doWork] account == null"); - return; - } - final accountPref = AccountPref.of(account); - - final service = FlutterBackgroundService(); - final metadataTask = _MetadataTask(service, account, accountPref); - _metadataTaskStateChangedListener.begin(); - try { - await metadataTask(); - } finally { - _metadataTaskStateChangedListener.end(); - } - } - - void _onReceiveData(Map data) { - try { - final event = data[_dataKeyEvent]; - switch (event) { - case _eventStop: - _stopSelf(); - break; - - default: - _log.severe("[_onReceiveData] Unknown event: $event"); - break; - } - } catch (e, stackTrace) { - _log.shout("[_onReceiveData] Uncaught exception", e, stackTrace); - } - } - - void _onMetadataTaskStateChanged(MetadataTaskStateChangedEvent ev) { - if (ev.state == _metadataTaskState) { - return; - } - _metadataTaskState = ev.state; - if (_isPaused != true) { - if (ev.state == MetadataTaskState.waitingForWifi) { - FlutterBackgroundService() - ..setNotificationInfo( - title: _L10n.global().metadataTaskPauseNoWiFiNotification, - ) - ..pauseWakeLock(); - _isPaused = true; - } else if (ev.state == MetadataTaskState.lowBattery) { - FlutterBackgroundService() - ..setNotificationInfo( - title: _L10n.global().metadataTaskPauseLowBatteryNotification, - ) - ..pauseWakeLock(); - _isPaused = true; - } - } else { - if (ev.state == MetadataTaskState.prcoessing) { - FlutterBackgroundService().resumeWakeLock(); - _isPaused = false; - } - } - } - - void _stopSelf() { - _log.info("[_stopSelf] Stopping service"); - FlutterBackgroundService().setNotificationInfo( - title: _L10n.global().backgroundServiceStopping, - ); - _shouldRun.value = false; - } - - var _metadataTaskState = MetadataTaskState.idle; - late final _metadataTaskStateChangedListener = - AppEventListener( - _onMetadataTaskStateChanged); - - bool? _isPaused; - - static final _shouldRun = ValueNotifier(true); -} - -/// Access localized string out of the main isolate -@npLog -class _L10n { - factory _L10n() => _inst; - - _L10n._(); - - Future init() async { - try { - final locale = language_util.getSelectedLocale(); - if (locale == null) { - _l10n = await _queryL10n(); - } else { - _l10n = lookupAppLocalizations(locale); - } - } catch (e, stackTrace) { - _log.shout("[init] Uncaught exception", e, stackTrace); - _l10n = AppLocalizationsEn(); - } - } - - static AppLocalizations global() => _L10n()._l10n; - - Future _queryL10n() async { - try { - final locale = await Devicelocale.currentAsLocale; - return lookupAppLocalizations(locale!); - } on FlutterError catch (_) { - // unsupported locale, use default (en) - return AppLocalizationsEn(); - } catch (e, stackTrace) { - _log.shout( - "[_queryL10n] Failed while lookupAppLocalizations", e, stackTrace); - return AppLocalizationsEn(); - } - } - - static final _inst = _L10n._(); - late AppLocalizations _l10n; -} - -@npLog -class _MetadataTask { - _MetadataTask(this.service, this.account, this.accountPref); - - Future call() async { - try { - await _updateMetadata(); - } catch (e, stackTrace) { - _log.shout("[call] Uncaught exception", e, stackTrace); - } - if (_processedIds.isNotEmpty) { - unawaited( - MessageRelay.broadcast(FileExifUpdatedEvent(_processedIds).toEvent()), - ); - _processedIds = []; - } - - final c = KiwiContainer().resolve(); - if (c.fileRepo.dataSrc is FileCachedDataSource) { - await (c.fileRepo.dataSrc as FileCachedDataSource).flushRemoteTouch(); - } - } - - Future _updateMetadata() async { - final shareFolder = File( - path: file_util.unstripPath(account, accountPref.getShareFolderOr())); - bool hasScanShareFolder = false; - final c = KiwiContainer().resolve(); - final geocoder = ReverseGeocoder(); - await geocoder.init(); - for (final r in account.roots) { - final dir = File(path: file_util.unstripPath(account, r)); - hasScanShareFolder |= file_util.isOrUnderDir(shareFolder, dir); - final updater = UpdateMissingMetadata( - c, const _UpdateMissingMetadataConfigProvider(), geocoder); - void onServiceStop() { - _log.info("[_updateMetadata] Stopping task: user canceled"); - updater.stop(); - _shouldRun = false; - } - - _Service._shouldRun.addListener(onServiceStop); - try { - await for (final ev in updater(account, dir)) { - if (ev is File) { - _onFileProcessed(ev); - } - } - } finally { - _Service._shouldRun.removeListener(onServiceStop); - } - if (!_shouldRun) { - return; - } - } - if (!hasScanShareFolder) { - final shareUpdater = UpdateMissingMetadata( - c, const _UpdateMissingMetadataConfigProvider(), geocoder); - void onServiceStop() { - _log.info("[_updateMetadata] Stopping task: user canceled"); - shareUpdater.stop(); - _shouldRun = false; - } - - _Service._shouldRun.addListener(onServiceStop); - try { - await for (final ev in shareUpdater( - account, - shareFolder, - isRecursive: false, - filter: (f) => f.ownerId != account.userId, - )) { - if (ev is File) { - _onFileProcessed(ev); - } - } - } finally { - _Service._shouldRun.removeListener(onServiceStop); - } - if (!_shouldRun) { - return; - } - } - } - - void _onFileProcessed(File file) { - ++_count; - service.setNotificationInfo( - title: _L10n.global().metadataTaskProcessingNotification, - content: file.strippedPath, - ); - - _processedIds.add(file.fileId!); - if (_processedIds.length >= 10) { - MessageRelay.broadcast(FileExifUpdatedEvent(_processedIds).toEvent()); - _processedIds = []; - } - } - - final FlutterBackgroundService service; - final Account account; - final AccountPref accountPref; - - var _shouldRun = true; - var _count = 0; - var _processedIds = []; -} - -class _UpdateMissingMetadataConfigProvider - implements UpdateMissingMetadataConfigProvider { - const _UpdateMissingMetadataConfigProvider(); - - @override - isWifiOnly() => - Preference.getBool(_servicePref, _servicePrefProcessWifiOnly, true) - .notNull(); -} - -class _C { - // needed to work with generator logger - static final log = Logger("service"); -} - -const _dataKeyEvent = "event"; -const _eventStop = "stop"; - -const _servicePref = "service"; -const _servicePrefProcessWifiOnly = "shouldProcessWifiOnly"; diff --git a/app/lib/service/config.dart b/app/lib/service/config.dart new file mode 100644 index 00000000..67690b77 --- /dev/null +++ b/app/lib/service/config.dart @@ -0,0 +1,15 @@ +part of 'service.dart'; + +class ServiceConfig { + static Future isProcessExifWifiOnly() async { + return Preference.getBool(_pref, _prefProcessWifiOnly, true) + .notNull(); + } + + static Future setProcessExifWifiOnly(bool flag) async { + await Preference.setBool(_pref, _prefProcessWifiOnly, flag); + } + + static const _pref = "service"; + static const _prefProcessWifiOnly = "shouldProcessWifiOnly"; +} diff --git a/app/lib/service/l10n.dart b/app/lib/service/l10n.dart new file mode 100644 index 00000000..343ef704 --- /dev/null +++ b/app/lib/service/l10n.dart @@ -0,0 +1,43 @@ +part of 'service.dart'; + +/// Access localized string out of the main isolate +@npLog +class _L10n { + _L10n._(); + + factory _L10n() => _inst; + + Future init() async { + try { + final locale = language_util.getSelectedLocale(); + if (locale == null) { + _l10n = await _queryL10n(); + } else { + _l10n = lookupAppLocalizations(locale); + } + } catch (e, stackTrace) { + _log.shout("[init] Uncaught exception", e, stackTrace); + _l10n = AppLocalizationsEn(); + } + } + + static AppLocalizations global() => _L10n()._l10n; + + Future _queryL10n() async { + try { + final locale = await Devicelocale.currentAsLocale; + return lookupAppLocalizations(locale!); + } on FlutterError catch (_) { + // unsupported locale, use default (en) + return AppLocalizationsEn(); + } catch (e, stackTrace) { + _log.shout( + "[_queryL10n] Failed while lookupAppLocalizations", e, stackTrace); + return AppLocalizationsEn(); + } + } + + late AppLocalizations _l10n; + + static final _inst = _L10n._(); +} diff --git a/app/lib/service/service.dart b/app/lib/service/service.dart new file mode 100644 index 00000000..cf689d87 --- /dev/null +++ b/app/lib/service/service.dart @@ -0,0 +1,192 @@ +import 'dart:async'; + +import 'package:devicelocale/devicelocale.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_background_service/flutter_background_service.dart'; +import 'package:flutter_gen/gen_l10n/app_localizations.dart'; +import 'package:flutter_gen/gen_l10n/app_localizations_en.dart'; +import 'package:kiwi/kiwi.dart'; +import 'package:logging/logging.dart'; +import 'package:nc_photos/app_init.dart' as app_init; +import 'package:nc_photos/app_localizations.dart'; +import 'package:nc_photos/controller/pref_controller.dart'; +import 'package:nc_photos/di_container.dart'; +import 'package:nc_photos/entity/file_descriptor.dart'; +import 'package:nc_photos/entity/pref.dart'; +import 'package:nc_photos/event/native_event.dart'; +import 'package:nc_photos/language_util.dart' as language_util; +import 'package:nc_photos/use_case/battery_ensurer.dart'; +import 'package:nc_photos/use_case/sync_metadata/sync_metadata.dart'; +import 'package:nc_photos/use_case/wifi_ensurer.dart'; +import 'package:nc_photos_plugin/nc_photos_plugin.dart'; +import 'package:np_async/np_async.dart'; +import 'package:np_codegen/np_codegen.dart'; +import 'package:np_platform_message_relay/np_platform_message_relay.dart'; + +part 'config.dart'; +part 'l10n.dart'; +part 'service.g.dart'; + +/// Start the background service +Future startService() async { + _$__NpLog.log.info("[startService] Starting service"); + final service = FlutterBackgroundService(); + await service.configure( + androidConfiguration: AndroidConfiguration( + onStart: serviceMain, + autoStart: false, + isForegroundMode: true, + foregroundServiceNotificationTitle: + L10n.global().metadataTaskProcessingNotification, + ), + iosConfiguration: IosConfiguration( + onForeground: () => throw UnimplementedError(), + onBackground: () => throw UnimplementedError(), + ), + ); + // sync settings + await ServiceConfig.setProcessExifWifiOnly( + Pref().shouldProcessExifWifiOnlyOr()); + await service.start(); +} + +/// Ask the background service to stop ASAP +void stopService() { + _$__NpLog.log.info("[stopService] Stopping service"); + FlutterBackgroundService().sendData({ + _dataKeyEvent: _eventStop, + }); +} + +@visibleForTesting +@pragma("vm:entry-point") +Future serviceMain() async { + WidgetsFlutterBinding.ensureInitialized(); + await _Service()(); +} + +@npLog +class _Service { + Future call() async { + final service = FlutterBackgroundService(); + service.setForegroundMode(true); + + await app_init.init(app_init.InitIsolateType.flutterIsolate); + await _L10n().init(); + + _log.info("[call] Service started"); + final onCancelSubscription = service.onCancel.listen((_) { + _log.info("[call] User canceled"); + _stopSelf(); + }); + final onDataSubscription = + service.onDataReceived.listen((event) => _onNotifAction(event ?? {})); + + try { + await _doWork(); + } catch (e, stackTrace) { + _log.shout("[call] Uncaught exception", e, stackTrace); + } + await onCancelSubscription.cancel(); + await onDataSubscription.cancel(); + await KiwiContainer().resolve().npDb.dispose(); + service.stopBackgroundService(); + _log.info("[call] Service stopped"); + } + + Future _doWork() async { + final prefController = PrefController(Pref()); + final account = prefController.currentAccountValue; + if (account == null) { + _log.shout("[_doWork] account == null"); + return; + } + final c = KiwiContainer().resolve(); + + final wifiEnsurer = WifiEnsurer( + interrupter: _shouldRun.stream, + ); + wifiEnsurer.isWaiting.listen((event) { + if (event) { + FlutterBackgroundService() + ..setNotificationInfo( + title: _L10n.global().metadataTaskPauseNoWiFiNotification, + ) + ..pauseWakeLock(); + } else { + FlutterBackgroundService().resumeWakeLock(); + } + }); + final batteryEnsurer = BatteryEnsurer( + interrupter: _shouldRun.stream, + ); + batteryEnsurer.isWaiting.listen((event) { + if (event) { + FlutterBackgroundService() + ..setNotificationInfo( + title: _L10n.global().metadataTaskPauseLowBatteryNotification, + ) + ..pauseWakeLock(); + } else { + FlutterBackgroundService().resumeWakeLock(); + } + }); + + final service = FlutterBackgroundService(); + final syncOp = SyncMetadata( + fileRepo: c.fileRepo, + fileRepo2: c.fileRepo2, + fileRepoRemote: c.fileRepoRemote, + db: c.npDb, + interrupter: _shouldRun.stream, + wifiEnsurer: wifiEnsurer, + batteryEnsurer: batteryEnsurer, + ); + final processedIds = []; + await for (final f in syncOp.syncAccount(account)) { + processedIds.add(f.fdId); + service.setNotificationInfo( + title: _L10n.global().metadataTaskProcessingNotification, + content: f.strippedPath, + ); + } + if (processedIds.isNotEmpty) { + await MessageRelay.broadcast( + FileExifUpdatedEvent(processedIds).toEvent()); + } + } + + void _onNotifAction(Map data) { + try { + final event = data[_dataKeyEvent]; + switch (event) { + case _eventStop: + _stopSelf(); + break; + + default: + _log.severe("[_onNotifAction] Unknown event: $event"); + break; + } + } catch (e, stackTrace) { + _log.shout("[_onNotifAction] Uncaught exception", e, stackTrace); + } + } + + void _stopSelf() { + _log.info("[_stopSelf] Stopping service"); + FlutterBackgroundService().setNotificationInfo( + title: _L10n.global().backgroundServiceStopping, + ); + _shouldRun.add(null); + } + + final _shouldRun = StreamController.broadcast(); +} + +@npLog +// ignore: camel_case_types +class __ {} + +const _dataKeyEvent = "event"; +const _eventStop = "stop"; diff --git a/app/lib/service.g.dart b/app/lib/service/service.g.dart similarity index 71% rename from app/lib/service.g.dart rename to app/lib/service/service.g.dart index 03542f4b..bdcfc699 100644 --- a/app/lib/service.g.dart +++ b/app/lib/service/service.g.dart @@ -10,19 +10,19 @@ extension _$_ServiceNpLog on _Service { // ignore: unused_element Logger get _log => log; - static final log = Logger("service._Service"); + static final log = Logger("service.service._Service"); +} + +extension _$__NpLog on __ { + // ignore: unused_element + Logger get _log => log; + + static final log = Logger("service.service.__"); } extension _$_L10nNpLog on _L10n { // ignore: unused_element Logger get _log => log; - static final log = Logger("service._L10n"); -} - -extension _$_MetadataTaskNpLog on _MetadataTask { - // ignore: unused_element - Logger get _log => log; - - static final log = Logger("service._MetadataTask"); + static final log = Logger("service.service._L10n"); } diff --git a/app/lib/use_case/battery_ensurer.dart b/app/lib/use_case/battery_ensurer.dart new file mode 100644 index 00000000..332f5b84 --- /dev/null +++ b/app/lib/use_case/battery_ensurer.dart @@ -0,0 +1,35 @@ +import 'package:battery_plus/battery_plus.dart'; +import 'package:nc_photos/exception.dart'; +import 'package:rxdart/rxdart.dart'; + +class BatteryEnsurer { + BatteryEnsurer({ + this.interrupter, + }) { + interrupter?.listen((event) { + _shouldRun = false; + }); + } + + Future call() async { + while (await Battery().batteryLevel <= 15) { + if (!_shouldRun) { + throw const InterruptedException(); + } + if (!_isWaiting.value) { + _isWaiting.add(true); + } + await Future.delayed(const Duration(seconds: 5)); + } + if (_isWaiting.value) { + _isWaiting.add(false); + } + } + + ValueStream get isWaiting => _isWaiting.stream; + + final Stream? interrupter; + + var _shouldRun = true; + final _isWaiting = BehaviorSubject.seeded(false); +} diff --git a/app/lib/use_case/scan_missing_metadata.dart b/app/lib/use_case/scan_missing_metadata.dart deleted file mode 100644 index 8d1ef6aa..00000000 --- a/app/lib/use_case/scan_missing_metadata.dart +++ /dev/null @@ -1,50 +0,0 @@ -import 'package:nc_photos/account.dart'; -import 'package:nc_photos/entity/file.dart'; -import 'package:nc_photos/entity/file_util.dart' as file_util; -import 'package:nc_photos/exception_event.dart'; -import 'package:nc_photos/use_case/ls.dart'; -import 'package:nc_photos/use_case/scan_dir.dart'; - -class ScanMissingMetadata { - ScanMissingMetadata(this.fileRepo); - - /// List all files that support metadata but yet having one under a dir - /// - /// The returned stream would emit either File data or ExceptionEvent - /// - /// If [isRecursive] is true, [root] and its sub dirs will be listed, - /// otherwise only [root] will be listed. Default to true - Stream call( - Account account, - File root, { - bool isRecursive = true, - }) async* { - if (isRecursive) { - yield* _doRecursive(account, root); - } else { - yield* _doSingle(account, root); - } - } - - Stream _doRecursive(Account account, File root) async* { - final dataStream = ScanDir(fileRepo)(account, root); - await for (final d in dataStream) { - if (d is ExceptionEvent) { - yield d; - continue; - } - for (final f in (d as List).where(file_util.isMissingMetadata)) { - yield f; - } - } - } - - Stream _doSingle(Account account, File root) async* { - final files = await Ls(fileRepo)(account, root); - for (final f in files.where(file_util.isMissingMetadata)) { - yield f; - } - } - - final FileRepo fileRepo; -} diff --git a/app/lib/use_case/sync_metadata/sync_by_app.dart b/app/lib/use_case/sync_metadata/sync_by_app.dart new file mode 100644 index 00000000..d7106905 --- /dev/null +++ b/app/lib/use_case/sync_metadata/sync_by_app.dart @@ -0,0 +1,120 @@ +part of 'sync_metadata.dart'; + +/// Sync metadata using the client side logic +@npLog +class _SyncByApp { + _SyncByApp({ + required this.account, + required this.fileRepo, + required this.fileRepo2, + required this.db, + this.interrupter, + required this.wifiEnsurer, + required this.batteryEnsurer, + }) { + interrupter?.listen((event) { + _shouldRun = false; + }); + } + + Future init() async { + await _geocoder.init(); + } + + Stream syncFiles({ + required List fileIds, + }) async* { + for (final ids in partition(fileIds, 100)) { + yield* _syncGroup(ids); + } + } + + Stream _syncGroup(List fileIds) async* { + final files = await db.getFilesByFileIds( + account: account.toDb(), + fileIds: fileIds, + ); + for (final f in files) { + final result = await _syncOne(f); + if (result != null) { + yield result; + } + if (!_shouldRun) { + return; + } + } + } + + Future _syncOne(DbFile file) async { + final f = DbFileConverter.fromDb( + account.userId.toCaseInsensitiveString(), + file, + ); + _log.fine("[_syncOne] Syncing ${file.relativePath}"); + try { + OrNull? metadataUpdate; + OrNull? locationUpdate; + if (f.metadata == null) { + // since we need to download multiple images in their original size, + // we only do it with WiFi + await wifiEnsurer(); + await batteryEnsurer(); + if (!_shouldRun) { + return null; + } + _log.fine("[_syncOne] Updating metadata for ${f.path}"); + final binary = await GetFileBinary(fileRepo)(account, f); + final metadata = + (await LoadMetadata().loadRemote(account, f, binary)).copyWith( + fileEtag: f.etag, + ); + metadataUpdate = OrNull(metadata); + } + + final lat = (metadataUpdate?.obj ?? f.metadata)?.exif?.gpsLatitudeDeg; + final lng = (metadataUpdate?.obj ?? f.metadata)?.exif?.gpsLongitudeDeg; + try { + ImageLocation? location; + if (lat != null && lng != null) { + _log.fine("[_syncOne] Reverse geocoding for ${f.path}"); + final l = await _geocoder(lat, lng); + if (l != null) { + location = l.toImageLocation(); + } + } + locationUpdate = OrNull(location ?? ImageLocation.empty()); + } catch (e, stackTrace) { + _log.severe("[_syncOne] Failed while reverse geocoding: ${f.path}", e, + stackTrace); + // if failed, we skip updating the location + } + + if (metadataUpdate != null || locationUpdate != null) { + await UpdateProperty(fileRepo: fileRepo2)( + account, + f, + metadata: metadataUpdate, + location: locationUpdate, + ); + return f; + } else { + return null; + } + } catch (e, stackTrace) { + _log.severe("[_syncOne] Failed while updating metadata: ${f.path}", e, + stackTrace); + return null; + } + } + + final Account account; + final FileRepo fileRepo; + final FileRepo2 fileRepo2; + final NpDb db; + final Stream? interrupter; + final WifiEnsurer wifiEnsurer; + final BatteryEnsurer batteryEnsurer; + + final _geocoder = ReverseGeocoder(); + var _shouldRun = true; +} diff --git a/app/lib/use_case/sync_metadata/sync_by_server.dart b/app/lib/use_case/sync_metadata/sync_by_server.dart new file mode 100644 index 00000000..cb4956f6 --- /dev/null +++ b/app/lib/use_case/sync_metadata/sync_by_server.dart @@ -0,0 +1,91 @@ +part of 'sync_metadata.dart'; + +/// Sync metadata using the client side logic +@npLog +class _SyncByServer { + _SyncByServer({ + required this.account, + required this.fileRepoRemote, + required this.fileRepo2, + required this.db, + this.interrupter, + }) { + interrupter?.listen((event) { + _shouldRun = false; + }); + } + + Future init() async { + await _geocoder.init(); + } + + Stream syncFiles({ + required List relativePaths, + }) async* { + final dirs = relativePaths.map(dirname).toSet(); + for (final dir in dirs) { + yield* _syncDir( + dir: File(path: file_util.unstripPath(account, dir)), + ); + } + } + + Stream _syncDir({ + required File dir, + }) async* { + try { + _log.fine("[_syncDir] Syncing dir $dir"); + final files = await fileRepoRemote.list(account, dir); + await FileSqliteCacheUpdater(db)(account, dir, remote: files); + for (final f in files) { + if (f.metadata != null && f.location == null) { + final result = await _syncOne(f); + if (result != null) { + yield result; + } + if (!_shouldRun) { + return; + } + } + } + } catch (e, stackTrace) { + _log.severe("[_syncDir] Failed to sync dir: $dir", e, stackTrace); + } + } + + Future _syncOne(File file) async { + try { + final lat = file.metadata!.exif?.gpsLatitudeDeg; + final lng = file.metadata!.exif?.gpsLongitudeDeg; + ImageLocation? location; + if (lat != null && lng != null) { + _log.fine("[_syncOne] Reverse geocoding for ${file.path}"); + final l = await _geocoder(lat, lng); + if (l != null) { + location = l.toImageLocation(); + } + } + final locationUpdate = OrNull(location ?? ImageLocation.empty()); + await UpdateProperty(fileRepo: fileRepo2)( + account, + file, + metadata: OrNull(file.metadata), + location: locationUpdate, + ); + return file; + } catch (e, stackTrace) { + _log.severe("[_syncOne] Failed while updating location: ${file.path}", e, + stackTrace); + return null; + } + } + + final Account account; + final FileRepo fileRepoRemote; + final FileRepo2 fileRepo2; + final NpDb db; + final Stream? interrupter; + + final _geocoder = ReverseGeocoder(); + var _shouldRun = true; +} diff --git a/app/lib/use_case/sync_metadata/sync_metadata.dart b/app/lib/use_case/sync_metadata/sync_metadata.dart new file mode 100644 index 00000000..cacc76d9 --- /dev/null +++ b/app/lib/use_case/sync_metadata/sync_metadata.dart @@ -0,0 +1,109 @@ +import 'dart:async'; + +import 'package:logging/logging.dart'; +import 'package:nc_photos/account.dart'; +import 'package:nc_photos/controller/server_controller.dart'; +import 'package:nc_photos/db/entity_converter.dart'; +import 'package:nc_photos/entity/exif_util.dart'; +import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/file_cache_manager.dart'; +import 'package:nc_photos/entity/file/repo.dart'; +import 'package:nc_photos/entity/file_util.dart' as file_util; +import 'package:nc_photos/geocoder_util.dart'; +import 'package:nc_photos/use_case/battery_ensurer.dart'; +import 'package:nc_photos/use_case/get_file_binary.dart'; +import 'package:nc_photos/use_case/load_metadata.dart'; +import 'package:nc_photos/use_case/update_property.dart'; +import 'package:nc_photos/use_case/wifi_ensurer.dart'; +import 'package:np_codegen/np_codegen.dart'; +import 'package:np_collection/np_collection.dart'; +import 'package:np_common/or_null.dart'; +import 'package:np_db/np_db.dart'; +import 'package:np_geocoder/np_geocoder.dart'; +import 'package:path/path.dart'; + +part 'sync_by_app.dart'; +part 'sync_by_server.dart'; +part 'sync_metadata.g.dart'; + +@npLog +class SyncMetadata { + const SyncMetadata({ + required this.fileRepo, + required this.fileRepo2, + required this.fileRepoRemote, + required this.db, + this.interrupter, + required this.wifiEnsurer, + required this.batteryEnsurer, + }); + + Stream syncAccount(Account account) async* { + final bool isNcMetadataSupported; + try { + isNcMetadataSupported = await _isNcMetadataSupported(account); + } catch (e) { + _log.severe("[syncAccount] Failed to get server version", e); + return; + } + final files = await db.getFilesByMissingMetadata( + account: account.toDb(), + mimes: file_util.metadataSupportedFormatMimes, + ownerId: account.userId.toCaseInsensitiveString(), + ); + _log.info("[syncAccount] Missing count: ${files.items.length}"); + if (isNcMetadataSupported) { + yield* _doWithServer(account, files); + } else { + yield* _doWithApp(account, files); + } + } + + Stream _doWithApp( + Account account, DbFileMissingMetadataResult files) async* { + final op = _SyncByApp( + account: account, + fileRepo: fileRepo, + fileRepo2: fileRepo2, + db: db, + interrupter: interrupter, + wifiEnsurer: wifiEnsurer, + batteryEnsurer: batteryEnsurer, + ); + await op.init(); + final stream = op.syncFiles( + fileIds: files.items.map((e) => e.fileId).toList(), + ); + yield* stream; + } + + Stream _doWithServer( + Account account, DbFileMissingMetadataResult files) async* { + final op = _SyncByServer( + account: account, + fileRepoRemote: fileRepoRemote, + fileRepo2: fileRepo2, + db: db, + interrupter: interrupter, + ); + await op.init(); + final stream = op.syncFiles( + relativePaths: files.items.map((e) => e.relativePath).toList(), + ); + yield* stream; + } + + Future _isNcMetadataSupported(Account account) async { + final serverController = ServerController(account: account); + await serverController.status.first.timeout(const Duration(seconds: 15)); + return serverController.isSupported(ServerFeature.ncMetadata); + } + + final FileRepo fileRepo; + final FileRepo2 fileRepo2; + final FileRepo fileRepoRemote; + final NpDb db; + final Stream? interrupter; + final WifiEnsurer wifiEnsurer; + final BatteryEnsurer batteryEnsurer; +} diff --git a/app/lib/use_case/sync_metadata/sync_metadata.g.dart b/app/lib/use_case/sync_metadata/sync_metadata.g.dart new file mode 100644 index 00000000..d27370e4 --- /dev/null +++ b/app/lib/use_case/sync_metadata/sync_metadata.g.dart @@ -0,0 +1,30 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'sync_metadata.dart'; + +// ************************************************************************** +// NpLogGenerator +// ************************************************************************** + +extension _$SyncMetadataNpLog on SyncMetadata { + // ignore: unused_element + Logger get _log => log; + + static final log = + Logger("use_case.sync_metadata.sync_metadata.SyncMetadata"); +} + +extension _$_SyncByAppNpLog on _SyncByApp { + // ignore: unused_element + Logger get _log => log; + + static final log = Logger("use_case.sync_metadata.sync_metadata._SyncByApp"); +} + +extension _$_SyncByServerNpLog on _SyncByServer { + // ignore: unused_element + Logger get _log => log; + + static final log = + Logger("use_case.sync_metadata.sync_metadata._SyncByServer"); +} diff --git a/app/lib/use_case/update_missing_metadata.dart b/app/lib/use_case/update_missing_metadata.dart deleted file mode 100644 index 89a52470..00000000 --- a/app/lib/use_case/update_missing_metadata.dart +++ /dev/null @@ -1,173 +0,0 @@ -import 'package:battery_plus/battery_plus.dart'; -import 'package:event_bus/event_bus.dart'; -import 'package:kiwi/kiwi.dart'; -import 'package:logging/logging.dart'; -import 'package:nc_photos/account.dart'; -import 'package:nc_photos/connectivity_util.dart' as connectivity_util; -import 'package:nc_photos/di_container.dart'; -import 'package:nc_photos/entity/exif_util.dart'; -import 'package:nc_photos/entity/file.dart'; -import 'package:nc_photos/event/event.dart'; -import 'package:nc_photos/exception.dart'; -import 'package:nc_photos/exception_event.dart'; -import 'package:nc_photos/geocoder_util.dart'; -import 'package:nc_photos/use_case/get_file_binary.dart'; -import 'package:nc_photos/use_case/load_metadata.dart'; -import 'package:nc_photos/use_case/scan_missing_metadata.dart'; -import 'package:nc_photos/use_case/update_property.dart'; -import 'package:np_codegen/np_codegen.dart'; -import 'package:np_common/or_null.dart'; -import 'package:np_geocoder/np_geocoder.dart'; - -part 'update_missing_metadata.g.dart'; - -abstract class UpdateMissingMetadataConfigProvider { - Future isWifiOnly(); -} - -@npLog -class UpdateMissingMetadata { - UpdateMissingMetadata(this._c, this.configProvider, this.geocoder); - - /// Update metadata for all files that support one under a dir - /// - /// The returned stream would emit either File data (for each updated files) - /// or ExceptionEvent - /// - /// If [isRecursive] is true, [root] and its sub dirs will be scanned, - /// otherwise only [root] will be scanned. Default to true - /// - /// [filter] can be used to filter files -- return true if a file should be - /// included. If [filter] is null, all files will be included. - Stream call( - Account account, - File root, { - bool isRecursive = true, - bool Function(File file)? filter, - }) async* { - final dataStream = ScanMissingMetadata(_c.fileRepo)( - account, - root, - isRecursive: isRecursive, - ); - await for (final d in dataStream) { - if (!_shouldRun) { - return; - } - if (d is ExceptionEvent) { - yield d; - continue; - } - final File file = d; - // check if this is a federation share. Nextcloud doesn't support - // properties for such files - if (file.ownerId?.contains("/") == true || filter?.call(d) == false) { - continue; - } - try { - OrNull? metadataUpdate; - OrNull? locationUpdate; - if (file.metadata == null) { - // since we need to download multiple images in their original size, - // we only do it with WiFi - await _ensureWifi(); - await _ensureBattery(); - KiwiContainer().resolve().fire( - const MetadataTaskStateChangedEvent( - MetadataTaskState.prcoessing)); - if (!_shouldRun) { - return; - } - _log.fine("[call] Updating metadata for ${file.path}"); - final binary = await GetFileBinary(_c.fileRepo)(account, file); - final metadata = - (await LoadMetadata().loadRemote(account, file, binary)).copyWith( - fileEtag: file.etag, - ); - metadataUpdate = OrNull(metadata); - } else { - _log.finer("[call] Skip updating metadata for ${file.path}"); - KiwiContainer().resolve().fire( - const MetadataTaskStateChangedEvent( - MetadataTaskState.prcoessing)); - } - - final lat = - (metadataUpdate?.obj ?? file.metadata)?.exif?.gpsLatitudeDeg; - final lng = - (metadataUpdate?.obj ?? file.metadata)?.exif?.gpsLongitudeDeg; - try { - ImageLocation? location; - if (lat != null && lng != null) { - _log.fine("[call] Reverse geocoding for ${file.path}"); - final l = await geocoder(lat, lng); - if (l != null) { - location = l.toImageLocation(); - } - } - locationUpdate = OrNull(location ?? ImageLocation.empty()); - } catch (e, stackTrace) { - _log.severe("[call] Failed while reverse geocoding: ${file.path}", e, - stackTrace); - } - - if (metadataUpdate != null || locationUpdate != null) { - await UpdateProperty(_c)( - account, - file, - metadata: metadataUpdate, - location: locationUpdate, - ); - yield file; - } - - // slow down a bit to give some space for the main isolate - await Future.delayed(const Duration(milliseconds: 10)); - } on InterruptedException catch (_) { - return; - } catch (e, stackTrace) { - _log.severe("[call] Failed while updating metadata: ${file.path}", e, - stackTrace); - yield ExceptionEvent(e, stackTrace); - } - } - } - - void stop() { - _shouldRun = false; - } - - Future _ensureWifi() async { - var count = 0; - while (await configProvider.isWifiOnly() && - !await connectivity_util.isWifi()) { - if (!_shouldRun) { - throw const InterruptedException(); - } - // give a chance to reconnect with the WiFi network - if (++count >= 6) { - KiwiContainer().resolve().fire( - const MetadataTaskStateChangedEvent( - MetadataTaskState.waitingForWifi)); - } - await Future.delayed(const Duration(seconds: 5)); - } - } - - Future _ensureBattery() async { - while (await Battery().batteryLevel <= 15) { - if (!_shouldRun) { - throw const InterruptedException(); - } - KiwiContainer().resolve().fire( - const MetadataTaskStateChangedEvent(MetadataTaskState.lowBattery)); - await Future.delayed(const Duration(seconds: 5)); - } - } - - final DiContainer _c; - final UpdateMissingMetadataConfigProvider configProvider; - final ReverseGeocoder geocoder; - - bool _shouldRun = true; -} diff --git a/app/lib/use_case/update_missing_metadata.g.dart b/app/lib/use_case/update_missing_metadata.g.dart deleted file mode 100644 index 1c045135..00000000 --- a/app/lib/use_case/update_missing_metadata.g.dart +++ /dev/null @@ -1,15 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'update_missing_metadata.dart'; - -// ************************************************************************** -// NpLogGenerator -// ************************************************************************** - -extension _$UpdateMissingMetadataNpLog on UpdateMissingMetadata { - // ignore: unused_element - Logger get _log => log; - - static final log = - Logger("use_case.update_missing_metadata.UpdateMissingMetadata"); -} diff --git a/app/lib/use_case/update_property.dart b/app/lib/use_case/update_property.dart index 65f5e970..8889b733 100644 --- a/app/lib/use_case/update_property.dart +++ b/app/lib/use_case/update_property.dart @@ -1,11 +1,8 @@ -import 'package:event_bus/event_bus.dart'; -import 'package:kiwi/kiwi.dart'; import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; -import 'package:nc_photos/di_container.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/repo.dart'; import 'package:nc_photos/entity/file_descriptor.dart'; -import 'package:nc_photos/event/event.dart'; import 'package:np_codegen/np_codegen.dart'; import 'package:np_common/or_null.dart'; @@ -13,7 +10,11 @@ part 'update_property.g.dart'; @npLog class UpdateProperty { - const UpdateProperty(this._c); + const UpdateProperty({ + required this.fileRepo, + }); + + final FileRepo2 fileRepo; Future call( Account account, @@ -34,17 +35,7 @@ class UpdateProperty { return; } - await _c.fileRepo2.updateProperty( - account, - file, - metadata: metadata, - isArchived: isArchived, - overrideDateTime: overrideDateTime, - favorite: favorite, - location: location, - ); - - _notify( + await fileRepo.updateProperty( account, file, metadata: metadata, @@ -54,40 +45,6 @@ class UpdateProperty { location: location, ); } - - @Deprecated("legacy") - void _notify( - Account account, - FileDescriptor file, { - OrNull? metadata, - OrNull? isArchived, - OrNull? overrideDateTime, - bool? favorite, - OrNull? location, - }) { - int properties = 0; - if (metadata != null) { - properties |= FilePropertyUpdatedEvent.propMetadata; - } - if (isArchived != null) { - properties |= FilePropertyUpdatedEvent.propIsArchived; - } - if (overrideDateTime != null) { - properties |= FilePropertyUpdatedEvent.propOverrideDateTime; - } - if (favorite != null) { - properties |= FilePropertyUpdatedEvent.propFavorite; - } - if (location != null) { - properties |= FilePropertyUpdatedEvent.propImageLocation; - } - assert(properties != 0); - KiwiContainer() - .resolve() - .fire(FilePropertyUpdatedEvent(account, file, properties)); - } - - final DiContainer _c; } extension UpdatePropertyExtension on UpdateProperty { diff --git a/app/lib/use_case/wifi_ensurer.dart b/app/lib/use_case/wifi_ensurer.dart new file mode 100644 index 00000000..ba239fb1 --- /dev/null +++ b/app/lib/use_case/wifi_ensurer.dart @@ -0,0 +1,41 @@ +import 'package:nc_photos/connectivity_util.dart' as connectivity_util; +import 'package:nc_photos/exception.dart'; +import 'package:nc_photos/service/service.dart'; +import 'package:rxdart/rxdart.dart'; + +class WifiEnsurer { + WifiEnsurer({ + this.interrupter, + }) { + interrupter?.listen((event) { + _shouldRun = false; + }); + } + + Future call() async { + var count = 0; + while (await ServiceConfig.isProcessExifWifiOnly() && + !await connectivity_util.isWifi()) { + if (!_shouldRun) { + throw const InterruptedException(); + } + // give a chance to reconnect with the WiFi network + if (++count >= 6) { + if (!_isWaiting.value) { + _isWaiting.add(true); + } + } + await Future.delayed(const Duration(seconds: 5)); + } + if (_isWaiting.value) { + _isWaiting.add(false); + } + } + + ValueStream get isWaiting => _isWaiting.stream; + + final Stream? interrupter; + + var _shouldRun = true; + final _isWaiting = BehaviorSubject.seeded(false); +} diff --git a/app/lib/widget/home_photos/bloc.dart b/app/lib/widget/home_photos/bloc.dart index 6e0da340..e438fb58 100644 --- a/app/lib/widget/home_photos/bloc.dart +++ b/app/lib/widget/home_photos/bloc.dart @@ -536,11 +536,7 @@ class _Bloc extends Bloc<_Event, _State> personsController: personsController, personProvider: accountPrefController.personProviderValue, ); - if (!serverController.isSupported(ServerFeature.ncMetadata)) { - metadataController.kickstart(); - } else { - _log.info("[_syncRemote] Skipping metadata service"); - } + metadataController.kickstart(); _log.info( "[_syncRemote] Elapsed time: ${stopwatch.elapsedMilliseconds}ms"); }); diff --git a/app/lib/widget/settings/metadata_settings.dart b/app/lib/widget/settings/metadata_settings.dart index 3916893d..98982b57 100644 --- a/app/lib/widget/settings/metadata_settings.dart +++ b/app/lib/widget/settings/metadata_settings.dart @@ -6,7 +6,7 @@ import 'package:nc_photos/app_localizations.dart'; import 'package:nc_photos/bloc_util.dart'; import 'package:nc_photos/controller/pref_controller.dart'; import 'package:nc_photos/exception_event.dart'; -import 'package:nc_photos/service.dart'; +import 'package:nc_photos/service/service.dart'; import 'package:nc_photos/snack_bar_manager.dart'; import 'package:nc_photos/widget/page_visibility_mixin.dart'; import 'package:np_codegen/np_codegen.dart'; diff --git a/app/lib/widget/viewer_detail_pane.dart b/app/lib/widget/viewer_detail_pane.dart index e0e16f52..a3e419e0 100644 --- a/app/lib/widget/viewer_detail_pane.dart +++ b/app/lib/widget/viewer_detail_pane.dart @@ -505,7 +505,7 @@ class _ViewerDetailPaneState extends State { return; } try { - await UpdateProperty(_c) + await UpdateProperty(fileRepo: _c.fileRepo2) .updateOverrideDateTime(widget.account, _file!, value); if (mounted) { setState(() { diff --git a/app/test/entity/file/file_cache_manager_test.dart b/app/test/entity/file/file_cache_manager_test.dart index d8ca587a..d58cc797 100644 --- a/app/test/entity/file/file_cache_manager_test.dart +++ b/app/test/entity/file/file_cache_manager_test.dart @@ -205,7 +205,7 @@ Future _updaterIdentical() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(account, files[0], remote: files.slice(0, 3)); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -240,7 +240,7 @@ Future _updaterNewFile() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(account, files[0], remote: [...files.slice(0, 3), newFile]); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -271,7 +271,7 @@ Future _updaterDeleteFile() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(account, files[0], remote: [files[0], files[2]]); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -305,7 +305,7 @@ Future _updaterDeleteDir() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(account, files[0], remote: files.slice(0, 2)); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -343,7 +343,7 @@ Future _updaterUpdateFile() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(account, files[0], remote: [files[0], newFile, ...files.slice(2)]); expect( @@ -382,7 +382,7 @@ Future _updaterNewSharedFile() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(user1Account, user1Files[0], remote: user1Files); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -419,7 +419,7 @@ Future _updaterNewSharedDir() async { await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(user1Account, user1Files[0], remote: user1Files); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -461,7 +461,7 @@ Future _updaterDeleteSharedFile() async { c.sqliteDb, user1Account, user1Files[0], [user1Files[1]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(user1Account, user1Files[0], remote: [user1Files[0]]); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -504,7 +504,7 @@ Future _updaterDeleteSharedDir() async { c.sqliteDb, user1Account, user1Files[0], [user1Files[1]]); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(user1Account, user1Files[0], remote: [user1Files[0]]); expect( await util.listSqliteDbFiles(c.sqliteDb), @@ -541,7 +541,7 @@ Future _updaterTooManyFiles() async { await util.insertDirRelation(c.sqliteDb, account, files[2], files.slice(3)); }); - final updater = FileSqliteCacheUpdater(c); + final updater = FileSqliteCacheUpdater(c.npDb); await updater(account, files[2], remote: [...files.slice(2), ...newFiles]); // we are testing to make sure the above function won't throw, so nothing to // expect here @@ -574,12 +574,12 @@ Future _updaterMovedFileToFront() async { final movedFile = files[3].copyWith( path: "remote.php/dav/files/admin/test1/test1.jpg", ); - await FileSqliteCacheUpdater(c)( + await FileSqliteCacheUpdater(c.npDb)( account, files[1], remote: [files[1], movedFile], ); - await FileSqliteCacheUpdater(c)( + await FileSqliteCacheUpdater(c.npDb)( account, files[2], remote: [files[2]], @@ -621,12 +621,12 @@ Future _updaterMovedFileToBehind() async { final movedFile = files[3].copyWith( path: "remote.php/dav/files/admin/test2/test1.jpg", ); - await FileSqliteCacheUpdater(c)( + await FileSqliteCacheUpdater(c.npDb)( account, files[1], remote: [files[1]], ); - await FileSqliteCacheUpdater(c)( + await FileSqliteCacheUpdater(c.npDb)( account, files[2], remote: [files[2], movedFile], diff --git a/np_collection/lib/np_collection.dart b/np_collection/lib/np_collection.dart index 5373cb4e..bae883df 100644 --- a/np_collection/lib/np_collection.dart +++ b/np_collection/lib/np_collection.dart @@ -1,5 +1,7 @@ library np_collection; +export 'package:quiver/iterables.dart' show partition; + export 'src/iterable_extension.dart'; export 'src/iterator_extension.dart'; export 'src/list_extension.dart'; diff --git a/np_db/lib/src/api.dart b/np_db/lib/src/api.dart index 3544f637..44184b9a 100644 --- a/np_db/lib/src/api.dart +++ b/np_db/lib/src/api.dart @@ -201,6 +201,19 @@ class DbFilesMemory { final Map> memories; } +@genCopyWith +@toString +class DbFileMissingMetadataResult { + const DbFileMissingMetadataResult({ + required this.items, + }); + + @override + String toString() => _$toString(); + + final List<({int fileId, String relativePath})> items; +} + @npLog abstract class NpDb { factory NpDb() => NpDbSqlite(); @@ -354,6 +367,13 @@ abstract class NpDb { required String ownerId, }); + /// Return files without metadata + Future getFilesByMissingMetadata({ + required DbAccount account, + required List mimes, + required String ownerId, + }); + /// Delete a file or dir from db Future deleteFile({ required DbAccount account, diff --git a/np_db/lib/src/api.g.dart b/np_db/lib/src/api.g.dart index ea32abb0..70a73028 100644 --- a/np_db/lib/src/api.g.dart +++ b/np_db/lib/src/api.g.dart @@ -81,6 +81,31 @@ extension $DbFilesMemoryCopyWith on DbFilesMemory { _$DbFilesMemoryCopyWithWorkerImpl(this); } +abstract class $DbFileMissingMetadataResultCopyWithWorker { + DbFileMissingMetadataResult call( + {List<({int fileId, String relativePath})>? items}); +} + +class _$DbFileMissingMetadataResultCopyWithWorkerImpl + implements $DbFileMissingMetadataResultCopyWithWorker { + _$DbFileMissingMetadataResultCopyWithWorkerImpl(this.that); + + @override + DbFileMissingMetadataResult call({dynamic items}) { + return DbFileMissingMetadataResult( + items: + items as List<({int fileId, String relativePath})>? ?? that.items); + } + + final DbFileMissingMetadataResult that; +} + +extension $DbFileMissingMetadataResultCopyWith on DbFileMissingMetadataResult { + $DbFileMissingMetadataResultCopyWithWorker get copyWith => _$copyWith; + $DbFileMissingMetadataResultCopyWithWorker get _$copyWith => + _$DbFileMissingMetadataResultCopyWithWorkerImpl(this); +} + // ************************************************************************** // NpLogGenerator // ************************************************************************** @@ -151,3 +176,10 @@ extension _$DbFilesMemoryToString on DbFilesMemory { return "DbFilesMemory {memories: $memories}"; } } + +extension _$DbFileMissingMetadataResultToString on DbFileMissingMetadataResult { + String _$toString() { + // ignore: unnecessary_string_interpolations + return "DbFileMissingMetadataResult {items: [length: ${items.length}]}"; + } +} diff --git a/np_db_sqlite/lib/src/database/file_extension.dart b/np_db_sqlite/lib/src/database/file_extension.dart index ffa54db5..3c7a2aec 100644 --- a/np_db_sqlite/lib/src/database/file_extension.dart +++ b/np_db_sqlite/lib/src/database/file_extension.dart @@ -343,7 +343,7 @@ extension SqliteDbFileExtension on SqliteDb { String? ownerId, }) async { _log.info( - "[countFiles] isMissingMetadata: $isMissingMetadata, mimes: $mimes"); + "[countFiles] isMissingMetadata: $isMissingMetadata, mimes: $mimes, ownerId: $ownerId"); Expression? filter; if (isMissingMetadata != null) { if (isMissingMetadata) { @@ -387,6 +387,69 @@ extension SqliteDbFileExtension on SqliteDb { return await query.map((r) => r.read(count)!).getSingle(); } + Future> + queryFileIdPathsByMissingMetadata({ + required ByAccount account, + bool? isMissingMetadata, + List? mimes, + String? ownerId, + int? offset, + int? limit, + }) async { + _log.info( + "[queryFileIdPathsByMissingMetadata] isMissingMetadata: $isMissingMetadata, mimes: $mimes, ownerId: $ownerId"); + final query = selectOnly(files).join([ + innerJoin(accountFiles, accountFiles.file.equalsExp(files.rowId), + useColumns: false), + if (account.dbAccount != null) ...[ + innerJoin(accounts, accounts.rowId.equalsExp(accountFiles.account), + useColumns: false), + innerJoin(servers, servers.rowId.equalsExp(accounts.server), + useColumns: false), + ], + leftOuterJoin(images, images.accountFile.equalsExp(accountFiles.rowId), + useColumns: false), + leftOuterJoin(imageLocations, + imageLocations.accountFile.equalsExp(accountFiles.rowId), + useColumns: false), + ]); + query.addColumns([files.fileId, accountFiles.relativePath]); + if (account.sqlAccount != null) { + query.where(accountFiles.account.equals(account.sqlAccount!.rowId)); + } else if (account.dbAccount != null) { + query + ..where(servers.address.equals(account.dbAccount!.serverAddress)) + ..where(accounts.userId + .equals(account.dbAccount!.userId.toCaseInsensitiveString())); + } + if (mimes != null) { + query.where(files.contentType.isIn(mimes)); + } + if (ownerId != null) { + query.where(files.ownerId.equals(ownerId)); + } + if (isMissingMetadata != null) { + if (isMissingMetadata) { + query.where( + images.lastUpdated.isNull() | imageLocations.version.isNull()); + } else { + query.where(images.lastUpdated.isNotNull() & + imageLocations.version.isNotNull()); + } + } + + query.orderBy([OrderingTerm.desc(files.fileId)]); + if (limit != null) { + query.limit(limit, offset: offset); + } + return await query + .map((r) => ( + fileId: r.read(files.fileId)!, + relativePath: r.read(accountFiles.relativePath)!, + )) + .get(); + } + Future> queryFileDescriptors({ required ByAccount account, List? fileIds, diff --git a/np_db_sqlite/lib/src/sqlite_api.dart b/np_db_sqlite/lib/src/sqlite_api.dart index 7c7a1b00..6acd63ad 100644 --- a/np_db_sqlite/lib/src/sqlite_api.dart +++ b/np_db_sqlite/lib/src/sqlite_api.dart @@ -395,6 +395,34 @@ class NpDbSqlite implements NpDb { }); } + @override + Future getFilesByMissingMetadata({ + required DbAccount account, + required List mimes, + required String ownerId, + }) async { + return _db.use((db) async { + final results = <({int fileId, String relativePath})>[]; + var i = 0; + while (true) { + final sqlObjs = await db.queryFileIdPathsByMissingMetadata( + account: ByAccount.db(account), + isMissingMetadata: true, + mimes: mimes, + ownerId: ownerId, + limit: 10000, + offset: i, + ); + if (sqlObjs.isEmpty) { + break; + } + results.addAll(sqlObjs); + i += 10000; + } + return DbFileMissingMetadataResult(items: results); + }); + } + @override Future deleteFile({ required DbAccount account,