Mirrors/nc-photos
1
0
Fork 0
mirror of https://gitlab.com/nkming2/nc-photos.git synced 2025-01-22 08:46:18 +01:00
Code Issues Projects Releases Packages Wiki Activity

Rewrite metadata service, add support for server side exif

Browse source
This commit is contained in:
Ming Ming 2024-11-21 01:34:24 +08:00
parent 5ec09d5c4c
commit 0e4ecf4f2b
31 changed files with 865 additions and 871 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/lib/controller/files_controller.dart
View file

@ -253,7 +253,7 @@ class FilesController {
final failures = <FileDescriptor>[];
for (final f in files) {
try {
await UpdateProperty(_c)(
await UpdateProperty(fileRepo: _c.fileRepo2)(
account,
f,
metadata: metadata,

2
app/lib/controller/metadata_controller.dart
View file

@ -6,7 +6,7 @@ import 'package:nc_photos/controller/pref_controller.dart';
import 'package:nc_photos/db/entity_converter.dart';
import 'package:nc_photos/di_container.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/service.dart' as service;
import 'package:nc_photos/service/service.dart' as service;
import 'package:np_codegen/np_codegen.dart';
part 'metadata_controller.g.dart';

4
app/lib/entity/file/data_source.dart
View file

@ -635,7 +635,7 @@ class FileCachedDataSource implements FileDataSource {
return state.files;
}
await FileSqliteCacheUpdater(_c)(state.account, state.dir,
await FileSqliteCacheUpdater(_c.npDb)(state.account, state.dir,
remote: state.files);
if (shouldCheckCache) {
// update our local touch token to match the remote one
@ -657,7 +657,7 @@ class FileCachedDataSource implements FileDataSource {
if (remote.isCollection != true) {
// only update regular files
_log.info("[listSingle] Cache single file: ${logFilename(f.path)}");
await FileSqliteCacheUpdater(_c).updateSingle(account, remote);
await FileSqliteCacheUpdater(_c.npDb).updateSingle(account, remote);
}
return remote;
}

9
app/lib/entity/file/file_cache_manager.dart
View file

@ -8,6 +8,7 @@ import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/entity/file_descriptor.dart';
import 'package:nc_photos/exception.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_db/np_db.dart';
part 'file_cache_manager.g.dart';
@ -81,7 +82,7 @@ class FileCacheLoader {
@npLog
class FileSqliteCacheUpdater {
const FileSqliteCacheUpdater(this._c);
const FileSqliteCacheUpdater(this.db);
Future<void> call(
Account account,
@ -90,7 +91,7 @@ class FileSqliteCacheUpdater {
}) async {
final s = Stopwatch()..start();
try {
await _c.npDb.syncDirFiles(
await db.syncDirFiles(
account: account.toDb(),
dirFile: dir.toDbKey(),
files: remote.map((e) => e.toDb()).toList(),
@ -101,13 +102,13 @@ class FileSqliteCacheUpdater {
}
Future<void> updateSingle(Account account, File remoteFile) async {
await _c.npDb.syncFile(
await db.syncFile(
account: account.toDb(),
file: remoteFile.toDb(),
);
}
final DiContainer _c;
final NpDb db;
}
class FileSqliteCacheEmptier {

21
app/lib/event/event.dart
View file

@ -46,6 +46,7 @@ class AccountPrefUpdatedEvent {
final dynamic value;
}
@Deprecated("not fired anymore, to be removed")
class FilePropertyUpdatedEvent {
FilePropertyUpdatedEvent(this.account, this.file, this.properties);
@ -97,26 +98,6 @@ class FavoriteResyncedEvent {
final Account account;
}
enum MetadataTaskState {
/// No work is being done
idle,
/// Processing images
prcoessing,
/// Paused on data network
waitingForWifi,
/// Paused on low battery
lowBattery,
}
class MetadataTaskStateChangedEvent {
const MetadataTaskStateChangedEvent(this.state);
final MetadataTaskState state;
}
@Deprecated("not fired anymore, to be removed")
class PrefUpdatedEvent {
PrefUpdatedEvent(this.key, this.value);

134
app/lib/metadata_task_manager.dart
View file

@ -1,134 +0,0 @@
import 'dart:async';
import 'package:event_bus/event_bus.dart';
import 'package:kiwi/kiwi.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/di_container.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/entity/pref.dart';
import 'package:nc_photos/event/event.dart';
import 'package:nc_photos/use_case/update_missing_metadata.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_geocoder/np_geocoder.dart';
import 'package:to_string/to_string.dart';
part 'metadata_task_manager.g.dart';
/// Task to update metadata for missing files
@npLog
@ToString(ignorePrivate: true)
class MetadataTask {
MetadataTask(this._c, this.account, this.pref) : assert(require(_c));
static bool require(DiContainer c) => DiContainer.has(c, DiType.fileRepo);
@override
String toString() => _$toString();
Future<void> call() async {
try {
final shareFolder =
File(path: file_util.unstripPath(account, pref.getShareFolderOr()));
bool hasScanShareFolder = false;
final geocoder = ReverseGeocoder();
await geocoder.init();
for (final r in account.roots) {
final dir = File(path: file_util.unstripPath(account, r));
hasScanShareFolder |= file_util.isOrUnderDir(shareFolder, dir);
final op = UpdateMissingMetadata(
_c, const _UpdateMissingMetadataConfigProvider(), geocoder);
await for (final _ in op(account, dir)) {
if (!Pref().isEnableExifOr()) {
_log.info("[call] EXIF disabled, task ending immaturely");
op.stop();
return;
}
}
}
if (!hasScanShareFolder) {
final op = UpdateMissingMetadata(
_c, const _UpdateMissingMetadataConfigProvider(), geocoder);
await for (final _ in op(
account,
shareFolder,
isRecursive: false,
filter: (f) => f.ownerId != account.userId,
)) {
if (!Pref().isEnableExifOr()) {
_log.info("[call] EXIF disabled, task ending immaturely");
op.stop();
return;
}
}
}
} finally {
KiwiContainer()
.resolve<EventBus>()
.fire(const MetadataTaskStateChangedEvent(MetadataTaskState.idle));
}
}
final DiContainer _c;
final Account account;
@ignore
final AccountPref pref;
}
/// Manage metadata tasks to run concurrently
@npLog
class MetadataTaskManager {
factory MetadataTaskManager() {
_inst ??= MetadataTaskManager._();
return _inst!;
}
MetadataTaskManager._() {
_stateChangedListener.begin();
_handleStream();
}
/// Add a task to the queue
void addTask(MetadataTask task) {
_log.info("[addTask] New task added: $task");
_streamController.add(task);
}
MetadataTaskState get state => _currentState;
void _onMetadataTaskStateChanged(MetadataTaskStateChangedEvent ev) {
if (ev.state != _currentState) {
_currentState = ev.state;
}
}
Future<void> _handleStream() async {
await for (final task in _streamController.stream) {
if (Pref().isEnableExifOr()) {
_log.info("[_doTask] Executing task: $task");
await task();
} else {
_log.info("[_doTask] Ignoring task: $task");
}
}
}
final _streamController = StreamController<MetadataTask>.broadcast();
var _currentState = MetadataTaskState.idle;
late final _stateChangedListener =
AppEventListener<MetadataTaskStateChangedEvent>(
_onMetadataTaskStateChanged);
static MetadataTaskManager? _inst;
}
class _UpdateMissingMetadataConfigProvider
implements UpdateMissingMetadataConfigProvider {
const _UpdateMissingMetadataConfigProvider();
@override
isWifiOnly() async => Pref().shouldProcessExifWifiOnlyOr();
}

32
app/lib/metadata_task_manager.g.dart
View file

@ -1,32 +0,0 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'metadata_task_manager.dart';
// **************************************************************************
// NpLogGenerator
// **************************************************************************
extension _$MetadataTaskNpLog on MetadataTask {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("metadata_task_manager.MetadataTask");
}
extension _$MetadataTaskManagerNpLog on MetadataTaskManager {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("metadata_task_manager.MetadataTaskManager");
}
// **************************************************************************
// ToStringGenerator
// **************************************************************************
extension _$MetadataTaskToString on MetadataTask {
String _$toString() {
// ignore: unnecessary_string_interpolations
return "MetadataTask {account: $account}";
}
}

358
app/lib/service.dart
View file

@ -1,358 +0,0 @@
import 'dart:async';
import 'package:devicelocale/devicelocale.dart';
import 'package:flutter/widgets.dart';
import 'package:flutter_background_service/flutter_background_service.dart';
import 'package:flutter_gen/gen_l10n/app_localizations.dart';
import 'package:flutter_gen/gen_l10n/app_localizations_en.dart';
import 'package:kiwi/kiwi.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/app_init.dart' as app_init;
import 'package:nc_photos/app_localizations.dart';
import 'package:nc_photos/controller/pref_controller.dart';
import 'package:nc_photos/di_container.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/entity/file_descriptor.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/entity/pref.dart';
import 'package:nc_photos/event/event.dart';
import 'package:nc_photos/event/native_event.dart';
import 'package:nc_photos/language_util.dart' as language_util;
import 'package:nc_photos/use_case/update_missing_metadata.dart';
import 'package:nc_photos_plugin/nc_photos_plugin.dart';
import 'package:np_async/np_async.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_geocoder/np_geocoder.dart';
import 'package:np_platform_message_relay/np_platform_message_relay.dart';
part 'service.g.dart';
/// Start the background service
Future<void> startService() async {
_C.log.info("[startService] Starting service");
final service = FlutterBackgroundService();
await service.configure(
androidConfiguration: AndroidConfiguration(
onStart: serviceMain,
autoStart: false,
isForegroundMode: true,
foregroundServiceNotificationTitle:
L10n.global().metadataTaskProcessingNotification,
),
iosConfiguration: IosConfiguration(
onForeground: () => throw UnimplementedError(),
onBackground: () => throw UnimplementedError(),
),
);
// sync settings
await ServiceConfig.setProcessExifWifiOnly(
Pref().shouldProcessExifWifiOnlyOr());
await service.start();
}
/// Ask the background service to stop ASAP
void stopService() {
_C.log.info("[stopService] Stopping service");
FlutterBackgroundService().sendData({
_dataKeyEvent: _eventStop,
});
}
@visibleForTesting
@pragma("vm:entry-point")
Future<void> serviceMain() async {
_Service._shouldRun.value = true;
WidgetsFlutterBinding.ensureInitialized();
await _Service()();
}
class ServiceConfig {
static Future<void> setProcessExifWifiOnly(bool flag) async {
await Preference.setBool(_servicePref, _servicePrefProcessWifiOnly, flag);
}
}
@npLog
class _Service {
Future<void> call() async {
final service = FlutterBackgroundService();
service.setForegroundMode(true);
await app_init.init(app_init.InitIsolateType.flutterIsolate);
await _L10n().init();
_log.info("[call] Service started");
final onCancelSubscription = service.onCancel.listen((_) {
_log.info("[call] User canceled");
_stopSelf();
});
final onDataSubscription =
service.onDataReceived.listen((event) => _onReceiveData(event ?? {}));
try {
await _doWork();
} catch (e, stackTrace) {
_log.shout("[call] Uncaught exception", e, stackTrace);
}
await onCancelSubscription.cancel();
await onDataSubscription.cancel();
await KiwiContainer().resolve<DiContainer>().npDb.dispose();
service.stopBackgroundService();
_log.info("[call] Service stopped");
}
Future<void> _doWork() async {
final prefController = PrefController(Pref());
final account = prefController.currentAccountValue;
if (account == null) {
_log.shout("[_doWork] account == null");
return;
}
final accountPref = AccountPref.of(account);
final service = FlutterBackgroundService();
final metadataTask = _MetadataTask(service, account, accountPref);
_metadataTaskStateChangedListener.begin();
try {
await metadataTask();
} finally {
_metadataTaskStateChangedListener.end();
}
}
void _onReceiveData(Map<String, dynamic> data) {
try {
final event = data[_dataKeyEvent];
switch (event) {
case _eventStop:
_stopSelf();
break;
default:
_log.severe("[_onReceiveData] Unknown event: $event");
break;
}
} catch (e, stackTrace) {
_log.shout("[_onReceiveData] Uncaught exception", e, stackTrace);
}
}
void _onMetadataTaskStateChanged(MetadataTaskStateChangedEvent ev) {
if (ev.state == _metadataTaskState) {
return;
}
_metadataTaskState = ev.state;
if (_isPaused != true) {
if (ev.state == MetadataTaskState.waitingForWifi) {
FlutterBackgroundService()
..setNotificationInfo(
title: _L10n.global().metadataTaskPauseNoWiFiNotification,
)
..pauseWakeLock();
_isPaused = true;
} else if (ev.state == MetadataTaskState.lowBattery) {
FlutterBackgroundService()
..setNotificationInfo(
title: _L10n.global().metadataTaskPauseLowBatteryNotification,
)
..pauseWakeLock();
_isPaused = true;
}
} else {
if (ev.state == MetadataTaskState.prcoessing) {
FlutterBackgroundService().resumeWakeLock();
_isPaused = false;
}
}
}
void _stopSelf() {
_log.info("[_stopSelf] Stopping service");
FlutterBackgroundService().setNotificationInfo(
title: _L10n.global().backgroundServiceStopping,
);
_shouldRun.value = false;
}
var _metadataTaskState = MetadataTaskState.idle;
late final _metadataTaskStateChangedListener =
AppEventListener<MetadataTaskStateChangedEvent>(
_onMetadataTaskStateChanged);
bool? _isPaused;
static final _shouldRun = ValueNotifier<bool>(true);
}
/// Access localized string out of the main isolate
@npLog
class _L10n {
factory _L10n() => _inst;
_L10n._();
Future<void> init() async {
try {
final locale = language_util.getSelectedLocale();
if (locale == null) {
_l10n = await _queryL10n();
} else {
_l10n = lookupAppLocalizations(locale);
}
} catch (e, stackTrace) {
_log.shout("[init] Uncaught exception", e, stackTrace);
_l10n = AppLocalizationsEn();
}
}
static AppLocalizations global() => _L10n()._l10n;
Future<AppLocalizations> _queryL10n() async {
try {
final locale = await Devicelocale.currentAsLocale;
return lookupAppLocalizations(locale!);
} on FlutterError catch (_) {
// unsupported locale, use default (en)
return AppLocalizationsEn();
} catch (e, stackTrace) {
_log.shout(
"[_queryL10n] Failed while lookupAppLocalizations", e, stackTrace);
return AppLocalizationsEn();
}
}
static final _inst = _L10n._();
late AppLocalizations _l10n;
}
@npLog
class _MetadataTask {
_MetadataTask(this.service, this.account, this.accountPref);
Future<void> call() async {
try {
await _updateMetadata();
} catch (e, stackTrace) {
_log.shout("[call] Uncaught exception", e, stackTrace);
}
if (_processedIds.isNotEmpty) {
unawaited(
MessageRelay.broadcast(FileExifUpdatedEvent(_processedIds).toEvent()),
);
_processedIds = [];
}
final c = KiwiContainer().resolve<DiContainer>();
if (c.fileRepo.dataSrc is FileCachedDataSource) {
await (c.fileRepo.dataSrc as FileCachedDataSource).flushRemoteTouch();
}
}
Future<void> _updateMetadata() async {
final shareFolder = File(
path: file_util.unstripPath(account, accountPref.getShareFolderOr()));
bool hasScanShareFolder = false;
final c = KiwiContainer().resolve<DiContainer>();
final geocoder = ReverseGeocoder();
await geocoder.init();
for (final r in account.roots) {
final dir = File(path: file_util.unstripPath(account, r));
hasScanShareFolder |= file_util.isOrUnderDir(shareFolder, dir);
final updater = UpdateMissingMetadata(
c, const _UpdateMissingMetadataConfigProvider(), geocoder);
void onServiceStop() {
_log.info("[_updateMetadata] Stopping task: user canceled");
updater.stop();
_shouldRun = false;
}
_Service._shouldRun.addListener(onServiceStop);
try {
await for (final ev in updater(account, dir)) {
if (ev is File) {
_onFileProcessed(ev);
}
}
} finally {
_Service._shouldRun.removeListener(onServiceStop);
}
if (!_shouldRun) {
return;
}
}
if (!hasScanShareFolder) {
final shareUpdater = UpdateMissingMetadata(
c, const _UpdateMissingMetadataConfigProvider(), geocoder);
void onServiceStop() {
_log.info("[_updateMetadata] Stopping task: user canceled");
shareUpdater.stop();
_shouldRun = false;
}
_Service._shouldRun.addListener(onServiceStop);
try {
await for (final ev in shareUpdater(
account,
shareFolder,
isRecursive: false,
filter: (f) => f.ownerId != account.userId,
)) {
if (ev is File) {
_onFileProcessed(ev);
}
}
} finally {
_Service._shouldRun.removeListener(onServiceStop);
}
if (!_shouldRun) {
return;
}
}
}
void _onFileProcessed(File file) {
++_count;
service.setNotificationInfo(
title: _L10n.global().metadataTaskProcessingNotification,
content: file.strippedPath,
);
_processedIds.add(file.fileId!);
if (_processedIds.length >= 10) {
MessageRelay.broadcast(FileExifUpdatedEvent(_processedIds).toEvent());
_processedIds = [];
}
}
final FlutterBackgroundService service;
final Account account;
final AccountPref accountPref;
var _shouldRun = true;
var _count = 0;
var _processedIds = <int>[];
}
class _UpdateMissingMetadataConfigProvider
implements UpdateMissingMetadataConfigProvider {
const _UpdateMissingMetadataConfigProvider();
@override
isWifiOnly() =>
Preference.getBool(_servicePref, _servicePrefProcessWifiOnly, true)
.notNull();
}
class _C {
// needed to work with generator logger
static final log = Logger("service");
}
const _dataKeyEvent = "event";
const _eventStop = "stop";
const _servicePref = "service";
const _servicePrefProcessWifiOnly = "shouldProcessWifiOnly";

15
app/lib/service/config.dart Normal file
View file

@ -0,0 +1,15 @@
part of 'service.dart';
class ServiceConfig {
static Future<bool> isProcessExifWifiOnly() async {
return Preference.getBool(_pref, _prefProcessWifiOnly, true)
.notNull();
}
static Future<void> setProcessExifWifiOnly(bool flag) async {
await Preference.setBool(_pref, _prefProcessWifiOnly, flag);
}
static const _pref = "service";
static const _prefProcessWifiOnly = "shouldProcessWifiOnly";
}

43
app/lib/service/l10n.dart Normal file
View file

@ -0,0 +1,43 @@
part of 'service.dart';
/// Access localized string out of the main isolate
@npLog
class _L10n {
_L10n._();
factory _L10n() => _inst;
Future<void> init() async {
try {
final locale = language_util.getSelectedLocale();
if (locale == null) {
_l10n = await _queryL10n();
} else {
_l10n = lookupAppLocalizations(locale);
}
} catch (e, stackTrace) {
_log.shout("[init] Uncaught exception", e, stackTrace);
_l10n = AppLocalizationsEn();
}
}
static AppLocalizations global() => _L10n()._l10n;
Future<AppLocalizations> _queryL10n() async {
try {
final locale = await Devicelocale.currentAsLocale;
return lookupAppLocalizations(locale!);
} on FlutterError catch (_) {
// unsupported locale, use default (en)
return AppLocalizationsEn();
} catch (e, stackTrace) {
_log.shout(
"[_queryL10n] Failed while lookupAppLocalizations", e, stackTrace);
return AppLocalizationsEn();
}
}
late AppLocalizations _l10n;
static final _inst = _L10n._();
}

192
app/lib/service/service.dart Normal file
View file

@ -0,0 +1,192 @@
import 'dart:async';
import 'package:devicelocale/devicelocale.dart';
import 'package:flutter/widgets.dart';
import 'package:flutter_background_service/flutter_background_service.dart';
import 'package:flutter_gen/gen_l10n/app_localizations.dart';
import 'package:flutter_gen/gen_l10n/app_localizations_en.dart';
import 'package:kiwi/kiwi.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/app_init.dart' as app_init;
import 'package:nc_photos/app_localizations.dart';
import 'package:nc_photos/controller/pref_controller.dart';
import 'package:nc_photos/di_container.dart';
import 'package:nc_photos/entity/file_descriptor.dart';
import 'package:nc_photos/entity/pref.dart';
import 'package:nc_photos/event/native_event.dart';
import 'package:nc_photos/language_util.dart' as language_util;
import 'package:nc_photos/use_case/battery_ensurer.dart';
import 'package:nc_photos/use_case/sync_metadata/sync_metadata.dart';
import 'package:nc_photos/use_case/wifi_ensurer.dart';
import 'package:nc_photos_plugin/nc_photos_plugin.dart';
import 'package:np_async/np_async.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_platform_message_relay/np_platform_message_relay.dart';
part 'config.dart';
part 'l10n.dart';
part 'service.g.dart';
/// Start the background service
Future<void> startService() async {
_$__NpLog.log.info("[startService] Starting service");
final service = FlutterBackgroundService();
await service.configure(
androidConfiguration: AndroidConfiguration(
onStart: serviceMain,
autoStart: false,
isForegroundMode: true,
foregroundServiceNotificationTitle:
L10n.global().metadataTaskProcessingNotification,
),
iosConfiguration: IosConfiguration(
onForeground: () => throw UnimplementedError(),
onBackground: () => throw UnimplementedError(),
),
);
// sync settings
await ServiceConfig.setProcessExifWifiOnly(
Pref().shouldProcessExifWifiOnlyOr());
await service.start();
}
/// Ask the background service to stop ASAP
void stopService() {
_$__NpLog.log.info("[stopService] Stopping service");
FlutterBackgroundService().sendData({
_dataKeyEvent: _eventStop,
});
}
@visibleForTesting
@pragma("vm:entry-point")
Future<void> serviceMain() async {
WidgetsFlutterBinding.ensureInitialized();
await _Service()();
}
@npLog
class _Service {
Future<void> call() async {
final service = FlutterBackgroundService();
service.setForegroundMode(true);
await app_init.init(app_init.InitIsolateType.flutterIsolate);
await _L10n().init();
_log.info("[call] Service started");
final onCancelSubscription = service.onCancel.listen((_) {
_log.info("[call] User canceled");
_stopSelf();
});
final onDataSubscription =
service.onDataReceived.listen((event) => _onNotifAction(event ?? {}));
try {
await _doWork();
} catch (e, stackTrace) {
_log.shout("[call] Uncaught exception", e, stackTrace);
}
await onCancelSubscription.cancel();
await onDataSubscription.cancel();
await KiwiContainer().resolve<DiContainer>().npDb.dispose();
service.stopBackgroundService();
_log.info("[call] Service stopped");
}
Future<void> _doWork() async {
final prefController = PrefController(Pref());
final account = prefController.currentAccountValue;
if (account == null) {
_log.shout("[_doWork] account == null");
return;
}
final c = KiwiContainer().resolve<DiContainer>();
final wifiEnsurer = WifiEnsurer(
interrupter: _shouldRun.stream,
);
wifiEnsurer.isWaiting.listen((event) {
if (event) {
FlutterBackgroundService()
..setNotificationInfo(
title: _L10n.global().metadataTaskPauseNoWiFiNotification,
)
..pauseWakeLock();
} else {
FlutterBackgroundService().resumeWakeLock();
}
});
final batteryEnsurer = BatteryEnsurer(
interrupter: _shouldRun.stream,
);
batteryEnsurer.isWaiting.listen((event) {
if (event) {
FlutterBackgroundService()
..setNotificationInfo(
title: _L10n.global().metadataTaskPauseLowBatteryNotification,
)
..pauseWakeLock();
} else {
FlutterBackgroundService().resumeWakeLock();
}
});
final service = FlutterBackgroundService();
final syncOp = SyncMetadata(
fileRepo: c.fileRepo,
fileRepo2: c.fileRepo2,
fileRepoRemote: c.fileRepoRemote,
db: c.npDb,
interrupter: _shouldRun.stream,
wifiEnsurer: wifiEnsurer,
batteryEnsurer: batteryEnsurer,
);
final processedIds = <int>[];
await for (final f in syncOp.syncAccount(account)) {
processedIds.add(f.fdId);
service.setNotificationInfo(
title: _L10n.global().metadataTaskProcessingNotification,
content: f.strippedPath,
);
}
if (processedIds.isNotEmpty) {
await MessageRelay.broadcast(
FileExifUpdatedEvent(processedIds).toEvent());
}
}
void _onNotifAction(Map<String, dynamic> data) {
try {
final event = data[_dataKeyEvent];
switch (event) {
case _eventStop:
_stopSelf();
break;
default:
_log.severe("[_onNotifAction] Unknown event: $event");
break;
}
} catch (e, stackTrace) {
_log.shout("[_onNotifAction] Uncaught exception", e, stackTrace);
}
}
void _stopSelf() {
_log.info("[_stopSelf] Stopping service");
FlutterBackgroundService().setNotificationInfo(
title: _L10n.global().backgroundServiceStopping,
);
_shouldRun.add(null);
}
final _shouldRun = StreamController<void>.broadcast();
}
@npLog
// ignore: camel_case_types
class __ {}
const _dataKeyEvent = "event";
const _eventStop = "stop";

18
app/lib/service.g.dart → app/lib/service/service.g.dart
View file

@ -10,19 +10,19 @@ extension _$_ServiceNpLog on _Service {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("service._Service");
static final log = Logger("service.service._Service");
}
extension _$__NpLog on __ {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("service.service.__");
}
extension _$_L10nNpLog on _L10n {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("service._L10n");
}
extension _$_MetadataTaskNpLog on _MetadataTask {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("service._MetadataTask");
static final log = Logger("service.service._L10n");
}

35
app/lib/use_case/battery_ensurer.dart Normal file
View file

@ -0,0 +1,35 @@
import 'package:battery_plus/battery_plus.dart';
import 'package:nc_photos/exception.dart';
import 'package:rxdart/rxdart.dart';
class BatteryEnsurer {
BatteryEnsurer({
this.interrupter,
}) {
interrupter?.listen((event) {
_shouldRun = false;
});
}
Future<void> call() async {
while (await Battery().batteryLevel <= 15) {
if (!_shouldRun) {
throw const InterruptedException();
}
if (!_isWaiting.value) {
_isWaiting.add(true);
}
await Future.delayed(const Duration(seconds: 5));
}
if (_isWaiting.value) {
_isWaiting.add(false);
}
}
ValueStream<bool> get isWaiting => _isWaiting.stream;
final Stream<void>? interrupter;
var _shouldRun = true;
final _isWaiting = BehaviorSubject.seeded(false);
}

50
app/lib/use_case/scan_missing_metadata.dart
View file

@ -1,50 +0,0 @@
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/exception_event.dart';
import 'package:nc_photos/use_case/ls.dart';
import 'package:nc_photos/use_case/scan_dir.dart';
class ScanMissingMetadata {
ScanMissingMetadata(this.fileRepo);
/// List all files that support metadata but yet having one under a dir
///
/// The returned stream would emit either File data or ExceptionEvent
///
/// If [isRecursive] is true, [root] and its sub dirs will be listed,
/// otherwise only [root] will be listed. Default to true
Stream<dynamic> call(
Account account,
File root, {
bool isRecursive = true,
}) async* {
if (isRecursive) {
yield* _doRecursive(account, root);
} else {
yield* _doSingle(account, root);
}
}
Stream<dynamic> _doRecursive(Account account, File root) async* {
final dataStream = ScanDir(fileRepo)(account, root);
await for (final d in dataStream) {
if (d is ExceptionEvent) {
yield d;
continue;
}
for (final f in (d as List<File>).where(file_util.isMissingMetadata)) {
yield f;
}
}
}
Stream<dynamic> _doSingle(Account account, File root) async* {
final files = await Ls(fileRepo)(account, root);
for (final f in files.where(file_util.isMissingMetadata)) {
yield f;
}
}
final FileRepo fileRepo;
}

120
app/lib/use_case/sync_metadata/sync_by_app.dart Normal file
View file

@ -0,0 +1,120 @@
part of 'sync_metadata.dart';
/// Sync metadata using the client side logic
@npLog
class _SyncByApp {
_SyncByApp({
required this.account,
required this.fileRepo,
required this.fileRepo2,
required this.db,
this.interrupter,
required this.wifiEnsurer,
required this.batteryEnsurer,
}) {
interrupter?.listen((event) {
_shouldRun = false;
});
}
Future<void> init() async {
await _geocoder.init();
}
Stream<File> syncFiles({
required List<int> fileIds,
}) async* {
for (final ids in partition(fileIds, 100)) {
yield* _syncGroup(ids);
}
}
Stream<File> _syncGroup(List<int> fileIds) async* {
final files = await db.getFilesByFileIds(
account: account.toDb(),
fileIds: fileIds,
);
for (final f in files) {
final result = await _syncOne(f);
if (result != null) {
yield result;
}
if (!_shouldRun) {
return;
}
}
}
Future<File?> _syncOne(DbFile file) async {
final f = DbFileConverter.fromDb(
account.userId.toCaseInsensitiveString(),
file,
);
_log.fine("[_syncOne] Syncing ${file.relativePath}");
try {
OrNull<Metadata>? metadataUpdate;
OrNull<ImageLocation>? locationUpdate;
if (f.metadata == null) {
// since we need to download multiple images in their original size,
// we only do it with WiFi
await wifiEnsurer();
await batteryEnsurer();
if (!_shouldRun) {
return null;
}
_log.fine("[_syncOne] Updating metadata for ${f.path}");
final binary = await GetFileBinary(fileRepo)(account, f);
final metadata =
(await LoadMetadata().loadRemote(account, f, binary)).copyWith(
fileEtag: f.etag,
);
metadataUpdate = OrNull(metadata);
}
final lat = (metadataUpdate?.obj ?? f.metadata)?.exif?.gpsLatitudeDeg;
final lng = (metadataUpdate?.obj ?? f.metadata)?.exif?.gpsLongitudeDeg;
try {
ImageLocation? location;
if (lat != null && lng != null) {
_log.fine("[_syncOne] Reverse geocoding for ${f.path}");
final l = await _geocoder(lat, lng);
if (l != null) {
location = l.toImageLocation();
}
}
locationUpdate = OrNull(location ?? ImageLocation.empty());
} catch (e, stackTrace) {
_log.severe("[_syncOne] Failed while reverse geocoding: ${f.path}", e,
stackTrace);
// if failed, we skip updating the location
}
if (metadataUpdate != null || locationUpdate != null) {
await UpdateProperty(fileRepo: fileRepo2)(
account,
f,
metadata: metadataUpdate,
location: locationUpdate,
);
return f;
} else {
return null;
}
} catch (e, stackTrace) {
_log.severe("[_syncOne] Failed while updating metadata: ${f.path}", e,
stackTrace);
return null;
}
}
final Account account;
final FileRepo fileRepo;
final FileRepo2 fileRepo2;
final NpDb db;
final Stream<void>? interrupter;
final WifiEnsurer wifiEnsurer;
final BatteryEnsurer batteryEnsurer;
final _geocoder = ReverseGeocoder();
var _shouldRun = true;
}

91
app/lib/use_case/sync_metadata/sync_by_server.dart Normal file
View file

@ -0,0 +1,91 @@
part of 'sync_metadata.dart';
/// Sync metadata using the client side logic
@npLog
class _SyncByServer {
_SyncByServer({
required this.account,
required this.fileRepoRemote,
required this.fileRepo2,
required this.db,
this.interrupter,
}) {
interrupter?.listen((event) {
_shouldRun = false;
});
}
Future<void> init() async {
await _geocoder.init();
}
Stream<File> syncFiles({
required List<String> relativePaths,
}) async* {
final dirs = relativePaths.map(dirname).toSet();
for (final dir in dirs) {
yield* _syncDir(
dir: File(path: file_util.unstripPath(account, dir)),
);
}
}
Stream<File> _syncDir({
required File dir,
}) async* {
try {
_log.fine("[_syncDir] Syncing dir $dir");
final files = await fileRepoRemote.list(account, dir);
await FileSqliteCacheUpdater(db)(account, dir, remote: files);
for (final f in files) {
if (f.metadata != null && f.location == null) {
final result = await _syncOne(f);
if (result != null) {
yield result;
}
if (!_shouldRun) {
return;
}
}
}
} catch (e, stackTrace) {
_log.severe("[_syncDir] Failed to sync dir: $dir", e, stackTrace);
}
}
Future<File?> _syncOne(File file) async {
try {
final lat = file.metadata!.exif?.gpsLatitudeDeg;
final lng = file.metadata!.exif?.gpsLongitudeDeg;
ImageLocation? location;
if (lat != null && lng != null) {
_log.fine("[_syncOne] Reverse geocoding for ${file.path}");
final l = await _geocoder(lat, lng);
if (l != null) {
location = l.toImageLocation();
}
}
final locationUpdate = OrNull(location ?? ImageLocation.empty());
await UpdateProperty(fileRepo: fileRepo2)(
account,
file,
metadata: OrNull(file.metadata),
location: locationUpdate,
);
return file;
} catch (e, stackTrace) {
_log.severe("[_syncOne] Failed while updating location: ${file.path}", e,
stackTrace);
return null;
}
}
final Account account;
final FileRepo fileRepoRemote;
final FileRepo2 fileRepo2;
final NpDb db;
final Stream<void>? interrupter;
final _geocoder = ReverseGeocoder();
var _shouldRun = true;
}

109
app/lib/use_case/sync_metadata/sync_metadata.dart Normal file
View file

@ -0,0 +1,109 @@
import 'dart:async';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/controller/server_controller.dart';
import 'package:nc_photos/db/entity_converter.dart';
import 'package:nc_photos/entity/exif_util.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/file_cache_manager.dart';
import 'package:nc_photos/entity/file/repo.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/geocoder_util.dart';
import 'package:nc_photos/use_case/battery_ensurer.dart';
import 'package:nc_photos/use_case/get_file_binary.dart';
import 'package:nc_photos/use_case/load_metadata.dart';
import 'package:nc_photos/use_case/update_property.dart';
import 'package:nc_photos/use_case/wifi_ensurer.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_collection/np_collection.dart';
import 'package:np_common/or_null.dart';
import 'package:np_db/np_db.dart';
import 'package:np_geocoder/np_geocoder.dart';
import 'package:path/path.dart';
part 'sync_by_app.dart';
part 'sync_by_server.dart';
part 'sync_metadata.g.dart';
@npLog
class SyncMetadata {
const SyncMetadata({
required this.fileRepo,
required this.fileRepo2,
required this.fileRepoRemote,
required this.db,
this.interrupter,
required this.wifiEnsurer,
required this.batteryEnsurer,
});
Stream<File> syncAccount(Account account) async* {
final bool isNcMetadataSupported;
try {
isNcMetadataSupported = await _isNcMetadataSupported(account);
} catch (e) {
_log.severe("[syncAccount] Failed to get server version", e);
return;
}
final files = await db.getFilesByMissingMetadata(
account: account.toDb(),
mimes: file_util.metadataSupportedFormatMimes,
ownerId: account.userId.toCaseInsensitiveString(),
);
_log.info("[syncAccount] Missing count: ${files.items.length}");
if (isNcMetadataSupported) {
yield* _doWithServer(account, files);
} else {
yield* _doWithApp(account, files);
}
}
Stream<File> _doWithApp(
Account account, DbFileMissingMetadataResult files) async* {
final op = _SyncByApp(
account: account,
fileRepo: fileRepo,
fileRepo2: fileRepo2,
db: db,
interrupter: interrupter,
wifiEnsurer: wifiEnsurer,
batteryEnsurer: batteryEnsurer,
);
await op.init();
final stream = op.syncFiles(
fileIds: files.items.map((e) => e.fileId).toList(),
);
yield* stream;
}
Stream<File> _doWithServer(
Account account, DbFileMissingMetadataResult files) async* {
final op = _SyncByServer(
account: account,
fileRepoRemote: fileRepoRemote,
fileRepo2: fileRepo2,
db: db,
interrupter: interrupter,
);
await op.init();
final stream = op.syncFiles(
relativePaths: files.items.map((e) => e.relativePath).toList(),
);
yield* stream;
}
Future<bool> _isNcMetadataSupported(Account account) async {
final serverController = ServerController(account: account);
await serverController.status.first.timeout(const Duration(seconds: 15));
return serverController.isSupported(ServerFeature.ncMetadata);
}
final FileRepo fileRepo;
final FileRepo2 fileRepo2;
final FileRepo fileRepoRemote;
final NpDb db;
final Stream<void>? interrupter;
final WifiEnsurer wifiEnsurer;
final BatteryEnsurer batteryEnsurer;
}

30
app/lib/use_case/sync_metadata/sync_metadata.g.dart Normal file
View file

@ -0,0 +1,30 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'sync_metadata.dart';
// **************************************************************************
// NpLogGenerator
// **************************************************************************
extension _$SyncMetadataNpLog on SyncMetadata {
// ignore: unused_element
Logger get _log => log;
static final log =
Logger("use_case.sync_metadata.sync_metadata.SyncMetadata");
}
extension _$_SyncByAppNpLog on _SyncByApp {
// ignore: unused_element
Logger get _log => log;
static final log = Logger("use_case.sync_metadata.sync_metadata._SyncByApp");
}
extension _$_SyncByServerNpLog on _SyncByServer {
// ignore: unused_element
Logger get _log => log;
static final log =
Logger("use_case.sync_metadata.sync_metadata._SyncByServer");
}

173
app/lib/use_case/update_missing_metadata.dart
View file

@ -1,173 +0,0 @@
import 'package:battery_plus/battery_plus.dart';
import 'package:event_bus/event_bus.dart';
import 'package:kiwi/kiwi.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/connectivity_util.dart' as connectivity_util;
import 'package:nc_photos/di_container.dart';
import 'package:nc_photos/entity/exif_util.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/event/event.dart';
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/exception_event.dart';
import 'package:nc_photos/geocoder_util.dart';
import 'package:nc_photos/use_case/get_file_binary.dart';
import 'package:nc_photos/use_case/load_metadata.dart';
import 'package:nc_photos/use_case/scan_missing_metadata.dart';
import 'package:nc_photos/use_case/update_property.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_common/or_null.dart';
import 'package:np_geocoder/np_geocoder.dart';
part 'update_missing_metadata.g.dart';
abstract class UpdateMissingMetadataConfigProvider {
Future<bool> isWifiOnly();
}
@npLog
class UpdateMissingMetadata {
UpdateMissingMetadata(this._c, this.configProvider, this.geocoder);
/// Update metadata for all files that support one under a dir
///
/// The returned stream would emit either File data (for each updated files)
/// or ExceptionEvent
///
/// If [isRecursive] is true, [root] and its sub dirs will be scanned,
/// otherwise only [root] will be scanned. Default to true
///
/// [filter] can be used to filter files -- return true if a file should be
/// included. If [filter] is null, all files will be included.
Stream<dynamic> call(
Account account,
File root, {
bool isRecursive = true,
bool Function(File file)? filter,
}) async* {
final dataStream = ScanMissingMetadata(_c.fileRepo)(
account,
root,
isRecursive: isRecursive,
);
await for (final d in dataStream) {
if (!_shouldRun) {
return;
}
if (d is ExceptionEvent) {
yield d;
continue;
}
final File file = d;
// check if this is a federation share. Nextcloud doesn't support
// properties for such files
if (file.ownerId?.contains("/") == true || filter?.call(d) == false) {
continue;
}
try {
OrNull<Metadata>? metadataUpdate;
OrNull<ImageLocation>? locationUpdate;
if (file.metadata == null) {
// since we need to download multiple images in their original size,
// we only do it with WiFi
await _ensureWifi();
await _ensureBattery();
KiwiContainer().resolve<EventBus>().fire(
const MetadataTaskStateChangedEvent(
MetadataTaskState.prcoessing));
if (!_shouldRun) {
return;
}
_log.fine("[call] Updating metadata for ${file.path}");
final binary = await GetFileBinary(_c.fileRepo)(account, file);
final metadata =
(await LoadMetadata().loadRemote(account, file, binary)).copyWith(
fileEtag: file.etag,
);
metadataUpdate = OrNull(metadata);
} else {
_log.finer("[call] Skip updating metadata for ${file.path}");
KiwiContainer().resolve<EventBus>().fire(
const MetadataTaskStateChangedEvent(
MetadataTaskState.prcoessing));
}
final lat =
(metadataUpdate?.obj ?? file.metadata)?.exif?.gpsLatitudeDeg;
final lng =
(metadataUpdate?.obj ?? file.metadata)?.exif?.gpsLongitudeDeg;
try {
ImageLocation? location;
if (lat != null && lng != null) {
_log.fine("[call] Reverse geocoding for ${file.path}");
final l = await geocoder(lat, lng);
if (l != null) {
location = l.toImageLocation();
}
}
locationUpdate = OrNull(location ?? ImageLocation.empty());
} catch (e, stackTrace) {
_log.severe("[call] Failed while reverse geocoding: ${file.path}", e,
stackTrace);
}
if (metadataUpdate != null || locationUpdate != null) {
await UpdateProperty(_c)(
account,
file,
metadata: metadataUpdate,
location: locationUpdate,
);
yield file;
}
// slow down a bit to give some space for the main isolate
await Future.delayed(const Duration(milliseconds: 10));
} on InterruptedException catch (_) {
return;
} catch (e, stackTrace) {
_log.severe("[call] Failed while updating metadata: ${file.path}", e,
stackTrace);
yield ExceptionEvent(e, stackTrace);
}
}
}
void stop() {
_shouldRun = false;
}
Future<void> _ensureWifi() async {
var count = 0;
while (await configProvider.isWifiOnly() &&
!await connectivity_util.isWifi()) {
if (!_shouldRun) {
throw const InterruptedException();
}
// give a chance to reconnect with the WiFi network
if (++count >= 6) {
KiwiContainer().resolve<EventBus>().fire(
const MetadataTaskStateChangedEvent(
MetadataTaskState.waitingForWifi));
}
await Future.delayed(const Duration(seconds: 5));
}
}
Future<void> _ensureBattery() async {
while (await Battery().batteryLevel <= 15) {
if (!_shouldRun) {
throw const InterruptedException();
}
KiwiContainer().resolve<EventBus>().fire(
const MetadataTaskStateChangedEvent(MetadataTaskState.lowBattery));
await Future.delayed(const Duration(seconds: 5));
}
}
final DiContainer _c;
final UpdateMissingMetadataConfigProvider configProvider;
final ReverseGeocoder geocoder;
bool _shouldRun = true;
}

15
app/lib/use_case/update_missing_metadata.g.dart
View file

@ -1,15 +0,0 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'update_missing_metadata.dart';
// **************************************************************************
// NpLogGenerator
// **************************************************************************
extension _$UpdateMissingMetadataNpLog on UpdateMissingMetadata {
// ignore: unused_element
Logger get _log => log;
static final log =
Logger("use_case.update_missing_metadata.UpdateMissingMetadata");
}

57
app/lib/use_case/update_property.dart
View file

@ -1,11 +1,8 @@
import 'package:event_bus/event_bus.dart';
import 'package:kiwi/kiwi.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/di_container.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/repo.dart';
import 'package:nc_photos/entity/file_descriptor.dart';
import 'package:nc_photos/event/event.dart';
import 'package:np_codegen/np_codegen.dart';
import 'package:np_common/or_null.dart';
@ -13,7 +10,11 @@ part 'update_property.g.dart';
@npLog
class UpdateProperty {
const UpdateProperty(this._c);
const UpdateProperty({
required this.fileRepo,
});
final FileRepo2 fileRepo;
Future<void> call(
Account account,
@ -34,17 +35,7 @@ class UpdateProperty {
return;
}
await _c.fileRepo2.updateProperty(
account,
file,
metadata: metadata,
isArchived: isArchived,
overrideDateTime: overrideDateTime,
favorite: favorite,
location: location,
);
_notify(
await fileRepo.updateProperty(
account,
file,
metadata: metadata,
@ -54,40 +45,6 @@ class UpdateProperty {
location: location,
);
}
@Deprecated("legacy")
void _notify(
Account account,
FileDescriptor file, {
OrNull<Metadata>? metadata,
OrNull<bool>? isArchived,
OrNull<DateTime>? overrideDateTime,
bool? favorite,
OrNull<ImageLocation>? location,
}) {
int properties = 0;
if (metadata != null) {
properties |= FilePropertyUpdatedEvent.propMetadata;
}
if (isArchived != null) {
properties |= FilePropertyUpdatedEvent.propIsArchived;
}
if (overrideDateTime != null) {
properties |= FilePropertyUpdatedEvent.propOverrideDateTime;
}
if (favorite != null) {
properties |= FilePropertyUpdatedEvent.propFavorite;
}
if (location != null) {
properties |= FilePropertyUpdatedEvent.propImageLocation;
}
assert(properties != 0);
KiwiContainer()
.resolve<EventBus>()
.fire(FilePropertyUpdatedEvent(account, file, properties));
}
final DiContainer _c;
}
extension UpdatePropertyExtension on UpdateProperty {

41
app/lib/use_case/wifi_ensurer.dart Normal file
View file

@ -0,0 +1,41 @@
import 'package:nc_photos/connectivity_util.dart' as connectivity_util;
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/service/service.dart';
import 'package:rxdart/rxdart.dart';
class WifiEnsurer {
WifiEnsurer({
this.interrupter,
}) {
interrupter?.listen((event) {
_shouldRun = false;
});
}
Future<void> call() async {
var count = 0;
while (await ServiceConfig.isProcessExifWifiOnly() &&
!await connectivity_util.isWifi()) {
if (!_shouldRun) {
throw const InterruptedException();
}
// give a chance to reconnect with the WiFi network
if (++count >= 6) {
if (!_isWaiting.value) {
_isWaiting.add(true);
}
}
await Future.delayed(const Duration(seconds: 5));
}
if (_isWaiting.value) {
_isWaiting.add(false);
}
}
ValueStream<bool> get isWaiting => _isWaiting.stream;
final Stream<void>? interrupter;
var _shouldRun = true;
final _isWaiting = BehaviorSubject.seeded(false);
}

6
app/lib/widget/home_photos/bloc.dart
View file

@ -536,11 +536,7 @@ class _Bloc extends Bloc<_Event, _State>
personsController: personsController,
personProvider: accountPrefController.personProviderValue,
);
if (!serverController.isSupported(ServerFeature.ncMetadata)) {
metadataController.kickstart();
} else {
_log.info("[_syncRemote] Skipping metadata service");
}
metadataController.kickstart();
_log.info(
"[_syncRemote] Elapsed time: ${stopwatch.elapsedMilliseconds}ms");
});

2
app/lib/widget/settings/metadata_settings.dart
View file

@ -6,7 +6,7 @@ import 'package:nc_photos/app_localizations.dart';
import 'package:nc_photos/bloc_util.dart';
import 'package:nc_photos/controller/pref_controller.dart';
import 'package:nc_photos/exception_event.dart';
import 'package:nc_photos/service.dart';
import 'package:nc_photos/service/service.dart';
import 'package:nc_photos/snack_bar_manager.dart';
import 'package:nc_photos/widget/page_visibility_mixin.dart';
import 'package:np_codegen/np_codegen.dart';

2
app/lib/widget/viewer_detail_pane.dart
View file

@ -505,7 +505,7 @@ class _ViewerDetailPaneState extends State<ViewerDetailPane> {
return;
}
try {
await UpdateProperty(_c)
await UpdateProperty(fileRepo: _c.fileRepo2)
.updateOverrideDateTime(widget.account, _file!, value);
if (mounted) {
setState(() {

28
app/test/entity/file/file_cache_manager_test.dart
View file

@ -205,7 +205,7 @@ Future<void> _updaterIdentical() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(account, files[0], remote: files.slice(0, 3));
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -240,7 +240,7 @@ Future<void> _updaterNewFile() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(account, files[0], remote: [...files.slice(0, 3), newFile]);
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -271,7 +271,7 @@ Future<void> _updaterDeleteFile() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(account, files[0], remote: [files[0], files[2]]);
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -305,7 +305,7 @@ Future<void> _updaterDeleteDir() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(account, files[0], remote: files.slice(0, 2));
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -343,7 +343,7 @@ Future<void> _updaterUpdateFile() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(account, files[0],
remote: [files[0], newFile, ...files.slice(2)]);
expect(
@ -382,7 +382,7 @@ Future<void> _updaterNewSharedFile() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(user1Account, user1Files[0], remote: user1Files);
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -419,7 +419,7 @@ Future<void> _updaterNewSharedDir() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], [files[3]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(user1Account, user1Files[0], remote: user1Files);
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -461,7 +461,7 @@ Future<void> _updaterDeleteSharedFile() async {
c.sqliteDb, user1Account, user1Files[0], [user1Files[1]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(user1Account, user1Files[0], remote: [user1Files[0]]);
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -504,7 +504,7 @@ Future<void> _updaterDeleteSharedDir() async {
c.sqliteDb, user1Account, user1Files[0], [user1Files[1]]);
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(user1Account, user1Files[0], remote: [user1Files[0]]);
expect(
await util.listSqliteDbFiles(c.sqliteDb),
@ -541,7 +541,7 @@ Future<void> _updaterTooManyFiles() async {
await util.insertDirRelation(c.sqliteDb, account, files[2], files.slice(3));
});
final updater = FileSqliteCacheUpdater(c);
final updater = FileSqliteCacheUpdater(c.npDb);
await updater(account, files[2], remote: [...files.slice(2), ...newFiles]);
// we are testing to make sure the above function won't throw, so nothing to
// expect here
@ -574,12 +574,12 @@ Future<void> _updaterMovedFileToFront() async {
final movedFile = files[3].copyWith(
path: "remote.php/dav/files/admin/test1/test1.jpg",
);
await FileSqliteCacheUpdater(c)(
await FileSqliteCacheUpdater(c.npDb)(
account,
files[1],
remote: [files[1], movedFile],
);
await FileSqliteCacheUpdater(c)(
await FileSqliteCacheUpdater(c.npDb)(
account,
files[2],
remote: [files[2]],
@ -621,12 +621,12 @@ Future<void> _updaterMovedFileToBehind() async {
final movedFile = files[3].copyWith(
path: "remote.php/dav/files/admin/test2/test1.jpg",
);
await FileSqliteCacheUpdater(c)(
await FileSqliteCacheUpdater(c.npDb)(
account,
files[1],
remote: [files[1]],
);
await FileSqliteCacheUpdater(c)(
await FileSqliteCacheUpdater(c.npDb)(
account,
files[2],
remote: [files[2], movedFile],

2
np_collection/lib/np_collection.dart
View file

@ -1,5 +1,7 @@
library np_collection;
export 'package:quiver/iterables.dart' show partition;
export 'src/iterable_extension.dart';
export 'src/iterator_extension.dart';
export 'src/list_extension.dart';

20
np_db/lib/src/api.dart
View file

@ -201,6 +201,19 @@ class DbFilesMemory {
final Map<int, List<DbFileDescriptor>> memories;
}
@genCopyWith
@toString
class DbFileMissingMetadataResult {
const DbFileMissingMetadataResult({
required this.items,
});
@override
String toString() => _$toString();
final List<({int fileId, String relativePath})> items;
}
@npLog
abstract class NpDb {
factory NpDb() => NpDbSqlite();
@ -354,6 +367,13 @@ abstract class NpDb {
required String ownerId,
});
/// Return files without metadata
Future<DbFileMissingMetadataResult> getFilesByMissingMetadata({
required DbAccount account,
required List<String> mimes,
required String ownerId,
});
/// Delete a file or dir from db
Future<void> deleteFile({
required DbAccount account,

32
np_db/lib/src/api.g.dart
View file

@ -81,6 +81,31 @@ extension $DbFilesMemoryCopyWith on DbFilesMemory {
_$DbFilesMemoryCopyWithWorkerImpl(this);
}
abstract class $DbFileMissingMetadataResultCopyWithWorker {
DbFileMissingMetadataResult call(
{List<({int fileId, String relativePath})>? items});
}
class _$DbFileMissingMetadataResultCopyWithWorkerImpl
implements $DbFileMissingMetadataResultCopyWithWorker {
_$DbFileMissingMetadataResultCopyWithWorkerImpl(this.that);
@override
DbFileMissingMetadataResult call({dynamic items}) {
return DbFileMissingMetadataResult(
items:
items as List<({int fileId, String relativePath})>? ?? that.items);
}
final DbFileMissingMetadataResult that;
}
extension $DbFileMissingMetadataResultCopyWith on DbFileMissingMetadataResult {
$DbFileMissingMetadataResultCopyWithWorker get copyWith => _$copyWith;
$DbFileMissingMetadataResultCopyWithWorker get _$copyWith =>
_$DbFileMissingMetadataResultCopyWithWorkerImpl(this);
}
// **************************************************************************
// NpLogGenerator
// **************************************************************************
@ -151,3 +176,10 @@ extension _$DbFilesMemoryToString on DbFilesMemory {
return "DbFilesMemory {memories: $memories}";
}
}
extension _$DbFileMissingMetadataResultToString on DbFileMissingMetadataResult {
String _$toString() {
// ignore: unnecessary_string_interpolations
return "DbFileMissingMetadataResult {items: [length: ${items.length}]}";
}
}

65
np_db_sqlite/lib/src/database/file_extension.dart
View file

@ -343,7 +343,7 @@ extension SqliteDbFileExtension on SqliteDb {
String? ownerId,
}) async {
_log.info(
"[countFiles] isMissingMetadata: $isMissingMetadata, mimes: $mimes");
"[countFiles] isMissingMetadata: $isMissingMetadata, mimes: $mimes, ownerId: $ownerId");
Expression<bool>? filter;
if (isMissingMetadata != null) {
if (isMissingMetadata) {
@ -387,6 +387,69 @@ extension SqliteDbFileExtension on SqliteDb {
return await query.map((r) => r.read(count)!).getSingle();
}
Future<List<({int fileId, String relativePath})>>
queryFileIdPathsByMissingMetadata({
required ByAccount account,
bool? isMissingMetadata,
List<String>? mimes,
String? ownerId,
int? offset,
int? limit,
}) async {
_log.info(
"[queryFileIdPathsByMissingMetadata] isMissingMetadata: $isMissingMetadata, mimes: $mimes, ownerId: $ownerId");
final query = selectOnly(files).join([
innerJoin(accountFiles, accountFiles.file.equalsExp(files.rowId),
useColumns: false),
if (account.dbAccount != null) ...[
innerJoin(accounts, accounts.rowId.equalsExp(accountFiles.account),
useColumns: false),
innerJoin(servers, servers.rowId.equalsExp(accounts.server),
useColumns: false),
],
leftOuterJoin(images, images.accountFile.equalsExp(accountFiles.rowId),
useColumns: false),
leftOuterJoin(imageLocations,
imageLocations.accountFile.equalsExp(accountFiles.rowId),
useColumns: false),
]);
query.addColumns([files.fileId, accountFiles.relativePath]);
if (account.sqlAccount != null) {
query.where(accountFiles.account.equals(account.sqlAccount!.rowId));
} else if (account.dbAccount != null) {
query
..where(servers.address.equals(account.dbAccount!.serverAddress))
..where(accounts.userId
.equals(account.dbAccount!.userId.toCaseInsensitiveString()));
}
if (mimes != null) {
query.where(files.contentType.isIn(mimes));
}
if (ownerId != null) {
query.where(files.ownerId.equals(ownerId));
}
if (isMissingMetadata != null) {
if (isMissingMetadata) {
query.where(
images.lastUpdated.isNull() | imageLocations.version.isNull());
} else {
query.where(images.lastUpdated.isNotNull() &
imageLocations.version.isNotNull());
}
}
query.orderBy([OrderingTerm.desc(files.fileId)]);
if (limit != null) {
query.limit(limit, offset: offset);
}
return await query
.map((r) => (
fileId: r.read(files.fileId)!,
relativePath: r.read(accountFiles.relativePath)!,
))
.get();
}
Future<List<FileDescriptor>> queryFileDescriptors({
required ByAccount account,
List<int>? fileIds,

28
np_db_sqlite/lib/src/sqlite_api.dart
View file

@ -395,6 +395,34 @@ class NpDbSqlite implements NpDb {
});
}
@override
Future<DbFileMissingMetadataResult> getFilesByMissingMetadata({
required DbAccount account,
required List<String> mimes,
required String ownerId,
}) async {
return _db.use((db) async {
final results = <({int fileId, String relativePath})>[];
var i = 0;
while (true) {
final sqlObjs = await db.queryFileIdPathsByMissingMetadata(
account: ByAccount.db(account),
isMissingMetadata: true,
mimes: mimes,
ownerId: ownerId,
limit: 10000,
offset: i,
);
if (sqlObjs.isEmpty) {
break;
}
results.addAll(sqlObjs);
i += 10000;
}
return DbFileMissingMetadataResult(items: results);
});
}
@override
Future<void> deleteFile({
required DbAccount account,