mirror of
https://gitlab.com/nkming2/nc-photos.git
synced 2025-02-02 06:46:22 +01:00
Rework touch hack to improve startup performance
This commit is contained in:
parent
c7322bccff
commit
241b1c4775
8 changed files with 254 additions and 292 deletions
|
@ -40,6 +40,7 @@ import 'package:nc_photos/platform/features.dart' as features;
|
||||||
import 'package:nc_photos/platform/k.dart' as platform_k;
|
import 'package:nc_photos/platform/k.dart' as platform_k;
|
||||||
import 'package:nc_photos/pref.dart';
|
import 'package:nc_photos/pref.dart';
|
||||||
import 'package:nc_photos/pref_util.dart' as pref_util;
|
import 'package:nc_photos/pref_util.dart' as pref_util;
|
||||||
|
import 'package:nc_photos/touch_manager.dart';
|
||||||
import 'package:visibility_detector/visibility_detector.dart';
|
import 'package:visibility_detector/visibility_detector.dart';
|
||||||
|
|
||||||
enum InitIsolateType {
|
enum InitIsolateType {
|
||||||
|
@ -213,6 +214,7 @@ Future<void> _initDiContainer(InitIsolateType isolateType) async {
|
||||||
c.tagRepoLocal = TagRepo(TagSqliteDbDataSource(c.sqliteDb));
|
c.tagRepoLocal = TagRepo(TagSqliteDbDataSource(c.sqliteDb));
|
||||||
c.taggedFileRepo = const TaggedFileRepo(TaggedFileRemoteDataSource());
|
c.taggedFileRepo = const TaggedFileRepo(TaggedFileRemoteDataSource());
|
||||||
c.searchRepo = SearchRepo(SearchSqliteDbDataSource(c));
|
c.searchRepo = SearchRepo(SearchSqliteDbDataSource(c));
|
||||||
|
c.touchManager = TouchManager(c);
|
||||||
|
|
||||||
if (platform_k.isAndroid) {
|
if (platform_k.isAndroid) {
|
||||||
// local file currently only supported on Android
|
// local file currently only supported on Android
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
import 'dart:async';
|
import 'dart:async';
|
||||||
|
|
||||||
import 'package:bloc/bloc.dart';
|
import 'package:bloc/bloc.dart';
|
||||||
import 'package:collection/collection.dart';
|
|
||||||
import 'package:kiwi/kiwi.dart';
|
import 'package:kiwi/kiwi.dart';
|
||||||
import 'package:logging/logging.dart';
|
import 'package:logging/logging.dart';
|
||||||
import 'package:nc_photos/account.dart';
|
import 'package:nc_photos/account.dart';
|
||||||
|
@ -17,7 +16,6 @@ import 'package:nc_photos/exception_event.dart';
|
||||||
import 'package:nc_photos/platform/k.dart' as platform_k;
|
import 'package:nc_photos/platform/k.dart' as platform_k;
|
||||||
import 'package:nc_photos/pref.dart';
|
import 'package:nc_photos/pref.dart';
|
||||||
import 'package:nc_photos/throttler.dart';
|
import 'package:nc_photos/throttler.dart';
|
||||||
import 'package:nc_photos/touch_token_manager.dart';
|
|
||||||
import 'package:nc_photos/use_case/ls.dart';
|
import 'package:nc_photos/use_case/ls.dart';
|
||||||
import 'package:nc_photos/use_case/scan_dir.dart';
|
import 'package:nc_photos/use_case/scan_dir.dart';
|
||||||
import 'package:nc_photos/use_case/scan_dir_offline.dart';
|
import 'package:nc_photos/use_case/scan_dir_offline.dart';
|
||||||
|
@ -135,7 +133,9 @@ class ScanAccountDirBloc
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool require(DiContainer c) => DiContainer.has(c, DiType.fileRepo);
|
static bool require(DiContainer c) =>
|
||||||
|
DiContainer.has(c, DiType.fileRepo) &&
|
||||||
|
DiContainer.has(c, DiType.touchManager);
|
||||||
|
|
||||||
static ScanAccountDirBloc of(Account account) {
|
static ScanAccountDirBloc of(Account account) {
|
||||||
final name =
|
final name =
|
||||||
|
@ -373,98 +373,35 @@ class ScanAccountDirBloc
|
||||||
// 1st pass: scan for new files
|
// 1st pass: scan for new files
|
||||||
var files = <File>[];
|
var files = <File>[];
|
||||||
final cacheMap = FileForwardCacheManager.prepareFileMap(cache);
|
final cacheMap = FileForwardCacheManager.prepareFileMap(cache);
|
||||||
{
|
final stopwatch = Stopwatch()..start();
|
||||||
final stopwatch = Stopwatch()..start();
|
_c.touchManager.clearTouchCache();
|
||||||
final fileRepo = FileRepo(FileCachedDataSource(
|
|
||||||
_c,
|
|
||||||
forwardCacheManager: FileForwardCacheManager(_c, cacheMap),
|
|
||||||
));
|
|
||||||
await for (final event in _queryWithFileRepo(fileRepo, ev,
|
|
||||||
fileRepoForShareDir: _c.fileRepo)) {
|
|
||||||
if (event is ExceptionEvent) {
|
|
||||||
_log.shout("[_queryOnline] Exception while request (1st pass)",
|
|
||||||
event.error, event.stackTrace);
|
|
||||||
emit(ScanAccountDirBlocFailure(
|
|
||||||
cache.isEmpty
|
|
||||||
? files
|
|
||||||
: cache.where((f) => file_util.isSupportedFormat(f)).toList(),
|
|
||||||
event.error));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
files.addAll(event);
|
|
||||||
if (cache.isEmpty) {
|
|
||||||
// only emit partial results if there's no cache
|
|
||||||
emit(ScanAccountDirBlocLoading(files.toList()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_log.info(
|
|
||||||
"[_queryOnline] Elapsed time (pass1): ${stopwatch.elapsedMilliseconds}ms");
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
if (_shouldCheckCache) {
|
|
||||||
// 2nd pass: check outdated cache
|
|
||||||
_shouldCheckCache = false;
|
|
||||||
|
|
||||||
// announce the result of the 1st pass
|
|
||||||
// if cache is empty, we have already emitted the results in the loop
|
|
||||||
if (cache.isNotEmpty || files.isEmpty) {
|
|
||||||
// emit results from remote
|
|
||||||
emit(ScanAccountDirBlocLoading(files));
|
|
||||||
}
|
|
||||||
|
|
||||||
// files = await _queryOnlinePass2(ev, cacheMap, files);
|
|
||||||
files = await _queryOnlinePass2(ev, {}, files);
|
|
||||||
}
|
|
||||||
} catch (e, stackTrace) {
|
|
||||||
_log.shout(
|
|
||||||
"[_queryOnline] Failed while _queryOnlinePass2", e, stackTrace);
|
|
||||||
}
|
|
||||||
emit(ScanAccountDirBlocSuccess(files));
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<List<File>> _queryOnlinePass2(ScanAccountDirBlocQueryBase ev,
|
|
||||||
Map<int, File> cacheMap, List<File> pass1Files) async {
|
|
||||||
final touchTokenManager = TouchTokenManager(_c);
|
|
||||||
// combine the file maps because [pass1Files] doesn't contain non-supported
|
|
||||||
// files
|
|
||||||
final pass2CacheMap = CombinedMapView(
|
|
||||||
[FileForwardCacheManager.prepareFileMap(pass1Files), cacheMap]);
|
|
||||||
final fileRepo = FileRepo(FileCachedDataSource(
|
final fileRepo = FileRepo(FileCachedDataSource(
|
||||||
_c,
|
_c,
|
||||||
|
forwardCacheManager: FileForwardCacheManager(_c, cacheMap),
|
||||||
shouldCheckCache: true,
|
shouldCheckCache: true,
|
||||||
forwardCacheManager: FileForwardCacheManager(_c, pass2CacheMap),
|
|
||||||
));
|
));
|
||||||
final remoteTouchEtag = await touchTokenManager.getRemoteRootEtag(account);
|
await for (final event
|
||||||
if (remoteTouchEtag == null) {
|
in _queryWithFileRepo(fileRepo, ev, fileRepoForShareDir: _c.fileRepo)) {
|
||||||
_log.info("[_queryOnlinePass2] remoteTouchEtag == null");
|
|
||||||
await touchTokenManager.setLocalRootEtag(account, null);
|
|
||||||
return pass1Files;
|
|
||||||
}
|
|
||||||
final localTouchEtag = await touchTokenManager.getLocalRootEtag(account);
|
|
||||||
if (remoteTouchEtag == localTouchEtag) {
|
|
||||||
_log.info("[_queryOnlinePass2] remoteTouchEtag matched");
|
|
||||||
return pass1Files;
|
|
||||||
}
|
|
||||||
|
|
||||||
final stopwatch = Stopwatch()..start();
|
|
||||||
final fileRepoNoCache =
|
|
||||||
FileRepo(FileCachedDataSource(_c, shouldCheckCache: true));
|
|
||||||
final newFiles = <File>[];
|
|
||||||
await for (final event in _queryWithFileRepo(fileRepo, ev,
|
|
||||||
fileRepoForShareDir: fileRepoNoCache)) {
|
|
||||||
if (event is ExceptionEvent) {
|
if (event is ExceptionEvent) {
|
||||||
_log.shout("[_queryOnlinePass2] Exception while request (2nd pass)",
|
_log.shout("[_queryOnline] Exception while request", event.error,
|
||||||
event.error, event.stackTrace);
|
event.stackTrace);
|
||||||
return pass1Files;
|
emit(ScanAccountDirBlocFailure(
|
||||||
|
cache.isEmpty
|
||||||
|
? files
|
||||||
|
: cache.where((f) => file_util.isSupportedFormat(f)).toList(),
|
||||||
|
event.error));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
files.addAll(event);
|
||||||
|
if (cache.isEmpty) {
|
||||||
|
// only emit partial results if there's no cache
|
||||||
|
emit(ScanAccountDirBlocLoading(files.toList()));
|
||||||
}
|
}
|
||||||
newFiles.addAll(event);
|
|
||||||
}
|
}
|
||||||
_log.info(
|
_log.info(
|
||||||
"[_queryOnlinePass2] Elapsed time (pass2): ${stopwatch.elapsedMilliseconds}ms");
|
"[_queryOnline] Elapsed time (_queryOnline): ${stopwatch.elapsedMilliseconds}ms, ${files.length} files");
|
||||||
_log.info("[_queryOnlinePass2] Save new touch root etag: $remoteTouchEtag");
|
|
||||||
await touchTokenManager.setLocalRootEtag(account, remoteTouchEtag);
|
emit(ScanAccountDirBlocSuccess(files));
|
||||||
return newFiles;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Emit all files under this account
|
/// Emit all files under this account
|
||||||
|
@ -560,7 +497,5 @@ class ScanAccountDirBloc
|
||||||
logTag: "ScanAccountDirBloc.refresh",
|
logTag: "ScanAccountDirBloc.refresh",
|
||||||
);
|
);
|
||||||
|
|
||||||
bool _shouldCheckCache = true;
|
|
||||||
|
|
||||||
static final _log = Logger("bloc.scan_dir.ScanAccountDirBloc");
|
static final _log = Logger("bloc.scan_dir.ScanAccountDirBloc");
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,6 +12,7 @@ import 'package:nc_photos/entity/tag.dart';
|
||||||
import 'package:nc_photos/entity/tagged_file.dart';
|
import 'package:nc_photos/entity/tagged_file.dart';
|
||||||
import 'package:nc_photos/or_null.dart';
|
import 'package:nc_photos/or_null.dart';
|
||||||
import 'package:nc_photos/pref.dart';
|
import 'package:nc_photos/pref.dart';
|
||||||
|
import 'package:nc_photos/touch_manager.dart';
|
||||||
|
|
||||||
enum DiType {
|
enum DiType {
|
||||||
albumRepo,
|
albumRepo,
|
||||||
|
@ -34,6 +35,7 @@ enum DiType {
|
||||||
searchRepo,
|
searchRepo,
|
||||||
pref,
|
pref,
|
||||||
sqliteDb,
|
sqliteDb,
|
||||||
|
touchManager,
|
||||||
}
|
}
|
||||||
|
|
||||||
class DiContainer {
|
class DiContainer {
|
||||||
|
@ -58,6 +60,7 @@ class DiContainer {
|
||||||
SearchRepo? searchRepo,
|
SearchRepo? searchRepo,
|
||||||
Pref? pref,
|
Pref? pref,
|
||||||
sql.SqliteDb? sqliteDb,
|
sql.SqliteDb? sqliteDb,
|
||||||
|
TouchManager? touchManager,
|
||||||
}) : _albumRepo = albumRepo,
|
}) : _albumRepo = albumRepo,
|
||||||
_albumRepoLocal = albumRepoLocal,
|
_albumRepoLocal = albumRepoLocal,
|
||||||
_faceRepo = faceRepo,
|
_faceRepo = faceRepo,
|
||||||
|
@ -77,7 +80,8 @@ class DiContainer {
|
||||||
_localFileRepo = localFileRepo,
|
_localFileRepo = localFileRepo,
|
||||||
_searchRepo = searchRepo,
|
_searchRepo = searchRepo,
|
||||||
_pref = pref,
|
_pref = pref,
|
||||||
_sqliteDb = sqliteDb;
|
_sqliteDb = sqliteDb,
|
||||||
|
_touchManager = touchManager;
|
||||||
|
|
||||||
DiContainer.late();
|
DiContainer.late();
|
||||||
|
|
||||||
|
@ -123,6 +127,8 @@ class DiContainer {
|
||||||
return contianer._pref != null;
|
return contianer._pref != null;
|
||||||
case DiType.sqliteDb:
|
case DiType.sqliteDb:
|
||||||
return contianer._sqliteDb != null;
|
return contianer._sqliteDb != null;
|
||||||
|
case DiType.touchManager:
|
||||||
|
return contianer._touchManager != null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -140,6 +146,7 @@ class DiContainer {
|
||||||
OrNull<SearchRepo>? searchRepo,
|
OrNull<SearchRepo>? searchRepo,
|
||||||
OrNull<Pref>? pref,
|
OrNull<Pref>? pref,
|
||||||
OrNull<sql.SqliteDb>? sqliteDb,
|
OrNull<sql.SqliteDb>? sqliteDb,
|
||||||
|
OrNull<TouchManager>? touchManager,
|
||||||
}) {
|
}) {
|
||||||
return DiContainer(
|
return DiContainer(
|
||||||
albumRepo: albumRepo == null ? _albumRepo : albumRepo.obj,
|
albumRepo: albumRepo == null ? _albumRepo : albumRepo.obj,
|
||||||
|
@ -156,6 +163,7 @@ class DiContainer {
|
||||||
searchRepo: searchRepo == null ? _searchRepo : searchRepo.obj,
|
searchRepo: searchRepo == null ? _searchRepo : searchRepo.obj,
|
||||||
pref: pref == null ? _pref : pref.obj,
|
pref: pref == null ? _pref : pref.obj,
|
||||||
sqliteDb: sqliteDb == null ? _sqliteDb : sqliteDb.obj,
|
sqliteDb: sqliteDb == null ? _sqliteDb : sqliteDb.obj,
|
||||||
|
touchManager: touchManager == null ? _touchManager : touchManager.obj,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -177,6 +185,7 @@ class DiContainer {
|
||||||
TaggedFileRepo get taggedFileRepo => _taggedFileRepo!;
|
TaggedFileRepo get taggedFileRepo => _taggedFileRepo!;
|
||||||
LocalFileRepo get localFileRepo => _localFileRepo!;
|
LocalFileRepo get localFileRepo => _localFileRepo!;
|
||||||
SearchRepo get searchRepo => _searchRepo!;
|
SearchRepo get searchRepo => _searchRepo!;
|
||||||
|
TouchManager get touchManager => _touchManager!;
|
||||||
|
|
||||||
sql.SqliteDb get sqliteDb => _sqliteDb!;
|
sql.SqliteDb get sqliteDb => _sqliteDb!;
|
||||||
Pref get pref => _pref!;
|
Pref get pref => _pref!;
|
||||||
|
@ -271,6 +280,11 @@ class DiContainer {
|
||||||
_searchRepo = v;
|
_searchRepo = v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
set touchManager(TouchManager v) {
|
||||||
|
assert(_touchManager == null);
|
||||||
|
_touchManager = v;
|
||||||
|
}
|
||||||
|
|
||||||
set sqliteDb(sql.SqliteDb v) {
|
set sqliteDb(sql.SqliteDb v) {
|
||||||
assert(_sqliteDb == null);
|
assert(_sqliteDb == null);
|
||||||
_sqliteDb = v;
|
_sqliteDb = v;
|
||||||
|
@ -302,6 +316,7 @@ class DiContainer {
|
||||||
TaggedFileRepo? _taggedFileRepo;
|
TaggedFileRepo? _taggedFileRepo;
|
||||||
LocalFileRepo? _localFileRepo;
|
LocalFileRepo? _localFileRepo;
|
||||||
SearchRepo? _searchRepo;
|
SearchRepo? _searchRepo;
|
||||||
|
TouchManager? _touchManager;
|
||||||
|
|
||||||
sql.SqliteDb? _sqliteDb;
|
sql.SqliteDb? _sqliteDb;
|
||||||
Pref? _pref;
|
Pref? _pref;
|
||||||
|
|
|
@ -17,11 +17,8 @@ import 'package:nc_photos/exception.dart';
|
||||||
import 'package:nc_photos/iterable_extension.dart';
|
import 'package:nc_photos/iterable_extension.dart';
|
||||||
import 'package:nc_photos/object_extension.dart';
|
import 'package:nc_photos/object_extension.dart';
|
||||||
import 'package:nc_photos/or_null.dart';
|
import 'package:nc_photos/or_null.dart';
|
||||||
import 'package:nc_photos/throttler.dart';
|
|
||||||
import 'package:nc_photos/touch_token_manager.dart';
|
|
||||||
import 'package:nc_photos/use_case/compat/v32.dart';
|
import 'package:nc_photos/use_case/compat/v32.dart';
|
||||||
import 'package:path/path.dart' as path_lib;
|
import 'package:path/path.dart' as path_lib;
|
||||||
import 'package:uuid/uuid.dart';
|
|
||||||
import 'package:xml/xml.dart';
|
import 'package:xml/xml.dart';
|
||||||
|
|
||||||
class FileWebdavDataSource implements FileDataSource {
|
class FileWebdavDataSource implements FileDataSource {
|
||||||
|
@ -583,10 +580,10 @@ class FileCachedDataSource implements FileDataSource {
|
||||||
await FileSqliteCacheUpdater(_c)(account, dir, remote: remote);
|
await FileSqliteCacheUpdater(_c)(account, dir, remote: remote);
|
||||||
if (shouldCheckCache) {
|
if (shouldCheckCache) {
|
||||||
// update our local touch token to match the remote one
|
// update our local touch token to match the remote one
|
||||||
final tokenManager = TouchTokenManager(_c);
|
|
||||||
try {
|
try {
|
||||||
await tokenManager.setLocalToken(
|
_log.info("[list] Update outdated local etag: ${dir.path}");
|
||||||
account, dir, cacheLoader.remoteTouchToken);
|
await _c.touchManager
|
||||||
|
.setLocalEtag(account, dir, cacheLoader.remoteTouchEtag);
|
||||||
} catch (e, stacktrace) {
|
} catch (e, stacktrace) {
|
||||||
_log.shout("[list] Failed while setLocalToken", e, stacktrace);
|
_log.shout("[list] Failed while setLocalToken", e, stacktrace);
|
||||||
// ignore error
|
// ignore error
|
||||||
|
@ -676,19 +673,8 @@ class FileCachedDataSource implements FileDataSource {
|
||||||
);
|
);
|
||||||
|
|
||||||
// generate a new random token
|
// generate a new random token
|
||||||
final token = const Uuid().v4().replaceAll("-", "");
|
|
||||||
final dir = File(path: path_lib.dirname(f.path));
|
final dir = File(path: path_lib.dirname(f.path));
|
||||||
await TouchTokenManager(_c).setLocalToken(account, dir, token);
|
await _c.touchManager.touch(account, dir);
|
||||||
// don't update remote token that frequently
|
|
||||||
(_touchTokenThrottlers["${account.url}/${dir.path}"] ??= Throttler(
|
|
||||||
onTriggered: _updateRemoteTouchToken,
|
|
||||||
logTag: "FileCachedDataSource._touchTokenThrottlers",
|
|
||||||
))
|
|
||||||
.trigger(
|
|
||||||
maxResponceTime: const Duration(seconds: 20),
|
|
||||||
maxPendingCount: 20,
|
|
||||||
data: _TouchTokenThrottlerData(account, dir, token),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@override
|
@override
|
||||||
|
@ -718,21 +704,8 @@ class FileCachedDataSource implements FileDataSource {
|
||||||
await _remoteSrc.createDir(account, path);
|
await _remoteSrc.createDir(account, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> updateRemoteTouchTokenNow() async {
|
Future<void> flushRemoteTouch() async {
|
||||||
for (final t in _touchTokenThrottlers.values) {
|
return _c.touchManager.flushRemote();
|
||||||
await t.triggerNow();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> _updateRemoteTouchToken(
|
|
||||||
List<_TouchTokenThrottlerData> data) async {
|
|
||||||
try {
|
|
||||||
final d = data.last;
|
|
||||||
await TouchTokenManager(_c).setRemoteToken(d.account, d.dir, d.token);
|
|
||||||
} catch (e, stackTrace) {
|
|
||||||
_log.shout("[_updateRemoteTouchToken] Failed while setRemoteToken", e,
|
|
||||||
stackTrace);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
final DiContainer _c;
|
final DiContainer _c;
|
||||||
|
@ -742,19 +715,9 @@ class FileCachedDataSource implements FileDataSource {
|
||||||
final _remoteSrc = const FileWebdavDataSource();
|
final _remoteSrc = const FileWebdavDataSource();
|
||||||
final FileSqliteDbDataSource _sqliteDbSrc;
|
final FileSqliteDbDataSource _sqliteDbSrc;
|
||||||
|
|
||||||
final _touchTokenThrottlers = <String, Throttler<_TouchTokenThrottlerData>>{};
|
|
||||||
|
|
||||||
static final _log = Logger("entity.file.data_source.FileCachedDataSource");
|
static final _log = Logger("entity.file.data_source.FileCachedDataSource");
|
||||||
}
|
}
|
||||||
|
|
||||||
class _TouchTokenThrottlerData {
|
|
||||||
const _TouchTokenThrottlerData(this.account, this.dir, this.token);
|
|
||||||
|
|
||||||
final Account account;
|
|
||||||
final File dir;
|
|
||||||
final String token;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Forward cache for listing AppDb dirs
|
/// Forward cache for listing AppDb dirs
|
||||||
///
|
///
|
||||||
/// It's very expensive to list a dir and its sub-dirs one by one in multiple
|
/// It's very expensive to list a dir and its sub-dirs one by one in multiple
|
||||||
|
|
|
@ -13,8 +13,6 @@ import 'package:nc_photos/exception.dart';
|
||||||
import 'package:nc_photos/iterable_extension.dart';
|
import 'package:nc_photos/iterable_extension.dart';
|
||||||
import 'package:nc_photos/list_util.dart' as list_util;
|
import 'package:nc_photos/list_util.dart' as list_util;
|
||||||
import 'package:nc_photos/object_extension.dart';
|
import 'package:nc_photos/object_extension.dart';
|
||||||
import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util;
|
|
||||||
import 'package:nc_photos/touch_token_manager.dart';
|
|
||||||
|
|
||||||
class FileCacheLoader {
|
class FileCacheLoader {
|
||||||
FileCacheLoader(
|
FileCacheLoader(
|
||||||
|
@ -51,7 +49,7 @@ class FileCacheLoader {
|
||||||
}
|
}
|
||||||
if (cacheEtag == remoteEtag) {
|
if (cacheEtag == remoteEtag) {
|
||||||
if (shouldCheckCache) {
|
if (shouldCheckCache) {
|
||||||
await _checkTouchToken(account, dir, cache);
|
await _checkTouchEtag(account, dir, cache);
|
||||||
} else {
|
} else {
|
||||||
_isGood = true;
|
_isGood = true;
|
||||||
}
|
}
|
||||||
|
@ -67,39 +65,15 @@ class FileCacheLoader {
|
||||||
}
|
}
|
||||||
|
|
||||||
bool get isGood => _isGood;
|
bool get isGood => _isGood;
|
||||||
String? get remoteTouchToken => _remoteToken;
|
String? get remoteTouchEtag => _remoteEtag;
|
||||||
|
|
||||||
Future<void> _checkTouchToken(
|
Future<void> _checkTouchEtag(
|
||||||
Account account, File f, List<File> cache) async {
|
Account account, File f, List<File> cache) async {
|
||||||
final touchPath =
|
final result = await _c.touchManager.checkTouchEtag(account, f);
|
||||||
"${remote_storage_util.getRemoteTouchDir(account)}/${f.strippedPath}";
|
if (result == null) {
|
||||||
final tokenManager = TouchTokenManager(_c);
|
|
||||||
String? remoteToken;
|
|
||||||
try {
|
|
||||||
remoteToken = await tokenManager.getRemoteToken(account, f);
|
|
||||||
} catch (e, stacktrace) {
|
|
||||||
_log.shout(
|
|
||||||
"[_checkTouchToken] Failed getting remote token at '$touchPath'",
|
|
||||||
e,
|
|
||||||
stacktrace);
|
|
||||||
}
|
|
||||||
_remoteToken = remoteToken;
|
|
||||||
|
|
||||||
String? localToken;
|
|
||||||
try {
|
|
||||||
localToken = await tokenManager.getLocalToken(account, f);
|
|
||||||
} catch (e, stacktrace) {
|
|
||||||
_log.shout(
|
|
||||||
"[_checkTouchToken] Failed getting local token at '$touchPath'",
|
|
||||||
e,
|
|
||||||
stacktrace);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (localToken != remoteToken) {
|
|
||||||
_log.info(
|
|
||||||
"[_checkTouchToken] Remote and local token differ, cache outdated");
|
|
||||||
} else {
|
|
||||||
_isGood = true;
|
_isGood = true;
|
||||||
|
} else {
|
||||||
|
_remoteEtag = result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -110,7 +84,7 @@ class FileCacheLoader {
|
||||||
final FileForwardCacheManager? forwardCacheManager;
|
final FileForwardCacheManager? forwardCacheManager;
|
||||||
|
|
||||||
var _isGood = false;
|
var _isGood = false;
|
||||||
String? _remoteToken;
|
String? _remoteEtag;
|
||||||
|
|
||||||
static final _log = Logger("entity.file.file_cache_manager.FileCacheLoader");
|
static final _log = Logger("entity.file.file_cache_manager.FileCacheLoader");
|
||||||
}
|
}
|
||||||
|
|
|
@ -239,8 +239,7 @@ class _MetadataTask {
|
||||||
|
|
||||||
final c = KiwiContainer().resolve<DiContainer>();
|
final c = KiwiContainer().resolve<DiContainer>();
|
||||||
if (c.fileRepo.dataSrc is FileCachedDataSource) {
|
if (c.fileRepo.dataSrc is FileCachedDataSource) {
|
||||||
await (c.fileRepo.dataSrc as FileCachedDataSource)
|
await (c.fileRepo.dataSrc as FileCachedDataSource).flushRemoteTouch();
|
||||||
.updateRemoteTouchTokenNow();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
197
app/lib/touch_manager.dart
Normal file
197
app/lib/touch_manager.dart
Normal file
|
@ -0,0 +1,197 @@
|
||||||
|
import 'dart:convert';
|
||||||
|
|
||||||
|
import 'package:logging/logging.dart';
|
||||||
|
import 'package:nc_photos/account.dart';
|
||||||
|
import 'package:nc_photos/di_container.dart';
|
||||||
|
import 'package:nc_photos/entity/file.dart';
|
||||||
|
import 'package:nc_photos/exception.dart';
|
||||||
|
import 'package:nc_photos/mobile/platform.dart'
|
||||||
|
if (dart.library.html) 'package:nc_photos/web/platform.dart' as platform;
|
||||||
|
import 'package:nc_photos/or_null.dart';
|
||||||
|
import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util;
|
||||||
|
import 'package:nc_photos/throttler.dart';
|
||||||
|
import 'package:nc_photos/use_case/ls_single_file.dart';
|
||||||
|
import 'package:nc_photos/use_case/put_file_binary.dart';
|
||||||
|
import 'package:path/path.dart' as path_lib;
|
||||||
|
import 'package:uuid/uuid.dart';
|
||||||
|
|
||||||
|
/// Manage touch events for files
|
||||||
|
///
|
||||||
|
/// Touch events are used to broadcast file changes that don't trigger an ETag
|
||||||
|
/// update to other devices. Such changes include custom properties like
|
||||||
|
/// metadata
|
||||||
|
class TouchManager {
|
||||||
|
TouchManager(this._c) : assert(require(_c));
|
||||||
|
|
||||||
|
static bool require(DiContainer c) =>
|
||||||
|
DiContainer.has(c, DiType.fileRepo) &&
|
||||||
|
DiContainer.has(c, DiType.fileRepoRemote);
|
||||||
|
|
||||||
|
static String newToken() {
|
||||||
|
return const Uuid().v4().replaceAll("-", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Clear the cached etags
|
||||||
|
///
|
||||||
|
/// You should call this before a complete re-scan
|
||||||
|
void clearTouchCache() {
|
||||||
|
_log.info("[clearTouchCache]");
|
||||||
|
_resultCache.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compare the remote and local etag
|
||||||
|
///
|
||||||
|
/// Return null if the two etags match, otherwise return the remote etag
|
||||||
|
Future<String?> checkTouchEtag(Account account, File dir) async {
|
||||||
|
if (dir.strippedPathWithEmpty.isNotEmpty) {
|
||||||
|
// check parent
|
||||||
|
if (await checkTouchEtag(
|
||||||
|
account, File(path: path_lib.dirname(dir.path))) ==
|
||||||
|
null) {
|
||||||
|
// parent ok == child ok
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
final cacheKey = "${account.url}/${dir.path}";
|
||||||
|
final cache = _resultCache[cacheKey];
|
||||||
|
if (cache != null) {
|
||||||
|
// we checked this dir already, return the cache
|
||||||
|
return cache.obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
String? remoteToken;
|
||||||
|
try {
|
||||||
|
remoteToken = await _getRemoteEtag(account, dir);
|
||||||
|
} catch (e, stacktrace) {
|
||||||
|
_log.shout("[checkTouchEtag] Failed getting remote etag", e, stacktrace);
|
||||||
|
}
|
||||||
|
|
||||||
|
String? localToken;
|
||||||
|
try {
|
||||||
|
localToken = await _getLocalEtag(account, dir);
|
||||||
|
} catch (e, stacktrace) {
|
||||||
|
_log.shout("[checkTouchEtag] Failed getting local etag", e, stacktrace);
|
||||||
|
}
|
||||||
|
|
||||||
|
final isMatch = localToken == remoteToken;
|
||||||
|
final result = OrNull(isMatch ? null : remoteToken);
|
||||||
|
_resultCache[cacheKey] = result;
|
||||||
|
if (!isMatch) {
|
||||||
|
_log.info(
|
||||||
|
"[checkTouchEtag] Remote and local etag differ, cache outdated: ${dir.strippedPath}");
|
||||||
|
} else {
|
||||||
|
_log.info("[checkTouchEtag] etags match: ${dir.strippedPath}");
|
||||||
|
}
|
||||||
|
return result.obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Touch a dir
|
||||||
|
Future<void> touch(Account account, File dir) async {
|
||||||
|
// _log.info("[touch] Touch dir '${dir.path}'");
|
||||||
|
// delete the local etag, we'll update it later. If the app is killed, then
|
||||||
|
// at least the app will update the cache in next run
|
||||||
|
await setLocalEtag(account, dir, null);
|
||||||
|
(_throttlers["${account.url}/${dir.path}"] ??= Throttler(
|
||||||
|
onTriggered: _triggerTouch,
|
||||||
|
logTag: "TouchManager._throttlers",
|
||||||
|
))
|
||||||
|
.trigger(
|
||||||
|
maxResponceTime: const Duration(seconds: 20),
|
||||||
|
maxPendingCount: 20,
|
||||||
|
data: _ThrottlerData(account, dir),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<void> flushRemote() async {
|
||||||
|
for (final t in _throttlers.values) {
|
||||||
|
await t.triggerNow();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<void> setLocalEtag(Account account, File dir, String? etag) {
|
||||||
|
final name = _getLocalStorageName(account, dir);
|
||||||
|
if (etag == null) {
|
||||||
|
return platform.UniversalStorage().remove(name);
|
||||||
|
} else {
|
||||||
|
_log.info("[setLocalEtag] Set local etag for file '${dir.path}': $etag");
|
||||||
|
return platform.UniversalStorage().putString(name, etag);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<void> _triggerTouch(List<_ThrottlerData> data) async {
|
||||||
|
try {
|
||||||
|
final d = data.last;
|
||||||
|
await _touchRemote(d.account, d.dir);
|
||||||
|
final etag = await _getRemoteEtag(d.account, d.dir);
|
||||||
|
_log.info("[_triggerTouch] Remote etag = $etag");
|
||||||
|
if (etag == null) {
|
||||||
|
_log.severe("[_triggerTouch] etag == null");
|
||||||
|
} else {
|
||||||
|
await setLocalEtag(d.account, d.dir, etag);
|
||||||
|
}
|
||||||
|
} catch (e, stackTrace) {
|
||||||
|
_log.shout("[_triggerTouch] Uncaught exception", e, stackTrace);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Update the remote touch dir
|
||||||
|
Future<void> _touchRemote(Account account, File dir) async {
|
||||||
|
_log.info("[touchRemote] Touch remote dir '${dir.path}'");
|
||||||
|
final path = _getRemoteEtagPath(account, dir);
|
||||||
|
return PutFileBinary(_c.fileRepo)(
|
||||||
|
account, "$path/token.txt", const Utf8Encoder().convert(newToken()),
|
||||||
|
shouldCreateMissingDir: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Return the corresponding touch etag for [dir] from remote source, or null
|
||||||
|
/// if no such file
|
||||||
|
Future<String?> _getRemoteEtag(Account account, File dir) async {
|
||||||
|
final path = _getRemoteEtagPath(account, dir);
|
||||||
|
try {
|
||||||
|
final f = await LsSingleFile(_c)(account, path);
|
||||||
|
return f.etag;
|
||||||
|
} on ApiException catch (e) {
|
||||||
|
if (e.response.statusCode == 404) {
|
||||||
|
return null;
|
||||||
|
} else {
|
||||||
|
rethrow;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String _getRemoteEtagPath(Account account, File dir) {
|
||||||
|
final strippedPath = dir.strippedPath;
|
||||||
|
if (strippedPath == ".") {
|
||||||
|
return remote_storage_util.getRemoteTouchDir(account);
|
||||||
|
} else {
|
||||||
|
return "${remote_storage_util.getRemoteTouchDir(account)}/$strippedPath";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<String?> _getLocalEtag(Account account, File file) async {
|
||||||
|
final name = _getLocalStorageName(account, file);
|
||||||
|
return platform.UniversalStorage().getString(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
String _getLocalStorageName(Account account, File file) {
|
||||||
|
final strippedPath = file.strippedPath;
|
||||||
|
if (strippedPath == ".") {
|
||||||
|
return "touch/${account.url.replaceFirst('://', '_')}/${account.userId}/token";
|
||||||
|
} else {
|
||||||
|
return "touch/${account.url.replaceFirst('://', '_')}/${account.userId}/${file.strippedPath}/token";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
final DiContainer _c;
|
||||||
|
final _throttlers = <String, Throttler<_ThrottlerData>>{};
|
||||||
|
final _resultCache = <String, OrNull<String>>{};
|
||||||
|
|
||||||
|
static final _log = Logger("touch_token_manager.TouchManager");
|
||||||
|
}
|
||||||
|
|
||||||
|
class _ThrottlerData {
|
||||||
|
const _ThrottlerData(this.account, this.dir);
|
||||||
|
|
||||||
|
final Account account;
|
||||||
|
final File dir;
|
||||||
|
}
|
|
@ -1,123 +0,0 @@
|
||||||
import 'dart:convert';
|
|
||||||
|
|
||||||
import 'package:logging/logging.dart';
|
|
||||||
import 'package:nc_photos/account.dart';
|
|
||||||
import 'package:nc_photos/di_container.dart';
|
|
||||||
import 'package:nc_photos/entity/file.dart';
|
|
||||||
import 'package:nc_photos/exception.dart';
|
|
||||||
import 'package:nc_photos/mobile/platform.dart'
|
|
||||||
if (dart.library.html) 'package:nc_photos/web/platform.dart' as platform;
|
|
||||||
import 'package:nc_photos/pref.dart';
|
|
||||||
import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util;
|
|
||||||
import 'package:nc_photos/use_case/get_file_binary.dart';
|
|
||||||
import 'package:nc_photos/use_case/ls_single_file.dart';
|
|
||||||
import 'package:nc_photos/use_case/put_file_binary.dart';
|
|
||||||
import 'package:nc_photos/use_case/remove.dart';
|
|
||||||
|
|
||||||
/// Manage touch token for files
|
|
||||||
///
|
|
||||||
/// Touch tokens are used to broadcast file changes that don't trigger an ETag
|
|
||||||
/// update to other devices. Such changes include custom properties like
|
|
||||||
/// metadata. In order to detect these hidden changes, you should get both
|
|
||||||
/// local and remote tokens and compare them. Beware that getting the remote
|
|
||||||
/// token requires downloading a file from the server so you may want to avoid
|
|
||||||
/// doing it on every query
|
|
||||||
class TouchTokenManager {
|
|
||||||
TouchTokenManager(this._c) : assert(require(_c));
|
|
||||||
|
|
||||||
static bool require(DiContainer c) =>
|
|
||||||
DiContainer.has(c, DiType.fileRepo) &&
|
|
||||||
DiContainer.has(c, DiType.fileRepoRemote);
|
|
||||||
|
|
||||||
Future<String?> getRemoteRootEtag(Account account) async {
|
|
||||||
try {
|
|
||||||
// we use the remote repo here to prevent it caching the result
|
|
||||||
final touchDir = await LsSingleFile(_c.withRemoteFileRepo())(
|
|
||||||
account, remote_storage_util.getRemoteTouchDir(account));
|
|
||||||
return touchDir.etag!;
|
|
||||||
} catch (_) {
|
|
||||||
// dir not found on server
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> setLocalRootEtag(Account account, String? etag) async {
|
|
||||||
if (etag == null) {
|
|
||||||
await AccountPref.of(account).removeTouchRootEtag();
|
|
||||||
} else {
|
|
||||||
await AccountPref.of(account).setTouchRootEtag(etag);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<String?> getLocalRootEtag(Account account) async {
|
|
||||||
return AccountPref.of(account).getTouchRootEtag();
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> setRemoteToken(Account account, File file, String? token) async {
|
|
||||||
_log.info(
|
|
||||||
"[setRemoteToken] Set remote token for file '${file.path}': $token");
|
|
||||||
final path = _getRemotePath(account, file);
|
|
||||||
if (token == null) {
|
|
||||||
return Remove(_c)(account, [file], shouldCleanUp: false);
|
|
||||||
} else {
|
|
||||||
return PutFileBinary(_c.fileRepo)(
|
|
||||||
account, path, const Utf8Encoder().convert(token),
|
|
||||||
shouldCreateMissingDir: true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Return the touch token for [file] from remote source, or null if no such
|
|
||||||
/// file
|
|
||||||
Future<String?> getRemoteToken(Account account, File file) async {
|
|
||||||
final path = _getRemotePath(account, file);
|
|
||||||
try {
|
|
||||||
final content =
|
|
||||||
await GetFileBinary(_c.fileRepo)(account, File(path: path));
|
|
||||||
return const Utf8Decoder().convert(content);
|
|
||||||
} on ApiException catch (e) {
|
|
||||||
if (e.response.statusCode == 404) {
|
|
||||||
return null;
|
|
||||||
} else {
|
|
||||||
rethrow;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> setLocalToken(Account account, File file, String? token) {
|
|
||||||
_log.info(
|
|
||||||
"[setLocalToken] Set local token for file '${file.path}': $token");
|
|
||||||
final name = _getLocalStorageName(account, file);
|
|
||||||
if (token == null) {
|
|
||||||
return platform.UniversalStorage().remove(name);
|
|
||||||
} else {
|
|
||||||
return platform.UniversalStorage().putString(name, token);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<String?> getLocalToken(Account account, File file) async {
|
|
||||||
final name = _getLocalStorageName(account, file);
|
|
||||||
return platform.UniversalStorage().getString(name);
|
|
||||||
}
|
|
||||||
|
|
||||||
String _getRemotePath(Account account, File file) {
|
|
||||||
final strippedPath = file.strippedPath;
|
|
||||||
if (strippedPath == ".") {
|
|
||||||
return "${remote_storage_util.getRemoteTouchDir(account)}/token.txt";
|
|
||||||
} else {
|
|
||||||
return "${remote_storage_util.getRemoteTouchDir(account)}/${file.strippedPath}/token.txt";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
String _getLocalStorageName(Account account, File file) {
|
|
||||||
final strippedPath = file.strippedPath;
|
|
||||||
if (strippedPath == ".") {
|
|
||||||
return "touch/${account.url.replaceFirst('://', '_')}/${account.userId}/token";
|
|
||||||
} else {
|
|
||||||
return "touch/${account.url.replaceFirst('://', '_')}/${account.userId}/${file.strippedPath}/token";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
final DiContainer _c;
|
|
||||||
|
|
||||||
static final _log = Logger("touch_token_manager.TouchTokenManager");
|
|
||||||
}
|
|
Loading…
Reference in a new issue