diff --git a/lib/bloc/list_album.dart b/lib/bloc/list_album.dart index e2385bc1..fbd143af 100644 --- a/lib/bloc/list_album.dart +++ b/lib/bloc/list_album.dart @@ -3,6 +3,7 @@ import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/event/event.dart'; import 'package:nc_photos/use_case/list_album.dart'; diff --git a/lib/bloc/ls_dir.dart b/lib/bloc/ls_dir.dart index 139076d2..ece5c474 100644 --- a/lib/bloc/ls_dir.dart +++ b/lib/bloc/ls_dir.dart @@ -4,6 +4,7 @@ import 'package:bloc/bloc.dart'; import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/iterable_extension.dart'; import 'package:nc_photos/use_case/ls.dart'; diff --git a/lib/bloc/scan_dir.dart b/lib/bloc/scan_dir.dart index 143601de..fced334d 100644 --- a/lib/bloc/scan_dir.dart +++ b/lib/bloc/scan_dir.dart @@ -7,6 +7,7 @@ import 'package:kiwi/kiwi.dart'; import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/event/event.dart'; import 'package:nc_photos/iterable_extension.dart'; import 'package:nc_photos/use_case/scan_dir.dart'; diff --git a/lib/entity/album.dart b/lib/entity/album.dart index 784f1a29..b016dfc1 100644 --- a/lib/entity/album.dart +++ b/lib/entity/album.dart @@ -10,6 +10,7 @@ import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/app_db.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/exception.dart'; import 'package:nc_photos/int_util.dart' as int_util; import 'package:nc_photos/iterable_extension.dart'; diff --git a/lib/entity/file.dart b/lib/entity/file.dart index 31ffe380..9e3d4ff0 100644 --- a/lib/entity/file.dart +++ b/lib/entity/file.dart @@ -1,26 +1,12 @@ -import 'dart:convert'; import 'dart:typed_data'; import 'package:equatable/equatable.dart'; import 'package:flutter/foundation.dart'; -import 'package:idb_sqflite/idb_sqflite.dart'; import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; -import 'package:nc_photos/api/api.dart'; -import 'package:nc_photos/app_db.dart'; import 'package:nc_photos/entity/exif.dart'; -import 'package:nc_photos/entity/webdav_response_parser.dart'; -import 'package:nc_photos/exception.dart'; -import 'package:nc_photos/int_util.dart' as int_util; -import 'package:nc_photos/iterable_extension.dart'; import 'package:nc_photos/or_null.dart'; -import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util; import 'package:nc_photos/string_extension.dart'; -import 'package:nc_photos/touch_token_manager.dart'; -import 'package:path/path.dart' as path; -import 'package:quiver/iterables.dart'; -import 'package:uuid/uuid.dart'; -import 'package:xml/xml.dart'; int compareFileDateTimeDescending(File x, File y) { final xDate = x.metadata?.exif?.dateTimeOriginal ?? x.lastModified; @@ -459,582 +445,3 @@ abstract class FileDataSource { /// remote.php/dav/files/admin/new/dir Future createDir(Account account, String path); } - -class FileWebdavDataSource implements FileDataSource { - @override - list( - Account account, - File f, { - int depth, - }) async { - _log.fine("[list] ${f.path}"); - final response = await Api(account).files().propfind( - path: f.path, - depth: depth, - getlastmodified: 1, - resourcetype: 1, - getetag: 1, - getcontenttype: 1, - getcontentlength: 1, - hasPreview: 1, - fileid: 1, - customNamespaces: { - "com.nkming.nc_photos": "app", - }, - customProperties: [ - "app:metadata", - ], - ); - if (!response.isGood) { - _log.severe("[list] Failed requesting server: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - - final xml = XmlDocument.parse(response.body); - final files = WebdavFileParser()(xml); - // _log.fine("[list] Parsed files: [$files]"); - return files.map((e) { - if (e.metadata == null || e.metadata.fileEtag == e.etag) { - return e; - } else { - _log.info("[list] Ignore outdated metadata for ${e.path}"); - return e.copyWith(metadata: OrNull(null)); - } - }).toList(); - } - - @override - remove(Account account, File f) async { - _log.info("[remove] ${f.path}"); - final response = await Api(account).files().delete(path: f.path); - if (!response.isGood) { - _log.severe("[remove] Failed requesting server: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - } - - @override - getBinary(Account account, File f) async { - _log.info("[getBinary] ${f.path}"); - final response = await Api(account).files().get(path: f.path); - if (!response.isGood) { - _log.severe("[getBinary] Failed requesting server: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - return response.body; - } - - @override - putBinary(Account account, String path, Uint8List content) async { - _log.info("[putBinary] $path"); - final response = - await Api(account).files().put(path: path, content: content); - if (!response.isGood) { - _log.severe("[putBinary] Failed requesting server: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - } - - @override - updateMetadata(Account account, File f, Metadata metadata) async { - _log.info("[updateMetadata] ${f.path}"); - if (metadata != null && metadata.fileEtag != f.etag) { - _log.warning( - "[updateMetadata] etag mismatch (metadata: ${metadata.fileEtag}, file: ${f.etag})"); - } - final setProps = { - if (metadata != null) "app:metadata": jsonEncode(metadata.toJson()), - }; - final removeProps = [ - if (metadata == null) "app:metadata", - ]; - final response = await Api(account).files().proppatch( - path: f.path, - namespaces: { - "com.nkming.nc_photos": "app", - }, - set: setProps.isNotEmpty ? setProps : null, - remove: removeProps.isNotEmpty ? removeProps : null, - ); - if (!response.isGood) { - _log.severe("[updateMetadata] Failed requesting server: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - } - - @override - copy( - Account account, - File f, - String destination, { - bool shouldOverwrite, - }) async { - _log.info("[copy] ${f.path} to $destination"); - final response = await Api(account).files().copy( - path: f.path, - destinationUrl: "${account.url}/$destination", - overwrite: shouldOverwrite, - ); - if (!response.isGood) { - _log.severe("[copy] Failed requesting sever: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - } - - @override - move( - Account account, - File f, - String destination, { - bool shouldOverwrite, - }) async { - _log.info("[move] ${f.path} to $destination"); - final response = await Api(account).files().move( - path: f.path, - destinationUrl: "${account.url}/$destination", - overwrite: shouldOverwrite, - ); - if (!response.isGood) { - _log.severe("[move] Failed requesting sever: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - } - - @override - createDir(Account account, String path) async { - _log.info("[createDir] $path"); - final response = await Api(account).files().mkcol( - path: path, - ); - if (!response.isGood) { - _log.severe("[createDir] Failed requesting sever: $response"); - throw ApiException( - response: response, - message: "Failed communicating with server: ${response.statusCode}"); - } - } - - static final _log = Logger("entity.file.FileWebdavDataSource"); -} - -class FileAppDbDataSource implements FileDataSource { - @override - list(Account account, File f) { - _log.info("[list] ${f.path}"); - return AppDb.use((db) async { - final transaction = db.transaction(AppDb.fileStoreName, idbModeReadOnly); - final store = transaction.objectStore(AppDb.fileStoreName); - return await _doList(store, account, f); - }); - } - - @override - remove(Account account, File f) { - _log.info("[remove] ${f.path}"); - return AppDb.use((db) async { - final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); - final store = transaction.objectStore(AppDb.fileStoreName); - final index = store.index(AppDbFileEntry.indexName); - final path = AppDbFileEntry.toPath(account, f); - final range = KeyRange.bound([path, 0], [path, int_util.int32Max]); - final keys = await index - .openKeyCursor(range: range, autoAdvance: true) - .map((cursor) => cursor.primaryKey) - .toList(); - for (final k in keys) { - _log.fine("[remove] Removing DB entry: $k"); - await store.delete(k); - } - }); - } - - @override - getBinary(Account account, File f) { - _log.info("[getBinary] ${f.path}"); - throw UnimplementedError(); - } - - @override - putBinary(Account account, String path, Uint8List content) async { - _log.info("[putBinary] $path"); - // do nothing, we currently don't store file contents locally - } - - @override - updateMetadata(Account account, File f, Metadata metadata) { - _log.info("[updateMetadata] ${f.path}"); - return AppDb.use((db) async { - final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); - final store = transaction.objectStore(AppDb.fileStoreName); - final parentDir = File(path: path.dirname(f.path)); - final parentList = await _doList(store, account, parentDir); - final jsonList = parentList.map((e) { - if (e.path == f.path) { - return e.copyWith(metadata: OrNull(metadata)); - } else { - return e; - } - }); - await _cacheListResults(store, account, parentDir, jsonList); - }); - } - - @override - copy( - Account account, - File f, - String destination, { - bool shouldOverwrite, - }) async { - // do nothing - } - - @override - move( - Account account, - File f, - String destination, { - bool shouldOverwrite, - }) async { - // do nothing - } - - @override - createDir(Account account, String path) async { - // do nothing - } - - Future> _doList(ObjectStore store, Account account, File f) async { - final index = store.index(AppDbFileEntry.indexName); - final path = AppDbFileEntry.toPath(account, f); - final range = KeyRange.bound([path, 0], [path, int_util.int32Max]); - final List results = await index.getAll(range); - if (results?.isNotEmpty == true) { - final entries = results - .map((e) => AppDbFileEntry.fromJson(e.cast())); - return entries.map((e) { - _log.info("[_doList] ${e.path}[${e.index}]"); - return e.data; - }).reduce((value, element) => value + element); - } else { - throw CacheNotFoundException("No entry: $path"); - } - } - - static final _log = Logger("entity.file.FileAppDbDataSource"); -} - -class FileCachedDataSource implements FileDataSource { - FileCachedDataSource({ - this.shouldCheckCache = false, - }); - - @override - list(Account account, File f) async { - final cacheManager = _CacheManager( - appDbSrc: _appDbSrc, - remoteSrc: _remoteSrc, - shouldCheckCache: shouldCheckCache, - ); - final cache = await cacheManager.list(account, f); - if (cacheManager.isGood) { - return cache; - } - - // no cache or outdated - try { - final remote = await _remoteSrc.list(account, f); - await _cacheResult(account, f, remote); - if (shouldCheckCache) { - // update our local touch token to match the remote one - final tokenManager = TouchTokenManager(); - try { - await tokenManager.setLocalToken( - account, f, cacheManager.remoteTouchToken); - } catch (e, stacktrace) { - _log.shout("[list] Failed while setLocalToken", e, stacktrace); - // ignore error - } - } - - if (cache != null) { - try { - await _cleanUpCachedDir(account, remote, cache); - } catch (e, stacktrace) { - _log.shout("[list] Failed while _cleanUpCachedList", e, stacktrace); - // ignore error - } - } - return remote; - } on ApiException catch (e) { - if (e.response.statusCode == 404) { - _log.info("[list] File removed: $f"); - _appDbSrc.remove(account, f); - return []; - } else { - rethrow; - } - } - } - - @override - remove(Account account, File f) async { - await _appDbSrc.remove(account, f); - await _remoteSrc.remove(account, f); - } - - @override - getBinary(Account account, File f) { - return _remoteSrc.getBinary(account, f); - } - - @override - putBinary(Account account, String path, Uint8List content) async { - await _remoteSrc.putBinary(account, path, content); - } - - @override - updateMetadata(Account account, File f, Metadata metadata) async { - await _remoteSrc - .updateMetadata(account, f, metadata) - .then((_) => _appDbSrc.updateMetadata(account, f, metadata)); - - // generate a new random token - final token = Uuid().v4().replaceAll("-", ""); - final tokenManager = TouchTokenManager(); - final dir = File(path: path.dirname(f.path)); - await tokenManager.setLocalToken(account, dir, token); - final fileRepo = FileRepo(this); - await tokenManager.setRemoteToken(fileRepo, account, dir, token); - _log.info( - "[updateMetadata] New touch token '$token' for dir '${dir.path}'"); - } - - @override - copy( - Account account, - File f, - String destination, { - bool shouldOverwrite, - }) async { - await _remoteSrc.copy(account, f, destination, - shouldOverwrite: shouldOverwrite); - } - - @override - move( - Account account, - File f, - String destination, { - bool shouldOverwrite, - }) async { - await _remoteSrc.move(account, f, destination, - shouldOverwrite: shouldOverwrite); - } - - @override - createDir(Account account, String path) async { - await _remoteSrc.createDir(account, path); - } - - Future _cacheResult(Account account, File f, List result) { - return AppDb.use((db) async { - final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); - final store = transaction.objectStore(AppDb.fileStoreName); - await _cacheListResults(store, account, f, result); - }); - } - - /// Remove dangling dir entries in the file object store - Future _cleanUpCachedDir( - Account account, List remoteResults, List cachedResults) { - final removed = cachedResults - .where((cache) => - !remoteResults.any((remote) => remote.path == cache.path)) - .toList(); - if (removed.isEmpty) { - return Future.delayed(Duration.zero); - } - return AppDb.use((db) async { - final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); - final store = transaction.objectStore(AppDb.fileStoreName); - final index = store.index(AppDbFileEntry.indexName); - for (final r in removed) { - final path = AppDbFileEntry.toPath(account, r); - final keys = []; - // delete the dir itself - final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]); - // delete with KeyRange is not supported in idb_shim/idb_sqflite - // await store.delete(dirRange); - keys.addAll(await index - .openKeyCursor(range: dirRange, autoAdvance: true) - .map((cursor) => cursor.primaryKey) - .toList()); - // then its children - final childrenRange = - KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]); - keys.addAll(await index - .openKeyCursor(range: childrenRange, autoAdvance: true) - .map((cursor) => cursor.primaryKey) - .toList()); - - for (final k in keys) { - _log.fine("[_cleanUpCachedDir] Removing DB entry: $k"); - await store.delete(k); - } - } - }); - } - - final bool shouldCheckCache; - - final _remoteSrc = FileWebdavDataSource(); - final _appDbSrc = FileAppDbDataSource(); - - static final _log = Logger("entity.file.FileCachedDataSource"); -} - -class _CacheManager { - _CacheManager({ - @required this.appDbSrc, - @required this.remoteSrc, - this.shouldCheckCache = false, - }); - - /// Return the cached results of listing a directory [f] - /// - /// Should check [isGood] before using the cache returning by this method - Future> list(Account account, File f) async { - final trimmedRootPath = f.path.trimAny("/"); - List cache; - try { - cache = await appDbSrc.list(account, f); - // compare the cached root - final cacheEtag = cache - .firstWhere((element) => element.path.trimAny("/") == trimmedRootPath) - .etag; - if (cacheEtag != null) { - // compare the etag to see if the content has been updated - var remoteEtag = f.etag; - if (remoteEtag == null) { - // no etag supplied, we need to query it form remote - final remote = await remoteSrc.list(account, f, depth: 0); - assert(remote.length == 1); - remoteEtag = remote.first.etag; - } - if (cacheEtag == remoteEtag) { - _log.fine( - "[_listCache] etag matched for ${AppDbFileEntry.toPath(account, f)}"); - if (shouldCheckCache) { - await _checkTouchToken(account, f, cache); - } else { - _isGood = true; - } - } - } else { - _log.info( - "[_list] Remote content updated for ${AppDbFileEntry.toPath(account, f)}"); - } - } on CacheNotFoundException catch (_) { - // normal when there's no cache - } catch (e, stacktrace) { - _log.shout("[_list] Cache failure", e, stacktrace); - } - return cache; - } - - bool get isGood => _isGood; - String get remoteTouchToken => _remoteToken; - - Future _checkTouchToken( - Account account, File f, List cache) async { - final touchPath = - "${remote_storage_util.getRemoteTouchDir(account)}/${f.strippedPath}"; - final fileRepo = FileRepo(FileCachedDataSource()); - final tokenManager = TouchTokenManager(); - String remoteToken; - try { - remoteToken = await tokenManager.getRemoteToken(fileRepo, account, f); - } catch (e, stacktrace) { - _log.shout( - "[_checkTouchToken] Failed getting remote token at '$touchPath'", - e, - stacktrace); - } - _remoteToken = remoteToken; - - String localToken; - try { - localToken = await tokenManager.getLocalToken(account, f); - } catch (e, stacktrace) { - _log.shout( - "[_checkTouchToken] Failed getting local token at '$touchPath'", - e, - stacktrace); - } - - if (localToken != remoteToken) { - _log.info( - "[_checkTouchToken] Remote and local token differ, cache outdated"); - } else { - _isGood = true; - } - } - - final FileWebdavDataSource remoteSrc; - final FileAppDbDataSource appDbSrc; - final bool shouldCheckCache; - - var _isGood = false; - String _remoteToken; - - static final _log = Logger("entity.file._CacheManager"); -} - -Future _cacheListResults( - ObjectStore store, Account account, File f, Iterable results) async { - final index = store.index(AppDbFileEntry.indexName); - final path = AppDbFileEntry.toPath(account, f); - final range = KeyRange.bound([path, 0], [path, int_util.int32Max]); - // count number of entries for this dir - final count = await index.count(range); - int newCount = 0; - for (final pair - in partition(results, AppDbFileEntry.maxDataSize).withIndex()) { - _log.info( - "[_cacheListResults] Caching $path[${pair.item1}], length: ${pair.item2.length}"); - await store.put( - AppDbFileEntry(path, pair.item1, pair.item2).toJson(), - AppDbFileEntry.toPrimaryKey(account, f, pair.item1), - ); - ++newCount; - } - if (count > newCount) { - // index is 0-based - final rmRange = KeyRange.bound([path, newCount], [path, int_util.int32Max]); - final rmKeys = await index - .openKeyCursor(range: rmRange, autoAdvance: true) - .map((cursor) => cursor.primaryKey) - .toList(); - for (final k in rmKeys) { - _log.fine("[_cacheListResults] Removing DB entry: $k"); - await store.delete(k); - } - } -} - -final _log = Logger("entity.file"); diff --git a/lib/entity/file/data_source.dart b/lib/entity/file/data_source.dart new file mode 100644 index 00000000..b9de38c4 --- /dev/null +++ b/lib/entity/file/data_source.dart @@ -0,0 +1,601 @@ +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; +import 'package:idb_shim/idb_client.dart'; +import 'package:logging/logging.dart'; +import 'package:nc_photos/account.dart'; +import 'package:nc_photos/api/api.dart'; +import 'package:nc_photos/app_db.dart'; +import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/webdav_response_parser.dart'; +import 'package:nc_photos/exception.dart'; +import 'package:nc_photos/int_util.dart' as int_util; +import 'package:nc_photos/iterable_extension.dart'; +import 'package:nc_photos/or_null.dart'; +import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util; +import 'package:nc_photos/string_extension.dart'; +import 'package:nc_photos/touch_token_manager.dart'; +import 'package:path/path.dart' as path; +import 'package:quiver/iterables.dart'; +import 'package:uuid/uuid.dart'; +import 'package:xml/xml.dart'; + +class FileWebdavDataSource implements FileDataSource { + @override + list( + Account account, + File f, { + int depth, + }) async { + _log.fine("[list] ${f.path}"); + final response = await Api(account).files().propfind( + path: f.path, + depth: depth, + getlastmodified: 1, + resourcetype: 1, + getetag: 1, + getcontenttype: 1, + getcontentlength: 1, + hasPreview: 1, + fileid: 1, + customNamespaces: { + "com.nkming.nc_photos": "app", + }, + customProperties: [ + "app:metadata", + ], + ); + if (!response.isGood) { + _log.severe("[list] Failed requesting server: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + + final xml = XmlDocument.parse(response.body); + final files = WebdavFileParser()(xml); + // _log.fine("[list] Parsed files: [$files]"); + return files.map((e) { + if (e.metadata == null || e.metadata.fileEtag == e.etag) { + return e; + } else { + _log.info("[list] Ignore outdated metadata for ${e.path}"); + return e.copyWith(metadata: OrNull(null)); + } + }).toList(); + } + + @override + remove(Account account, File f) async { + _log.info("[remove] ${f.path}"); + final response = await Api(account).files().delete(path: f.path); + if (!response.isGood) { + _log.severe("[remove] Failed requesting server: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + } + + @override + getBinary(Account account, File f) async { + _log.info("[getBinary] ${f.path}"); + final response = await Api(account).files().get(path: f.path); + if (!response.isGood) { + _log.severe("[getBinary] Failed requesting server: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + return response.body; + } + + @override + putBinary(Account account, String path, Uint8List content) async { + _log.info("[putBinary] $path"); + final response = + await Api(account).files().put(path: path, content: content); + if (!response.isGood) { + _log.severe("[putBinary] Failed requesting server: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + } + + @override + updateMetadata(Account account, File f, Metadata metadata) async { + _log.info("[updateMetadata] ${f.path}"); + if (metadata != null && metadata.fileEtag != f.etag) { + _log.warning( + "[updateMetadata] etag mismatch (metadata: ${metadata.fileEtag}, file: ${f.etag})"); + } + final setProps = { + if (metadata != null) "app:metadata": jsonEncode(metadata.toJson()), + }; + final removeProps = [ + if (metadata == null) "app:metadata", + ]; + final response = await Api(account).files().proppatch( + path: f.path, + namespaces: { + "com.nkming.nc_photos": "app", + }, + set: setProps.isNotEmpty ? setProps : null, + remove: removeProps.isNotEmpty ? removeProps : null, + ); + if (!response.isGood) { + _log.severe("[updateMetadata] Failed requesting server: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + } + + @override + copy( + Account account, + File f, + String destination, { + bool shouldOverwrite, + }) async { + _log.info("[copy] ${f.path} to $destination"); + final response = await Api(account).files().copy( + path: f.path, + destinationUrl: "${account.url}/$destination", + overwrite: shouldOverwrite, + ); + if (!response.isGood) { + _log.severe("[copy] Failed requesting sever: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + } + + @override + move( + Account account, + File f, + String destination, { + bool shouldOverwrite, + }) async { + _log.info("[move] ${f.path} to $destination"); + final response = await Api(account).files().move( + path: f.path, + destinationUrl: "${account.url}/$destination", + overwrite: shouldOverwrite, + ); + if (!response.isGood) { + _log.severe("[move] Failed requesting sever: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + } + + @override + createDir(Account account, String path) async { + _log.info("[createDir] $path"); + final response = await Api(account).files().mkcol( + path: path, + ); + if (!response.isGood) { + _log.severe("[createDir] Failed requesting sever: $response"); + throw ApiException( + response: response, + message: "Failed communicating with server: ${response.statusCode}"); + } + } + + static final _log = Logger("entity.file.data_source.FileWebdavDataSource"); +} + +class FileAppDbDataSource implements FileDataSource { + @override + list(Account account, File f) { + _log.info("[list] ${f.path}"); + return AppDb.use((db) async { + final transaction = db.transaction(AppDb.fileStoreName, idbModeReadOnly); + final store = transaction.objectStore(AppDb.fileStoreName); + return await _doList(store, account, f); + }); + } + + @override + remove(Account account, File f) { + _log.info("[remove] ${f.path}"); + return AppDb.use((db) async { + final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); + final store = transaction.objectStore(AppDb.fileStoreName); + final index = store.index(AppDbFileEntry.indexName); + final path = AppDbFileEntry.toPath(account, f); + final range = KeyRange.bound([path, 0], [path, int_util.int32Max]); + final keys = await index + .openKeyCursor(range: range, autoAdvance: true) + .map((cursor) => cursor.primaryKey) + .toList(); + for (final k in keys) { + _log.fine("[remove] Removing DB entry: $k"); + await store.delete(k); + } + }); + } + + @override + getBinary(Account account, File f) { + _log.info("[getBinary] ${f.path}"); + throw UnimplementedError(); + } + + @override + putBinary(Account account, String path, Uint8List content) async { + _log.info("[putBinary] $path"); + // do nothing, we currently don't store file contents locally + } + + @override + updateMetadata(Account account, File f, Metadata metadata) { + _log.info("[updateMetadata] ${f.path}"); + return AppDb.use((db) async { + final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); + final store = transaction.objectStore(AppDb.fileStoreName); + final parentDir = File(path: path.dirname(f.path)); + final parentList = await _doList(store, account, parentDir); + final jsonList = parentList.map((e) { + if (e.path == f.path) { + return e.copyWith(metadata: OrNull(metadata)); + } else { + return e; + } + }); + await _cacheListResults(store, account, parentDir, jsonList); + }); + } + + @override + copy( + Account account, + File f, + String destination, { + bool shouldOverwrite, + }) async { + // do nothing + } + + @override + move( + Account account, + File f, + String destination, { + bool shouldOverwrite, + }) async { + // do nothing + } + + @override + createDir(Account account, String path) async { + // do nothing + } + + Future> _doList(ObjectStore store, Account account, File f) async { + final index = store.index(AppDbFileEntry.indexName); + final path = AppDbFileEntry.toPath(account, f); + final range = KeyRange.bound([path, 0], [path, int_util.int32Max]); + final List results = await index.getAll(range); + if (results?.isNotEmpty == true) { + final entries = results + .map((e) => AppDbFileEntry.fromJson(e.cast())); + return entries.map((e) { + _log.info("[_doList] ${e.path}[${e.index}]"); + return e.data; + }).reduce((value, element) => value + element); + } else { + throw CacheNotFoundException("No entry: $path"); + } + } + + static final _log = Logger("entity.file.data_source.FileAppDbDataSource"); +} + +class FileCachedDataSource implements FileDataSource { + FileCachedDataSource({ + this.shouldCheckCache = false, + }); + + @override + list(Account account, File f) async { + final cacheManager = _CacheManager( + appDbSrc: _appDbSrc, + remoteSrc: _remoteSrc, + shouldCheckCache: shouldCheckCache, + ); + final cache = await cacheManager.list(account, f); + if (cacheManager.isGood) { + return cache; + } + + // no cache or outdated + try { + final remote = await _remoteSrc.list(account, f); + await _cacheResult(account, f, remote); + if (shouldCheckCache) { + // update our local touch token to match the remote one + final tokenManager = TouchTokenManager(); + try { + await tokenManager.setLocalToken( + account, f, cacheManager.remoteTouchToken); + } catch (e, stacktrace) { + _log.shout("[list] Failed while setLocalToken", e, stacktrace); + // ignore error + } + } + + if (cache != null) { + try { + await _cleanUpCachedDir(account, remote, cache); + } catch (e, stacktrace) { + _log.shout("[list] Failed while _cleanUpCachedList", e, stacktrace); + // ignore error + } + } + return remote; + } on ApiException catch (e) { + if (e.response.statusCode == 404) { + _log.info("[list] File removed: $f"); + _appDbSrc.remove(account, f); + return []; + } else { + rethrow; + } + } + } + + @override + remove(Account account, File f) async { + await _appDbSrc.remove(account, f); + await _remoteSrc.remove(account, f); + } + + @override + getBinary(Account account, File f) { + return _remoteSrc.getBinary(account, f); + } + + @override + putBinary(Account account, String path, Uint8List content) async { + await _remoteSrc.putBinary(account, path, content); + } + + @override + updateMetadata(Account account, File f, Metadata metadata) async { + await _remoteSrc + .updateMetadata(account, f, metadata) + .then((_) => _appDbSrc.updateMetadata(account, f, metadata)); + + // generate a new random token + final token = Uuid().v4().replaceAll("-", ""); + final tokenManager = TouchTokenManager(); + final dir = File(path: path.dirname(f.path)); + await tokenManager.setLocalToken(account, dir, token); + final fileRepo = FileRepo(this); + await tokenManager.setRemoteToken(fileRepo, account, dir, token); + _log.info( + "[updateMetadata] New touch token '$token' for dir '${dir.path}'"); + } + + @override + copy( + Account account, + File f, + String destination, { + bool shouldOverwrite, + }) async { + await _remoteSrc.copy(account, f, destination, + shouldOverwrite: shouldOverwrite); + } + + @override + move( + Account account, + File f, + String destination, { + bool shouldOverwrite, + }) async { + await _remoteSrc.move(account, f, destination, + shouldOverwrite: shouldOverwrite); + } + + @override + createDir(Account account, String path) async { + await _remoteSrc.createDir(account, path); + } + + Future _cacheResult(Account account, File f, List result) { + return AppDb.use((db) async { + final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); + final store = transaction.objectStore(AppDb.fileStoreName); + await _cacheListResults(store, account, f, result); + }); + } + + /// Remove dangling dir entries in the file object store + Future _cleanUpCachedDir( + Account account, List remoteResults, List cachedResults) { + final removed = cachedResults + .where((cache) => + !remoteResults.any((remote) => remote.path == cache.path)) + .toList(); + if (removed.isEmpty) { + return Future.delayed(Duration.zero); + } + return AppDb.use((db) async { + final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite); + final store = transaction.objectStore(AppDb.fileStoreName); + final index = store.index(AppDbFileEntry.indexName); + for (final r in removed) { + final path = AppDbFileEntry.toPath(account, r); + final keys = []; + // delete the dir itself + final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]); + // delete with KeyRange is not supported in idb_shim/idb_sqflite + // await store.delete(dirRange); + keys.addAll(await index + .openKeyCursor(range: dirRange, autoAdvance: true) + .map((cursor) => cursor.primaryKey) + .toList()); + // then its children + final childrenRange = + KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]); + keys.addAll(await index + .openKeyCursor(range: childrenRange, autoAdvance: true) + .map((cursor) => cursor.primaryKey) + .toList()); + + for (final k in keys) { + _log.fine("[_cleanUpCachedDir] Removing DB entry: $k"); + await store.delete(k); + } + } + }); + } + + final bool shouldCheckCache; + + final _remoteSrc = FileWebdavDataSource(); + final _appDbSrc = FileAppDbDataSource(); + + static final _log = Logger("entity.file.data_source.FileCachedDataSource"); +} + +class _CacheManager { + _CacheManager({ + @required this.appDbSrc, + @required this.remoteSrc, + this.shouldCheckCache = false, + }); + + /// Return the cached results of listing a directory [f] + /// + /// Should check [isGood] before using the cache returning by this method + Future> list(Account account, File f) async { + final trimmedRootPath = f.path.trimAny("/"); + List cache; + try { + cache = await appDbSrc.list(account, f); + // compare the cached root + final cacheEtag = cache + .firstWhere((element) => element.path.trimAny("/") == trimmedRootPath) + .etag; + if (cacheEtag != null) { + // compare the etag to see if the content has been updated + var remoteEtag = f.etag; + if (remoteEtag == null) { + // no etag supplied, we need to query it form remote + final remote = await remoteSrc.list(account, f, depth: 0); + assert(remote.length == 1); + remoteEtag = remote.first.etag; + } + if (cacheEtag == remoteEtag) { + _log.fine( + "[_listCache] etag matched for ${AppDbFileEntry.toPath(account, f)}"); + if (shouldCheckCache) { + await _checkTouchToken(account, f, cache); + } else { + _isGood = true; + } + } + } else { + _log.info( + "[_list] Remote content updated for ${AppDbFileEntry.toPath(account, f)}"); + } + } on CacheNotFoundException catch (_) { + // normal when there's no cache + } catch (e, stacktrace) { + _log.shout("[_list] Cache failure", e, stacktrace); + } + return cache; + } + + bool get isGood => _isGood; + String get remoteTouchToken => _remoteToken; + + Future _checkTouchToken( + Account account, File f, List cache) async { + final touchPath = + "${remote_storage_util.getRemoteTouchDir(account)}/${f.strippedPath}"; + final fileRepo = FileRepo(FileCachedDataSource()); + final tokenManager = TouchTokenManager(); + String remoteToken; + try { + remoteToken = await tokenManager.getRemoteToken(fileRepo, account, f); + } catch (e, stacktrace) { + _log.shout( + "[_checkTouchToken] Failed getting remote token at '$touchPath'", + e, + stacktrace); + } + _remoteToken = remoteToken; + + String localToken; + try { + localToken = await tokenManager.getLocalToken(account, f); + } catch (e, stacktrace) { + _log.shout( + "[_checkTouchToken] Failed getting local token at '$touchPath'", + e, + stacktrace); + } + + if (localToken != remoteToken) { + _log.info( + "[_checkTouchToken] Remote and local token differ, cache outdated"); + } else { + _isGood = true; + } + } + + final FileWebdavDataSource remoteSrc; + final FileAppDbDataSource appDbSrc; + final bool shouldCheckCache; + + var _isGood = false; + String _remoteToken; + + static final _log = Logger("entity.file.data_source._CacheManager"); +} + +Future _cacheListResults( + ObjectStore store, Account account, File f, Iterable results) async { + final index = store.index(AppDbFileEntry.indexName); + final path = AppDbFileEntry.toPath(account, f); + final range = KeyRange.bound([path, 0], [path, int_util.int32Max]); + // count number of entries for this dir + final count = await index.count(range); + int newCount = 0; + for (final pair + in partition(results, AppDbFileEntry.maxDataSize).withIndex()) { + _log.info( + "[_cacheListResults] Caching $path[${pair.item1}], length: ${pair.item2.length}"); + await store.put( + AppDbFileEntry(path, pair.item1, pair.item2).toJson(), + AppDbFileEntry.toPrimaryKey(account, f, pair.item1), + ); + ++newCount; + } + if (count > newCount) { + // index is 0-based + final rmRange = KeyRange.bound([path, newCount], [path, int_util.int32Max]); + final rmKeys = await index + .openKeyCursor(range: rmRange, autoAdvance: true) + .map((cursor) => cursor.primaryKey) + .toList(); + for (final k in rmKeys) { + _log.fine("[_cacheListResults] Removing DB entry: $k"); + await store.delete(k); + } + } +} + +final _log = Logger("entity.file.data_source"); diff --git a/lib/metadata_task_manager.dart b/lib/metadata_task_manager.dart index 49d8628a..2068f065 100644 --- a/lib/metadata_task_manager.dart +++ b/lib/metadata_task_manager.dart @@ -4,6 +4,7 @@ import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/api/api_util.dart' as api_util; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/pref.dart'; import 'package:nc_photos/use_case/update_missing_metadata.dart'; diff --git a/lib/mobile/downloader.dart b/lib/mobile/downloader.dart index 93f3508b..e6444ae2 100644 --- a/lib/mobile/downloader.dart +++ b/lib/mobile/downloader.dart @@ -3,6 +3,7 @@ import 'dart:io'; import 'package:flutter/services.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/exception.dart'; import 'package:nc_photos/mobile/android/media_store.dart'; import 'package:nc_photos/platform/downloader.dart' as itf; diff --git a/lib/use_case/update_missing_metadata.dart b/lib/use_case/update_missing_metadata.dart index 482cd670..e6eaaae1 100644 --- a/lib/use_case/update_missing_metadata.dart +++ b/lib/use_case/update_missing_metadata.dart @@ -4,6 +4,7 @@ import 'package:nc_photos/connectivity_util.dart' as connectivity_util; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/exif.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/mobile/platform.dart' if (dart.library.html) 'package:nc_photos/web/platform.dart' as platform; import 'package:nc_photos/use_case/scan_missing_metadata.dart'; diff --git a/lib/web/downloader.dart b/lib/web/downloader.dart index c4890295..cb114fc4 100644 --- a/lib/web/downloader.dart +++ b/lib/web/downloader.dart @@ -5,6 +5,7 @@ import 'dart:js' as js; import 'package:nc_photos/account.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/platform/downloader.dart' as itf; import 'package:path/path.dart' as path; diff --git a/lib/widget/home_albums.dart b/lib/widget/home_albums.dart index a4a172bd..43e24c76 100644 --- a/lib/widget/home_albums.dart +++ b/lib/widget/home_albums.dart @@ -13,6 +13,7 @@ import 'package:nc_photos/api/api_util.dart' as api_util; import 'package:nc_photos/bloc/list_album.dart'; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/entity/file_util.dart' as file_util; import 'package:nc_photos/exception_util.dart' as exception_util; import 'package:nc_photos/iterable_extension.dart'; diff --git a/lib/widget/home_photos.dart b/lib/widget/home_photos.dart index f66b99ec..052b8c54 100644 --- a/lib/widget/home_photos.dart +++ b/lib/widget/home_photos.dart @@ -13,6 +13,7 @@ import 'package:nc_photos/api/api_util.dart' as api_util; import 'package:nc_photos/bloc/scan_dir.dart'; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/entity/file_util.dart' as file_util; import 'package:nc_photos/exception_util.dart' as exception_util; import 'package:nc_photos/iterable_extension.dart'; diff --git a/lib/widget/viewer.dart b/lib/widget/viewer.dart index 166bd887..99332405 100644 --- a/lib/widget/viewer.dart +++ b/lib/widget/viewer.dart @@ -9,6 +9,7 @@ import 'package:logging/logging.dart'; import 'package:nc_photos/account.dart'; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/entity/file_util.dart' as file_util; import 'package:nc_photos/exception.dart'; import 'package:nc_photos/exception_util.dart' as exception_util; diff --git a/lib/widget/viewer_detail_pane.dart b/lib/widget/viewer_detail_pane.dart index a67ee80c..68a1ee94 100644 --- a/lib/widget/viewer_detail_pane.dart +++ b/lib/widget/viewer_detail_pane.dart @@ -13,6 +13,7 @@ import 'package:nc_photos/double_extension.dart'; import 'package:nc_photos/entity/album.dart'; import 'package:nc_photos/entity/exif.dart'; import 'package:nc_photos/entity/file.dart'; +import 'package:nc_photos/entity/file/data_source.dart'; import 'package:nc_photos/exception_util.dart' as exception_util; import 'package:nc_photos/iterable_extension.dart'; import 'package:nc_photos/k.dart' as k;