Mirrors/nc-photos
1
0
Fork 0
mirror of https://gitlab.com/nkming2/nc-photos.git synced 2025-02-24 18:38:48 +01:00
Code Issues Projects Releases Packages Wiki Activity

Group data srcs in new file

Browse source
This commit is contained in:
Ming Ming 2021-05-24 15:09:25 +08:00
parent 5c3655d881
commit f7d0a41540
14 changed files with 613 additions and 593 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/bloc/list_album.dart
View file

@ -3,6 +3,7 @@ import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/album.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/event/event.dart';
import 'package:nc_photos/use_case/list_album.dart';

1
lib/bloc/ls_dir.dart
View file

@ -4,6 +4,7 @@ import 'package:bloc/bloc.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/iterable_extension.dart';
import 'package:nc_photos/use_case/ls.dart';

1
lib/bloc/scan_dir.dart
View file

@ -7,6 +7,7 @@ import 'package:kiwi/kiwi.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/event/event.dart';
import 'package:nc_photos/iterable_extension.dart';
import 'package:nc_photos/use_case/scan_dir.dart';

1
lib/entity/album.dart
View file

@ -10,6 +10,7 @@ import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/app_db.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/int_util.dart' as int_util;
import 'package:nc_photos/iterable_extension.dart';

593
lib/entity/file.dart
View file

@ -1,26 +1,12 @@
import 'dart:convert';
import 'dart:typed_data';
import 'package:equatable/equatable.dart';
import 'package:flutter/foundation.dart';
import 'package:idb_sqflite/idb_sqflite.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/api/api.dart';
import 'package:nc_photos/app_db.dart';
import 'package:nc_photos/entity/exif.dart';
import 'package:nc_photos/entity/webdav_response_parser.dart';
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/int_util.dart' as int_util;
import 'package:nc_photos/iterable_extension.dart';
import 'package:nc_photos/or_null.dart';
import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util;
import 'package:nc_photos/string_extension.dart';
import 'package:nc_photos/touch_token_manager.dart';
import 'package:path/path.dart' as path;
import 'package:quiver/iterables.dart';
import 'package:uuid/uuid.dart';
import 'package:xml/xml.dart';
int compareFileDateTimeDescending(File x, File y) {
final xDate = x.metadata?.exif?.dateTimeOriginal ?? x.lastModified;
@ -459,582 +445,3 @@ abstract class FileDataSource {
/// remote.php/dav/files/admin/new/dir
Future<void> createDir(Account account, String path);
}
class FileWebdavDataSource implements FileDataSource {
@override
list(
Account account,
File f, {
int depth,
}) async {
_log.fine("[list] ${f.path}");
final response = await Api(account).files().propfind(
path: f.path,
depth: depth,
getlastmodified: 1,
resourcetype: 1,
getetag: 1,
getcontenttype: 1,
getcontentlength: 1,
hasPreview: 1,
fileid: 1,
customNamespaces: {
"com.nkming.nc_photos": "app",
},
customProperties: [
"app:metadata",
],
);
if (!response.isGood) {
_log.severe("[list] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
final xml = XmlDocument.parse(response.body);
final files = WebdavFileParser()(xml);
// _log.fine("[list] Parsed files: [$files]");
return files.map((e) {
if (e.metadata == null || e.metadata.fileEtag == e.etag) {
return e;
} else {
_log.info("[list] Ignore outdated metadata for ${e.path}");
return e.copyWith(metadata: OrNull(null));
}
}).toList();
}
@override
remove(Account account, File f) async {
_log.info("[remove] ${f.path}");
final response = await Api(account).files().delete(path: f.path);
if (!response.isGood) {
_log.severe("[remove] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
getBinary(Account account, File f) async {
_log.info("[getBinary] ${f.path}");
final response = await Api(account).files().get(path: f.path);
if (!response.isGood) {
_log.severe("[getBinary] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
return response.body;
}
@override
putBinary(Account account, String path, Uint8List content) async {
_log.info("[putBinary] $path");
final response =
await Api(account).files().put(path: path, content: content);
if (!response.isGood) {
_log.severe("[putBinary] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
updateMetadata(Account account, File f, Metadata metadata) async {
_log.info("[updateMetadata] ${f.path}");
if (metadata != null && metadata.fileEtag != f.etag) {
_log.warning(
"[updateMetadata] etag mismatch (metadata: ${metadata.fileEtag}, file: ${f.etag})");
}
final setProps = {
if (metadata != null) "app:metadata": jsonEncode(metadata.toJson()),
};
final removeProps = [
if (metadata == null) "app:metadata",
];
final response = await Api(account).files().proppatch(
path: f.path,
namespaces: {
"com.nkming.nc_photos": "app",
},
set: setProps.isNotEmpty ? setProps : null,
remove: removeProps.isNotEmpty ? removeProps : null,
);
if (!response.isGood) {
_log.severe("[updateMetadata] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
copy(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
_log.info("[copy] ${f.path} to $destination");
final response = await Api(account).files().copy(
path: f.path,
destinationUrl: "${account.url}/$destination",
overwrite: shouldOverwrite,
);
if (!response.isGood) {
_log.severe("[copy] Failed requesting sever: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
move(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
_log.info("[move] ${f.path} to $destination");
final response = await Api(account).files().move(
path: f.path,
destinationUrl: "${account.url}/$destination",
overwrite: shouldOverwrite,
);
if (!response.isGood) {
_log.severe("[move] Failed requesting sever: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
createDir(Account account, String path) async {
_log.info("[createDir] $path");
final response = await Api(account).files().mkcol(
path: path,
);
if (!response.isGood) {
_log.severe("[createDir] Failed requesting sever: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
static final _log = Logger("entity.file.FileWebdavDataSource");
}
class FileAppDbDataSource implements FileDataSource {
@override
list(Account account, File f) {
_log.info("[list] ${f.path}");
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadOnly);
final store = transaction.objectStore(AppDb.fileStoreName);
return await _doList(store, account, f);
});
}
@override
remove(Account account, File f) {
_log.info("[remove] ${f.path}");
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
final index = store.index(AppDbFileEntry.indexName);
final path = AppDbFileEntry.toPath(account, f);
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
final keys = await index
.openKeyCursor(range: range, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList();
for (final k in keys) {
_log.fine("[remove] Removing DB entry: $k");
await store.delete(k);
}
});
}
@override
getBinary(Account account, File f) {
_log.info("[getBinary] ${f.path}");
throw UnimplementedError();
}
@override
putBinary(Account account, String path, Uint8List content) async {
_log.info("[putBinary] $path");
// do nothing, we currently don't store file contents locally
}
@override
updateMetadata(Account account, File f, Metadata metadata) {
_log.info("[updateMetadata] ${f.path}");
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
final parentDir = File(path: path.dirname(f.path));
final parentList = await _doList(store, account, parentDir);
final jsonList = parentList.map((e) {
if (e.path == f.path) {
return e.copyWith(metadata: OrNull(metadata));
} else {
return e;
}
});
await _cacheListResults(store, account, parentDir, jsonList);
});
}
@override
copy(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
// do nothing
}
@override
move(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
// do nothing
}
@override
createDir(Account account, String path) async {
// do nothing
}
Future<List<File>> _doList(ObjectStore store, Account account, File f) async {
final index = store.index(AppDbFileEntry.indexName);
final path = AppDbFileEntry.toPath(account, f);
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
final List results = await index.getAll(range);
if (results?.isNotEmpty == true) {
final entries = results
.map((e) => AppDbFileEntry.fromJson(e.cast<String, dynamic>()));
return entries.map((e) {
_log.info("[_doList] ${e.path}[${e.index}]");
return e.data;
}).reduce((value, element) => value + element);
} else {
throw CacheNotFoundException("No entry: $path");
}
}
static final _log = Logger("entity.file.FileAppDbDataSource");
}
class FileCachedDataSource implements FileDataSource {
FileCachedDataSource({
this.shouldCheckCache = false,
});
@override
list(Account account, File f) async {
final cacheManager = _CacheManager(
appDbSrc: _appDbSrc,
remoteSrc: _remoteSrc,
shouldCheckCache: shouldCheckCache,
);
final cache = await cacheManager.list(account, f);
if (cacheManager.isGood) {
return cache;
}
// no cache or outdated
try {
final remote = await _remoteSrc.list(account, f);
await _cacheResult(account, f, remote);
if (shouldCheckCache) {
// update our local touch token to match the remote one
final tokenManager = TouchTokenManager();
try {
await tokenManager.setLocalToken(
account, f, cacheManager.remoteTouchToken);
} catch (e, stacktrace) {
_log.shout("[list] Failed while setLocalToken", e, stacktrace);
// ignore error
}
}
if (cache != null) {
try {
await _cleanUpCachedDir(account, remote, cache);
} catch (e, stacktrace) {
_log.shout("[list] Failed while _cleanUpCachedList", e, stacktrace);
// ignore error
}
}
return remote;
} on ApiException catch (e) {
if (e.response.statusCode == 404) {
_log.info("[list] File removed: $f");
_appDbSrc.remove(account, f);
return [];
} else {
rethrow;
}
}
}
@override
remove(Account account, File f) async {
await _appDbSrc.remove(account, f);
await _remoteSrc.remove(account, f);
}
@override
getBinary(Account account, File f) {
return _remoteSrc.getBinary(account, f);
}
@override
putBinary(Account account, String path, Uint8List content) async {
await _remoteSrc.putBinary(account, path, content);
}
@override
updateMetadata(Account account, File f, Metadata metadata) async {
await _remoteSrc
.updateMetadata(account, f, metadata)
.then((_) => _appDbSrc.updateMetadata(account, f, metadata));
// generate a new random token
final token = Uuid().v4().replaceAll("-", "");
final tokenManager = TouchTokenManager();
final dir = File(path: path.dirname(f.path));
await tokenManager.setLocalToken(account, dir, token);
final fileRepo = FileRepo(this);
await tokenManager.setRemoteToken(fileRepo, account, dir, token);
_log.info(
"[updateMetadata] New touch token '$token' for dir '${dir.path}'");
}
@override
copy(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
await _remoteSrc.copy(account, f, destination,
shouldOverwrite: shouldOverwrite);
}
@override
move(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
await _remoteSrc.move(account, f, destination,
shouldOverwrite: shouldOverwrite);
}
@override
createDir(Account account, String path) async {
await _remoteSrc.createDir(account, path);
}
Future<void> _cacheResult(Account account, File f, List<File> result) {
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
await _cacheListResults(store, account, f, result);
});
}
/// Remove dangling dir entries in the file object store
Future<void> _cleanUpCachedDir(
Account account, List<File> remoteResults, List<File> cachedResults) {
final removed = cachedResults
.where((cache) =>
!remoteResults.any((remote) => remote.path == cache.path))
.toList();
if (removed.isEmpty) {
return Future.delayed(Duration.zero);
}
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
final index = store.index(AppDbFileEntry.indexName);
for (final r in removed) {
final path = AppDbFileEntry.toPath(account, r);
final keys = [];
// delete the dir itself
final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]);
// delete with KeyRange is not supported in idb_shim/idb_sqflite
// await store.delete(dirRange);
keys.addAll(await index
.openKeyCursor(range: dirRange, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList());
// then its children
final childrenRange =
KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]);
keys.addAll(await index
.openKeyCursor(range: childrenRange, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList());
for (final k in keys) {
_log.fine("[_cleanUpCachedDir] Removing DB entry: $k");
await store.delete(k);
}
}
});
}
final bool shouldCheckCache;
final _remoteSrc = FileWebdavDataSource();
final _appDbSrc = FileAppDbDataSource();
static final _log = Logger("entity.file.FileCachedDataSource");
}
class _CacheManager {
_CacheManager({
@required this.appDbSrc,
@required this.remoteSrc,
this.shouldCheckCache = false,
});
/// Return the cached results of listing a directory [f]
///
/// Should check [isGood] before using the cache returning by this method
Future<List<File>> list(Account account, File f) async {
final trimmedRootPath = f.path.trimAny("/");
List<File> cache;
try {
cache = await appDbSrc.list(account, f);
// compare the cached root
final cacheEtag = cache
.firstWhere((element) => element.path.trimAny("/") == trimmedRootPath)
.etag;
if (cacheEtag != null) {
// compare the etag to see if the content has been updated
var remoteEtag = f.etag;
if (remoteEtag == null) {
// no etag supplied, we need to query it form remote
final remote = await remoteSrc.list(account, f, depth: 0);
assert(remote.length == 1);
remoteEtag = remote.first.etag;
}
if (cacheEtag == remoteEtag) {
_log.fine(
"[_listCache] etag matched for ${AppDbFileEntry.toPath(account, f)}");
if (shouldCheckCache) {
await _checkTouchToken(account, f, cache);
} else {
_isGood = true;
}
}
} else {
_log.info(
"[_list] Remote content updated for ${AppDbFileEntry.toPath(account, f)}");
}
} on CacheNotFoundException catch (_) {
// normal when there's no cache
} catch (e, stacktrace) {
_log.shout("[_list] Cache failure", e, stacktrace);
}
return cache;
}
bool get isGood => _isGood;
String get remoteTouchToken => _remoteToken;
Future<void> _checkTouchToken(
Account account, File f, List<File> cache) async {
final touchPath =
"${remote_storage_util.getRemoteTouchDir(account)}/${f.strippedPath}";
final fileRepo = FileRepo(FileCachedDataSource());
final tokenManager = TouchTokenManager();
String remoteToken;
try {
remoteToken = await tokenManager.getRemoteToken(fileRepo, account, f);
} catch (e, stacktrace) {
_log.shout(
"[_checkTouchToken] Failed getting remote token at '$touchPath'",
e,
stacktrace);
}
_remoteToken = remoteToken;
String localToken;
try {
localToken = await tokenManager.getLocalToken(account, f);
} catch (e, stacktrace) {
_log.shout(
"[_checkTouchToken] Failed getting local token at '$touchPath'",
e,
stacktrace);
}
if (localToken != remoteToken) {
_log.info(
"[_checkTouchToken] Remote and local token differ, cache outdated");
} else {
_isGood = true;
}
}
final FileWebdavDataSource remoteSrc;
final FileAppDbDataSource appDbSrc;
final bool shouldCheckCache;
var _isGood = false;
String _remoteToken;
static final _log = Logger("entity.file._CacheManager");
}
Future<void> _cacheListResults(
ObjectStore store, Account account, File f, Iterable<File> results) async {
final index = store.index(AppDbFileEntry.indexName);
final path = AppDbFileEntry.toPath(account, f);
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
// count number of entries for this dir
final count = await index.count(range);
int newCount = 0;
for (final pair
in partition(results, AppDbFileEntry.maxDataSize).withIndex()) {
_log.info(
"[_cacheListResults] Caching $path[${pair.item1}], length: ${pair.item2.length}");
await store.put(
AppDbFileEntry(path, pair.item1, pair.item2).toJson(),
AppDbFileEntry.toPrimaryKey(account, f, pair.item1),
);
++newCount;
}
if (count > newCount) {
// index is 0-based
final rmRange = KeyRange.bound([path, newCount], [path, int_util.int32Max]);
final rmKeys = await index
.openKeyCursor(range: rmRange, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList();
for (final k in rmKeys) {
_log.fine("[_cacheListResults] Removing DB entry: $k");
await store.delete(k);
}
}
}
final _log = Logger("entity.file");

601
lib/entity/file/data_source.dart Normal file
View file

@ -0,0 +1,601 @@
import 'dart:convert';
import 'dart:typed_data';
import 'package:flutter/foundation.dart';
import 'package:idb_shim/idb_client.dart';
import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/api/api.dart';
import 'package:nc_photos/app_db.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/webdav_response_parser.dart';
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/int_util.dart' as int_util;
import 'package:nc_photos/iterable_extension.dart';
import 'package:nc_photos/or_null.dart';
import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util;
import 'package:nc_photos/string_extension.dart';
import 'package:nc_photos/touch_token_manager.dart';
import 'package:path/path.dart' as path;
import 'package:quiver/iterables.dart';
import 'package:uuid/uuid.dart';
import 'package:xml/xml.dart';
class FileWebdavDataSource implements FileDataSource {
@override
list(
Account account,
File f, {
int depth,
}) async {
_log.fine("[list] ${f.path}");
final response = await Api(account).files().propfind(
path: f.path,
depth: depth,
getlastmodified: 1,
resourcetype: 1,
getetag: 1,
getcontenttype: 1,
getcontentlength: 1,
hasPreview: 1,
fileid: 1,
customNamespaces: {
"com.nkming.nc_photos": "app",
},
customProperties: [
"app:metadata",
],
);
if (!response.isGood) {
_log.severe("[list] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
final xml = XmlDocument.parse(response.body);
final files = WebdavFileParser()(xml);
// _log.fine("[list] Parsed files: [$files]");
return files.map((e) {
if (e.metadata == null || e.metadata.fileEtag == e.etag) {
return e;
} else {
_log.info("[list] Ignore outdated metadata for ${e.path}");
return e.copyWith(metadata: OrNull(null));
}
}).toList();
}
@override
remove(Account account, File f) async {
_log.info("[remove] ${f.path}");
final response = await Api(account).files().delete(path: f.path);
if (!response.isGood) {
_log.severe("[remove] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
getBinary(Account account, File f) async {
_log.info("[getBinary] ${f.path}");
final response = await Api(account).files().get(path: f.path);
if (!response.isGood) {
_log.severe("[getBinary] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
return response.body;
}
@override
putBinary(Account account, String path, Uint8List content) async {
_log.info("[putBinary] $path");
final response =
await Api(account).files().put(path: path, content: content);
if (!response.isGood) {
_log.severe("[putBinary] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
updateMetadata(Account account, File f, Metadata metadata) async {
_log.info("[updateMetadata] ${f.path}");
if (metadata != null && metadata.fileEtag != f.etag) {
_log.warning(
"[updateMetadata] etag mismatch (metadata: ${metadata.fileEtag}, file: ${f.etag})");
}
final setProps = {
if (metadata != null) "app:metadata": jsonEncode(metadata.toJson()),
};
final removeProps = [
if (metadata == null) "app:metadata",
];
final response = await Api(account).files().proppatch(
path: f.path,
namespaces: {
"com.nkming.nc_photos": "app",
},
set: setProps.isNotEmpty ? setProps : null,
remove: removeProps.isNotEmpty ? removeProps : null,
);
if (!response.isGood) {
_log.severe("[updateMetadata] Failed requesting server: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
copy(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
_log.info("[copy] ${f.path} to $destination");
final response = await Api(account).files().copy(
path: f.path,
destinationUrl: "${account.url}/$destination",
overwrite: shouldOverwrite,
);
if (!response.isGood) {
_log.severe("[copy] Failed requesting sever: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
move(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
_log.info("[move] ${f.path} to $destination");
final response = await Api(account).files().move(
path: f.path,
destinationUrl: "${account.url}/$destination",
overwrite: shouldOverwrite,
);
if (!response.isGood) {
_log.severe("[move] Failed requesting sever: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
@override
createDir(Account account, String path) async {
_log.info("[createDir] $path");
final response = await Api(account).files().mkcol(
path: path,
);
if (!response.isGood) {
_log.severe("[createDir] Failed requesting sever: $response");
throw ApiException(
response: response,
message: "Failed communicating with server: ${response.statusCode}");
}
}
static final _log = Logger("entity.file.data_source.FileWebdavDataSource");
}
class FileAppDbDataSource implements FileDataSource {
@override
list(Account account, File f) {
_log.info("[list] ${f.path}");
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadOnly);
final store = transaction.objectStore(AppDb.fileStoreName);
return await _doList(store, account, f);
});
}
@override
remove(Account account, File f) {
_log.info("[remove] ${f.path}");
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
final index = store.index(AppDbFileEntry.indexName);
final path = AppDbFileEntry.toPath(account, f);
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
final keys = await index
.openKeyCursor(range: range, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList();
for (final k in keys) {
_log.fine("[remove] Removing DB entry: $k");
await store.delete(k);
}
});
}
@override
getBinary(Account account, File f) {
_log.info("[getBinary] ${f.path}");
throw UnimplementedError();
}
@override
putBinary(Account account, String path, Uint8List content) async {
_log.info("[putBinary] $path");
// do nothing, we currently don't store file contents locally
}
@override
updateMetadata(Account account, File f, Metadata metadata) {
_log.info("[updateMetadata] ${f.path}");
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
final parentDir = File(path: path.dirname(f.path));
final parentList = await _doList(store, account, parentDir);
final jsonList = parentList.map((e) {
if (e.path == f.path) {
return e.copyWith(metadata: OrNull(metadata));
} else {
return e;
}
});
await _cacheListResults(store, account, parentDir, jsonList);
});
}
@override
copy(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
// do nothing
}
@override
move(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
// do nothing
}
@override
createDir(Account account, String path) async {
// do nothing
}
Future<List<File>> _doList(ObjectStore store, Account account, File f) async {
final index = store.index(AppDbFileEntry.indexName);
final path = AppDbFileEntry.toPath(account, f);
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
final List results = await index.getAll(range);
if (results?.isNotEmpty == true) {
final entries = results
.map((e) => AppDbFileEntry.fromJson(e.cast<String, dynamic>()));
return entries.map((e) {
_log.info("[_doList] ${e.path}[${e.index}]");
return e.data;
}).reduce((value, element) => value + element);
} else {
throw CacheNotFoundException("No entry: $path");
}
}
static final _log = Logger("entity.file.data_source.FileAppDbDataSource");
}
class FileCachedDataSource implements FileDataSource {
FileCachedDataSource({
this.shouldCheckCache = false,
});
@override
list(Account account, File f) async {
final cacheManager = _CacheManager(
appDbSrc: _appDbSrc,
remoteSrc: _remoteSrc,
shouldCheckCache: shouldCheckCache,
);
final cache = await cacheManager.list(account, f);
if (cacheManager.isGood) {
return cache;
}
// no cache or outdated
try {
final remote = await _remoteSrc.list(account, f);
await _cacheResult(account, f, remote);
if (shouldCheckCache) {
// update our local touch token to match the remote one
final tokenManager = TouchTokenManager();
try {
await tokenManager.setLocalToken(
account, f, cacheManager.remoteTouchToken);
} catch (e, stacktrace) {
_log.shout("[list] Failed while setLocalToken", e, stacktrace);
// ignore error
}
}
if (cache != null) {
try {
await _cleanUpCachedDir(account, remote, cache);
} catch (e, stacktrace) {
_log.shout("[list] Failed while _cleanUpCachedList", e, stacktrace);
// ignore error
}
}
return remote;
} on ApiException catch (e) {
if (e.response.statusCode == 404) {
_log.info("[list] File removed: $f");
_appDbSrc.remove(account, f);
return [];
} else {
rethrow;
}
}
}
@override
remove(Account account, File f) async {
await _appDbSrc.remove(account, f);
await _remoteSrc.remove(account, f);
}
@override
getBinary(Account account, File f) {
return _remoteSrc.getBinary(account, f);
}
@override
putBinary(Account account, String path, Uint8List content) async {
await _remoteSrc.putBinary(account, path, content);
}
@override
updateMetadata(Account account, File f, Metadata metadata) async {
await _remoteSrc
.updateMetadata(account, f, metadata)
.then((_) => _appDbSrc.updateMetadata(account, f, metadata));
// generate a new random token
final token = Uuid().v4().replaceAll("-", "");
final tokenManager = TouchTokenManager();
final dir = File(path: path.dirname(f.path));
await tokenManager.setLocalToken(account, dir, token);
final fileRepo = FileRepo(this);
await tokenManager.setRemoteToken(fileRepo, account, dir, token);
_log.info(
"[updateMetadata] New touch token '$token' for dir '${dir.path}'");
}
@override
copy(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
await _remoteSrc.copy(account, f, destination,
shouldOverwrite: shouldOverwrite);
}
@override
move(
Account account,
File f,
String destination, {
bool shouldOverwrite,
}) async {
await _remoteSrc.move(account, f, destination,
shouldOverwrite: shouldOverwrite);
}
@override
createDir(Account account, String path) async {
await _remoteSrc.createDir(account, path);
}
Future<void> _cacheResult(Account account, File f, List<File> result) {
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
await _cacheListResults(store, account, f, result);
});
}
/// Remove dangling dir entries in the file object store
Future<void> _cleanUpCachedDir(
Account account, List<File> remoteResults, List<File> cachedResults) {
final removed = cachedResults
.where((cache) =>
!remoteResults.any((remote) => remote.path == cache.path))
.toList();
if (removed.isEmpty) {
return Future.delayed(Duration.zero);
}
return AppDb.use((db) async {
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
final store = transaction.objectStore(AppDb.fileStoreName);
final index = store.index(AppDbFileEntry.indexName);
for (final r in removed) {
final path = AppDbFileEntry.toPath(account, r);
final keys = [];
// delete the dir itself
final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]);
// delete with KeyRange is not supported in idb_shim/idb_sqflite
// await store.delete(dirRange);
keys.addAll(await index
.openKeyCursor(range: dirRange, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList());
// then its children
final childrenRange =
KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]);
keys.addAll(await index
.openKeyCursor(range: childrenRange, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList());
for (final k in keys) {
_log.fine("[_cleanUpCachedDir] Removing DB entry: $k");
await store.delete(k);
}
}
});
}
final bool shouldCheckCache;
final _remoteSrc = FileWebdavDataSource();
final _appDbSrc = FileAppDbDataSource();
static final _log = Logger("entity.file.data_source.FileCachedDataSource");
}
class _CacheManager {
_CacheManager({
@required this.appDbSrc,
@required this.remoteSrc,
this.shouldCheckCache = false,
});
/// Return the cached results of listing a directory [f]
///
/// Should check [isGood] before using the cache returning by this method
Future<List<File>> list(Account account, File f) async {
final trimmedRootPath = f.path.trimAny("/");
List<File> cache;
try {
cache = await appDbSrc.list(account, f);
// compare the cached root
final cacheEtag = cache
.firstWhere((element) => element.path.trimAny("/") == trimmedRootPath)
.etag;
if (cacheEtag != null) {
// compare the etag to see if the content has been updated
var remoteEtag = f.etag;
if (remoteEtag == null) {
// no etag supplied, we need to query it form remote
final remote = await remoteSrc.list(account, f, depth: 0);
assert(remote.length == 1);
remoteEtag = remote.first.etag;
}
if (cacheEtag == remoteEtag) {
_log.fine(
"[_listCache] etag matched for ${AppDbFileEntry.toPath(account, f)}");
if (shouldCheckCache) {
await _checkTouchToken(account, f, cache);
} else {
_isGood = true;
}
}
} else {
_log.info(
"[_list] Remote content updated for ${AppDbFileEntry.toPath(account, f)}");
}
} on CacheNotFoundException catch (_) {
// normal when there's no cache
} catch (e, stacktrace) {
_log.shout("[_list] Cache failure", e, stacktrace);
}
return cache;
}
bool get isGood => _isGood;
String get remoteTouchToken => _remoteToken;
Future<void> _checkTouchToken(
Account account, File f, List<File> cache) async {
final touchPath =
"${remote_storage_util.getRemoteTouchDir(account)}/${f.strippedPath}";
final fileRepo = FileRepo(FileCachedDataSource());
final tokenManager = TouchTokenManager();
String remoteToken;
try {
remoteToken = await tokenManager.getRemoteToken(fileRepo, account, f);
} catch (e, stacktrace) {
_log.shout(
"[_checkTouchToken] Failed getting remote token at '$touchPath'",
e,
stacktrace);
}
_remoteToken = remoteToken;
String localToken;
try {
localToken = await tokenManager.getLocalToken(account, f);
} catch (e, stacktrace) {
_log.shout(
"[_checkTouchToken] Failed getting local token at '$touchPath'",
e,
stacktrace);
}
if (localToken != remoteToken) {
_log.info(
"[_checkTouchToken] Remote and local token differ, cache outdated");
} else {
_isGood = true;
}
}
final FileWebdavDataSource remoteSrc;
final FileAppDbDataSource appDbSrc;
final bool shouldCheckCache;
var _isGood = false;
String _remoteToken;
static final _log = Logger("entity.file.data_source._CacheManager");
}
Future<void> _cacheListResults(
ObjectStore store, Account account, File f, Iterable<File> results) async {
final index = store.index(AppDbFileEntry.indexName);
final path = AppDbFileEntry.toPath(account, f);
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
// count number of entries for this dir
final count = await index.count(range);
int newCount = 0;
for (final pair
in partition(results, AppDbFileEntry.maxDataSize).withIndex()) {
_log.info(
"[_cacheListResults] Caching $path[${pair.item1}], length: ${pair.item2.length}");
await store.put(
AppDbFileEntry(path, pair.item1, pair.item2).toJson(),
AppDbFileEntry.toPrimaryKey(account, f, pair.item1),
);
++newCount;
}
if (count > newCount) {
// index is 0-based
final rmRange = KeyRange.bound([path, newCount], [path, int_util.int32Max]);
final rmKeys = await index
.openKeyCursor(range: rmRange, autoAdvance: true)
.map((cursor) => cursor.primaryKey)
.toList();
for (final k in rmKeys) {
_log.fine("[_cacheListResults] Removing DB entry: $k");
await store.delete(k);
}
}
}
final _log = Logger("entity.file.data_source");

1
lib/metadata_task_manager.dart
View file

@ -4,6 +4,7 @@ import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/api/api_util.dart' as api_util;
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/pref.dart';
import 'package:nc_photos/use_case/update_missing_metadata.dart';

1
lib/mobile/downloader.dart
View file

@ -3,6 +3,7 @@ import 'dart:io';
import 'package:flutter/services.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/mobile/android/media_store.dart';
import 'package:nc_photos/platform/downloader.dart' as itf;

1
lib/use_case/update_missing_metadata.dart
View file

@ -4,6 +4,7 @@ import 'package:nc_photos/connectivity_util.dart' as connectivity_util;
import 'package:nc_photos/entity/album.dart';
import 'package:nc_photos/entity/exif.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/mobile/platform.dart'
if (dart.library.html) 'package:nc_photos/web/platform.dart' as platform;
import 'package:nc_photos/use_case/scan_missing_metadata.dart';

1
lib/web/downloader.dart
View file

@ -5,6 +5,7 @@ import 'dart:js' as js;
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/platform/downloader.dart' as itf;
import 'package:path/path.dart' as path;

1
lib/widget/home_albums.dart
View file

@ -13,6 +13,7 @@ import 'package:nc_photos/api/api_util.dart' as api_util;
import 'package:nc_photos/bloc/list_album.dart';
import 'package:nc_photos/entity/album.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/exception_util.dart' as exception_util;
import 'package:nc_photos/iterable_extension.dart';

1
lib/widget/home_photos.dart
View file

@ -13,6 +13,7 @@ import 'package:nc_photos/api/api_util.dart' as api_util;
import 'package:nc_photos/bloc/scan_dir.dart';
import 'package:nc_photos/entity/album.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/exception_util.dart' as exception_util;
import 'package:nc_photos/iterable_extension.dart';

1
lib/widget/viewer.dart
View file

@ -9,6 +9,7 @@ import 'package:logging/logging.dart';
import 'package:nc_photos/account.dart';
import 'package:nc_photos/entity/album.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/entity/file_util.dart' as file_util;
import 'package:nc_photos/exception.dart';
import 'package:nc_photos/exception_util.dart' as exception_util;

1
lib/widget/viewer_detail_pane.dart
View file

@ -13,6 +13,7 @@ import 'package:nc_photos/double_extension.dart';
import 'package:nc_photos/entity/album.dart';
import 'package:nc_photos/entity/exif.dart';
import 'package:nc_photos/entity/file.dart';
import 'package:nc_photos/entity/file/data_source.dart';
import 'package:nc_photos/exception_util.dart' as exception_util;
import 'package:nc_photos/iterable_extension.dart';
import 'package:nc_photos/k.dart' as k;