mirror of
https://gitlab.com/nkming2/nc-photos.git
synced 2025-03-28 09:51:36 +01:00
Rewrite how file data are stored in local DB
This commit is contained in:
Ming Ming
parent
deb3014f54
commit
430943e678
13 changed files with 589 additions and 379 deletions
236
lib/app_db.dart
236
lib/app_db.dart
|
|
@ -1,24 +1,25 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:equatable/equatable.dart';
|
||||
import 'package:idb_shim/idb.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:nc_photos/account.dart';
|
||||
import 'package:nc_photos/ci_string.dart';
|
||||
import 'package:nc_photos/entity/album.dart';
|
||||
import 'package:nc_photos/entity/album/upgrader.dart';
|
||||
import 'package:nc_photos/entity/file.dart';
|
||||
import 'package:nc_photos/mobile/platform.dart'
|
||||
if (dart.library.html) 'package:nc_photos/web/platform.dart' as platform;
|
||||
import 'package:nc_photos/object_extension.dart';
|
||||
import 'package:nc_photos/type.dart';
|
||||
import 'package:synchronized/synchronized.dart';
|
||||
|
||||
class AppDb {
|
||||
static const dbName = "app.db";
|
||||
static const dbVersion = 3;
|
||||
static const fileStoreName = "files";
|
||||
static const dbVersion = 4;
|
||||
static const albumStoreName = "albums";
|
||||
|
||||
/// this is a stupid name but 'files' is already being used so...
|
||||
static const fileDbStoreName = "filesDb";
|
||||
static const file2StoreName = "files2";
|
||||
static const dirStoreName = "dirs";
|
||||
|
||||
factory AppDb() => _inst;
|
||||
|
||||
|
|
@ -49,7 +50,8 @@ class AppDb {
|
|||
_log.info("[_open] Upgrade database: ${event.oldVersion} -> $dbVersion");
|
||||
|
||||
final db = event.database;
|
||||
ObjectStore fileStore, albumStore, fileDbStore;
|
||||
// ignore: unused_local_variable
|
||||
ObjectStore? albumStore, file2Store, dirStore;
|
||||
if (event.oldVersion < 2) {
|
||||
// version 2 store things in a new way, just drop all
|
||||
try {
|
||||
|
|
@ -61,20 +63,24 @@ class AppDb {
|
|||
}
|
||||
if (event.oldVersion < 3) {
|
||||
// new object store in v3
|
||||
try {
|
||||
db.deleteObjectStore(fileDbStoreName);
|
||||
} catch (_) {}
|
||||
fileDbStore = db.createObjectStore(fileDbStoreName);
|
||||
fileDbStore.createIndex(
|
||||
AppDbFileDbEntry.indexName, AppDbFileDbEntry.keyPath,
|
||||
unique: false);
|
||||
// no longer relevant in v4
|
||||
|
||||
// recreate file store from scratch
|
||||
// no longer relevant in v4
|
||||
}
|
||||
if (event.oldVersion < 4) {
|
||||
try {
|
||||
db.deleteObjectStore(fileStoreName);
|
||||
db.deleteObjectStore(_fileDbStoreName);
|
||||
} catch (_) {}
|
||||
fileStore = db.createObjectStore(fileStoreName);
|
||||
fileStore.createIndex(AppDbFileEntry.indexName, AppDbFileEntry.keyPath);
|
||||
try {
|
||||
db.deleteObjectStore(_fileStoreName);
|
||||
} catch (_) {}
|
||||
|
||||
file2Store = db.createObjectStore(file2StoreName);
|
||||
file2Store.createIndex(AppDbFile2Entry.strippedPathIndexName,
|
||||
AppDbFile2Entry.strippedPathKeyPath);
|
||||
|
||||
dirStore = db.createObjectStore(dirStoreName);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -82,46 +88,12 @@ class AppDb {
|
|||
static late final _inst = AppDb._();
|
||||
final _lock = Lock(reentrant: true);
|
||||
|
||||
static const _fileDbStoreName = "filesDb";
|
||||
static const _fileStoreName = "files";
|
||||
|
||||
static final _log = Logger("app_db.AppDb");
|
||||
}
|
||||
|
||||
class AppDbFileEntry {
|
||||
static const indexName = "fileStore_path_index";
|
||||
static const keyPath = ["path", "index"];
|
||||
static const maxDataSize = 160;
|
||||
|
||||
AppDbFileEntry(this.path, this.index, this.data);
|
||||
|
||||
JsonObj toJson() {
|
||||
return {
|
||||
"path": path,
|
||||
"index": index,
|
||||
"data": data.map((e) => e.toJson()).toList(),
|
||||
};
|
||||
}
|
||||
|
||||
factory AppDbFileEntry.fromJson(JsonObj json) {
|
||||
return AppDbFileEntry(
|
||||
json["path"],
|
||||
json["index"],
|
||||
json["data"]
|
||||
.map((e) => File.fromJson(e.cast<String, dynamic>()))
|
||||
.cast<File>()
|
||||
.toList(),
|
||||
);
|
||||
}
|
||||
|
||||
static String toPath(Account account, File dir) =>
|
||||
"${account.url}/${dir.path}";
|
||||
|
||||
static String toPrimaryKey(Account account, File dir, int index) =>
|
||||
"${toPath(account, dir)}[$index]";
|
||||
|
||||
final String path;
|
||||
final int index;
|
||||
final List<File> data;
|
||||
}
|
||||
|
||||
class AppDbAlbumEntry {
|
||||
static const indexName = "albumStore_path_index";
|
||||
static const keyPath = ["path", "index"];
|
||||
|
|
@ -164,37 +136,141 @@ class AppDbAlbumEntry {
|
|||
final Album album;
|
||||
}
|
||||
|
||||
class AppDbFileDbEntry {
|
||||
static const indexName = "fileDbStore_namespacedFileId";
|
||||
static const keyPath = "namespacedFileId";
|
||||
class AppDbFile2Entry with EquatableMixin {
|
||||
static const strippedPathIndexName = "server_userId_strippedPath";
|
||||
static const strippedPathKeyPath = ["server", "userId", "strippedPath"];
|
||||
|
||||
AppDbFileDbEntry(this.namespacedFileId, this.file);
|
||||
AppDbFile2Entry._(this.server, this.userId, this.strippedPath, this.file);
|
||||
|
||||
factory AppDbFileDbEntry.fromFile(Account account, File file) {
|
||||
return AppDbFileDbEntry(toNamespacedFileId(account, file.fileId!), file);
|
||||
factory AppDbFile2Entry.fromFile(Account account, File file) =>
|
||||
AppDbFile2Entry._(
|
||||
account.url, account.username, file.strippedPathWithEmpty, file);
|
||||
|
||||
factory AppDbFile2Entry.fromJson(JsonObj json) => AppDbFile2Entry._(
|
||||
json["server"],
|
||||
(json["userId"] as String).toCi(),
|
||||
json["strippedPath"],
|
||||
File.fromJson(json["file"].cast<String, dynamic>()),
|
||||
);
|
||||
|
||||
JsonObj toJson() => {
|
||||
"server": server,
|
||||
"userId": userId.toCaseInsensitiveString(),
|
||||
"strippedPath": strippedPath,
|
||||
"file": file.toJson(),
|
||||
};
|
||||
|
||||
static String toPrimaryKey(Account account, int fileId) =>
|
||||
"${account.url}/${account.username.toCaseInsensitiveString()}/$fileId";
|
||||
|
||||
static String toPrimaryKeyForFile(Account account, File file) =>
|
||||
toPrimaryKey(account, file.fileId!);
|
||||
|
||||
static List<Object> toStrippedPathIndexKey(
|
||||
Account account, String strippedPath) =>
|
||||
[
|
||||
account.url,
|
||||
account.username.toCaseInsensitiveString(),
|
||||
strippedPath == "." ? "" : strippedPath
|
||||
];
|
||||
|
||||
static List<Object> toStrippedPathIndexKeyForFile(
|
||||
Account account, File file) =>
|
||||
toStrippedPathIndexKey(account, file.strippedPathWithEmpty);
|
||||
|
||||
/// Return the lower bound key used to query files under [dir] and its sub
|
||||
/// dirs
|
||||
static List<Object> toStrippedPathIndexLowerKeyForDir(
|
||||
Account account, File dir) =>
|
||||
[
|
||||
account.url,
|
||||
account.username.toCaseInsensitiveString(),
|
||||
dir.strippedPath.run((p) => p == "." ? "" : "$p/")
|
||||
];
|
||||
|
||||
/// Return the upper bound key used to query files under [dir] and its sub
|
||||
/// dirs
|
||||
static List<Object> toStrippedPathIndexUpperKeyForDir(
|
||||
Account account, File dir) {
|
||||
return toStrippedPathIndexLowerKeyForDir(account, dir).run((k) {
|
||||
k[2] = (k[2] as String) + "\uffff";
|
||||
return k;
|
||||
});
|
||||
}
|
||||
|
||||
JsonObj toJson() {
|
||||
return {
|
||||
"namespacedFileId": namespacedFileId,
|
||||
"file": file.toJson(),
|
||||
};
|
||||
}
|
||||
@override
|
||||
get props => [
|
||||
server,
|
||||
userId,
|
||||
strippedPath,
|
||||
file,
|
||||
];
|
||||
|
||||
factory AppDbFileDbEntry.fromJson(JsonObj json) {
|
||||
return AppDbFileDbEntry(
|
||||
json["namespacedFileId"],
|
||||
File.fromJson(json["file"].cast<String, dynamic>()),
|
||||
);
|
||||
}
|
||||
|
||||
/// File ID namespaced by the server URL
|
||||
final String namespacedFileId;
|
||||
/// Server URL where this file belongs to
|
||||
final String server;
|
||||
final CiString userId;
|
||||
final String strippedPath;
|
||||
final File file;
|
||||
|
||||
static String toPrimaryKey(Account account, File file) =>
|
||||
"${account.url}/${file.path}";
|
||||
|
||||
static String toNamespacedFileId(Account account, int fileId) =>
|
||||
"${account.url}/$fileId";
|
||||
}
|
||||
|
||||
class AppDbDirEntry with EquatableMixin {
|
||||
AppDbDirEntry._(
|
||||
this.server, this.userId, this.strippedPath, this.dir, this.children);
|
||||
|
||||
factory AppDbDirEntry.fromFiles(
|
||||
Account account, File dir, List<File> children) =>
|
||||
AppDbDirEntry._(
|
||||
account.url,
|
||||
account.username,
|
||||
dir.strippedPathWithEmpty,
|
||||
dir,
|
||||
children.map((f) => f.fileId!).toList(),
|
||||
);
|
||||
|
||||
factory AppDbDirEntry.fromJson(JsonObj json) => AppDbDirEntry._(
|
||||
json["server"],
|
||||
(json["userId"] as String).toCi(),
|
||||
json["strippedPath"],
|
||||
File.fromJson((json["dir"] as Map).cast<String, dynamic>()),
|
||||
json["children"].cast<int>(),
|
||||
);
|
||||
|
||||
JsonObj toJson() => {
|
||||
"server": server,
|
||||
"userId": userId.toCaseInsensitiveString(),
|
||||
"strippedPath": strippedPath,
|
||||
"dir": dir.toJson(),
|
||||
"children": children,
|
||||
};
|
||||
|
||||
static String toPrimaryKeyForDir(Account account, File dir) =>
|
||||
"${account.url}/${account.username.toCaseInsensitiveString()}/${dir.strippedPathWithEmpty}";
|
||||
|
||||
/// Return the lower bound key used to query dirs under [root] and its sub
|
||||
/// dirs
|
||||
static String toPrimaryLowerKeyForSubDirs(Account account, File root) {
|
||||
final strippedPath = root.strippedPath.run((p) => p == "." ? "" : "$p/");
|
||||
return "${account.url}/${account.username.toCaseInsensitiveString()}/$strippedPath";
|
||||
}
|
||||
|
||||
/// Return the upper bound key used to query dirs under [root] and its sub
|
||||
/// dirs
|
||||
static String toPrimaryUpperKeyForSubDirs(Account account, File root) =>
|
||||
toPrimaryLowerKeyForSubDirs(account, root) + "\uffff";
|
||||
|
||||
@override
|
||||
get props => [
|
||||
server,
|
||||
userId,
|
||||
strippedPath,
|
||||
dir,
|
||||
children,
|
||||
];
|
||||
|
||||
/// Server URL where this file belongs to
|
||||
final String server;
|
||||
final CiString userId;
|
||||
final String strippedPath;
|
||||
final File dir;
|
||||
final List<int> children;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import 'package:kiwi/kiwi.dart';
|
|||
import 'package:logging/logging.dart';
|
||||
import 'package:nc_photos/account.dart';
|
||||
import 'package:nc_photos/app_db.dart';
|
||||
import 'package:nc_photos/debug_util.dart';
|
||||
import 'package:nc_photos/di_container.dart';
|
||||
import 'package:nc_photos/entity/file.dart';
|
||||
import 'package:nc_photos/entity/file/data_source.dart';
|
||||
import 'package:nc_photos/entity/file_util.dart' as file_util;
|
||||
|
|
@ -14,6 +16,7 @@ import 'package:nc_photos/pref.dart';
|
|||
import 'package:nc_photos/throttler.dart';
|
||||
import 'package:nc_photos/use_case/ls.dart';
|
||||
import 'package:nc_photos/use_case/scan_dir.dart';
|
||||
import 'package:nc_photos/use_case/scan_dir_offline.dart';
|
||||
|
||||
abstract class ScanAccountDirBlocEvent {
|
||||
const ScanAccountDirBlocEvent();
|
||||
|
|
@ -170,12 +173,12 @@ class ScanAccountDirBloc
|
|||
|
||||
if (!hasContent) {
|
||||
// show something instantly on first load
|
||||
ScanAccountDirBlocState cacheState = const ScanAccountDirBlocInit();
|
||||
await for (final s in _queryOffline(ev, () => cacheState)) {
|
||||
cacheState = s;
|
||||
}
|
||||
yield ScanAccountDirBlocLoading(cacheState.files);
|
||||
hasContent = cacheState.files.isNotEmpty;
|
||||
final stopwatch = Stopwatch()..start();
|
||||
final cacheFiles = await _queryOffline(ev);
|
||||
_log.info(
|
||||
"[_onEventQuery] Elapsed time (_queryOffline): ${stopwatch.elapsedMilliseconds}ms");
|
||||
yield ScanAccountDirBlocLoading(cacheFiles);
|
||||
hasContent = cacheFiles.isNotEmpty;
|
||||
}
|
||||
|
||||
ScanAccountDirBlocState newState = const ScanAccountDirBlocInit();
|
||||
|
|
@ -185,9 +188,12 @@ class ScanAccountDirBloc
|
|||
yield s;
|
||||
}
|
||||
} else {
|
||||
final stopwatch = Stopwatch()..start();
|
||||
await for (final s in _queryOnline(ev, () => newState)) {
|
||||
newState = s;
|
||||
}
|
||||
_log.info(
|
||||
"[_onEventQuery] Elapsed time (_queryOnline): ${stopwatch.elapsedMilliseconds}ms");
|
||||
if (newState is ScanAccountDirBlocSuccess) {
|
||||
yield newState;
|
||||
} else if (newState is ScanAccountDirBlocFailure) {
|
||||
|
|
@ -298,9 +304,22 @@ class ScanAccountDirBloc
|
|||
}
|
||||
}
|
||||
|
||||
Stream<ScanAccountDirBlocState> _queryOffline(ScanAccountDirBlocQueryBase ev,
|
||||
ScanAccountDirBlocState Function() getState) =>
|
||||
_queryWithFileDataSource(ev, getState, FileAppDbDataSource(AppDb()));
|
||||
Future<List<File>> _queryOffline(ScanAccountDirBlocQueryBase ev) async {
|
||||
final c = KiwiContainer().resolve<DiContainer>();
|
||||
final files = <File>[];
|
||||
for (final r in account.roots) {
|
||||
try {
|
||||
final dir = File(path: file_util.unstripPath(account, r));
|
||||
files.addAll(await ScanDirOffline(c)(account, dir));
|
||||
} catch (e, stackTrace) {
|
||||
_log.shout(
|
||||
"[_queryOffline] Failed while ScanDirOffline: ${logFilename(r)}",
|
||||
e,
|
||||
stackTrace);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
Stream<ScanAccountDirBlocState> _queryOnline(ScanAccountDirBlocQueryBase ev,
|
||||
ScanAccountDirBlocState Function() getState) {
|
||||
|
|
|
|||
|
|
@ -468,7 +468,10 @@ extension FileExtension on File {
|
|||
|
||||
/// Return the path of this file with the DAV part stripped
|
||||
///
|
||||
/// WebDAV file path: remote.php/dav/files/{username}/{strippedPath}
|
||||
/// WebDAV file path: remote.php/dav/files/{username}/{strippedPath}. If this
|
||||
/// file points to the user's root dir, return "."
|
||||
///
|
||||
/// See: [strippedPathWithEmpty]
|
||||
String get strippedPath {
|
||||
if (path.contains("remote.php/dav/files")) {
|
||||
final position = path.indexOf("/", "remote.php/dav/files/".length) + 1;
|
||||
|
|
@ -483,6 +486,17 @@ extension FileExtension on File {
|
|||
}
|
||||
}
|
||||
|
||||
/// Return the path of this file with the DAV part stripped
|
||||
///
|
||||
/// WebDAV file path: remote.php/dav/files/{username}/{strippedPath}. If this
|
||||
/// file points to the user's root dir, return an empty string
|
||||
///
|
||||
/// See: [strippedPath]
|
||||
String get strippedPathWithEmpty {
|
||||
final path = strippedPath;
|
||||
return path == "." ? "" : path;
|
||||
}
|
||||
|
||||
String get filename => path_util.basename(path);
|
||||
|
||||
/// Compare the server identity of two Files
|
||||
|
|
|
|||
|
|
@ -1,23 +1,23 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:idb_shim/idb_client.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:nc_photos/account.dart';
|
||||
import 'package:nc_photos/api/api.dart';
|
||||
import 'package:nc_photos/app_db.dart';
|
||||
import 'package:nc_photos/debug_util.dart';
|
||||
import 'package:nc_photos/entity/file.dart';
|
||||
import 'package:nc_photos/entity/webdav_response_parser.dart';
|
||||
import 'package:nc_photos/exception.dart';
|
||||
import 'package:nc_photos/int_util.dart' as int_util;
|
||||
import 'package:nc_photos/iterable_extension.dart';
|
||||
import 'package:nc_photos/object_extension.dart';
|
||||
import 'package:nc_photos/or_null.dart';
|
||||
import 'package:nc_photos/remote_storage_util.dart' as remote_storage_util;
|
||||
import 'package:nc_photos/string_extension.dart';
|
||||
import 'package:nc_photos/touch_token_manager.dart';
|
||||
import 'package:nc_photos/use_case/compat/v32.dart';
|
||||
import 'package:path/path.dart' as path;
|
||||
import 'package:quiver/iterables.dart';
|
||||
import 'package:uuid/uuid.dart';
|
||||
import 'package:xml/xml.dart';
|
||||
|
||||
|
|
@ -246,12 +246,32 @@ class FileAppDbDataSource implements FileDataSource {
|
|||
const FileAppDbDataSource(this.appDb);
|
||||
|
||||
@override
|
||||
list(Account account, File f) {
|
||||
_log.info("[list] ${f.path}");
|
||||
list(Account account, File dir) {
|
||||
_log.info("[list] ${dir.path}");
|
||||
return appDb.use((db) async {
|
||||
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.fileStoreName);
|
||||
return await _doList(store, account, f);
|
||||
final transaction = db.transaction(
|
||||
[AppDb.dirStoreName, AppDb.file2StoreName], idbModeReadOnly);
|
||||
final fileStore = transaction.objectStore(AppDb.file2StoreName);
|
||||
final dirStore = transaction.objectStore(AppDb.dirStoreName);
|
||||
final dirItem = await dirStore
|
||||
.getObject(AppDbDirEntry.toPrimaryKeyForDir(account, dir)) as Map?;
|
||||
if (dirItem == null) {
|
||||
throw CacheNotFoundException("No entry: ${dir.path}");
|
||||
}
|
||||
final dirEntry = AppDbDirEntry.fromJson(dirItem.cast<String, dynamic>());
|
||||
final entries = await Future.wait(dirEntry.children.map((c) async {
|
||||
final fileItem = await fileStore
|
||||
.getObject(AppDbFile2Entry.toPrimaryKey(account, c)) as Map?;
|
||||
if (fileItem == null) {
|
||||
_log.warning(
|
||||
"[list] Missing file ($c) in db for dir: ${logFilename(dir.path)}");
|
||||
throw CacheNotFoundException("No entry for dir child: $c");
|
||||
}
|
||||
return AppDbFile2Entry.fromJson(fileItem.cast<String, dynamic>());
|
||||
}));
|
||||
// we need to add dir to match the remote query
|
||||
return [dirEntry.dir] +
|
||||
entries.map((e) => e.file).where((f) => _validateFile(f)).toList();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -261,22 +281,29 @@ class FileAppDbDataSource implements FileDataSource {
|
|||
throw UnimplementedError();
|
||||
}
|
||||
|
||||
/// Remove a file/dir from database
|
||||
///
|
||||
/// If [f] is a dir, the dir and its sub-dirs will be removed from dirStore.
|
||||
/// The files inside any of these dirs will be removed from file2Store.
|
||||
///
|
||||
/// If [f] is a file, the file will be removed from file2Store, but no changes
|
||||
/// to dirStore.
|
||||
@override
|
||||
remove(Account account, File f) {
|
||||
remove(Account account, File f) async {
|
||||
_log.info("[remove] ${f.path}");
|
||||
return appDb.use((db) async {
|
||||
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
|
||||
final store = transaction.objectStore(AppDb.fileStoreName);
|
||||
final index = store.index(AppDbFileEntry.indexName);
|
||||
final path = AppDbFileEntry.toPath(account, f);
|
||||
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
|
||||
final keys = await index
|
||||
.openKeyCursor(range: range, autoAdvance: true)
|
||||
.map((cursor) => cursor.primaryKey)
|
||||
.toList();
|
||||
for (final k in keys) {
|
||||
_log.fine("[remove] Removing DB entry: $k");
|
||||
await store.delete(k);
|
||||
await appDb.use((db) async {
|
||||
if (f.isCollection == true) {
|
||||
final transaction = db.transaction(
|
||||
[AppDb.dirStoreName, AppDb.file2StoreName], idbModeReadWrite);
|
||||
final dirStore = transaction.objectStore(AppDb.dirStoreName);
|
||||
final fileStore = transaction.objectStore(AppDb.file2StoreName);
|
||||
await _removeDirFromAppDb(account, f,
|
||||
dirStore: dirStore, fileStore: fileStore);
|
||||
} else {
|
||||
final transaction =
|
||||
db.transaction(AppDb.file2StoreName, idbModeReadWrite);
|
||||
final fileStore = transaction.objectStore(AppDb.file2StoreName);
|
||||
await _removeFileFromAppDb(account, f, fileStore: fileStore);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -303,36 +330,18 @@ class FileAppDbDataSource implements FileDataSource {
|
|||
}) {
|
||||
_log.info("[updateProperty] ${f.path}");
|
||||
return appDb.use((db) async {
|
||||
final transaction = db.transaction(
|
||||
[AppDb.fileStoreName, AppDb.fileDbStoreName], idbModeReadWrite);
|
||||
final transaction =
|
||||
db.transaction(AppDb.file2StoreName, idbModeReadWrite);
|
||||
|
||||
// update file store
|
||||
final fileStore = transaction.objectStore(AppDb.fileStoreName);
|
||||
final parentDir = File(path: path.dirname(f.path));
|
||||
final parentList = await _doList(fileStore, account, parentDir);
|
||||
final jsonList = parentList.map((e) {
|
||||
if (e.path == f.path) {
|
||||
return e.copyWith(
|
||||
metadata: metadata,
|
||||
isArchived: isArchived,
|
||||
overrideDateTime: overrideDateTime,
|
||||
);
|
||||
} else {
|
||||
return e;
|
||||
}
|
||||
});
|
||||
await _cacheListResults(fileStore, account, parentDir, jsonList);
|
||||
|
||||
// update file db store
|
||||
final fileDbStore = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
final newFile = f.copyWith(
|
||||
metadata: metadata,
|
||||
isArchived: isArchived,
|
||||
overrideDateTime: overrideDateTime,
|
||||
);
|
||||
await fileDbStore.put(
|
||||
AppDbFileDbEntry.fromFile(account, newFile).toJson(),
|
||||
AppDbFileDbEntry.toPrimaryKey(account, newFile));
|
||||
final fileStore = transaction.objectStore(AppDb.file2StoreName);
|
||||
await fileStore.put(AppDbFile2Entry.fromFile(account, newFile).toJson(),
|
||||
AppDbFile2Entry.toPrimaryKeyForFile(account, newFile));
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -361,27 +370,6 @@ class FileAppDbDataSource implements FileDataSource {
|
|||
// do nothing
|
||||
}
|
||||
|
||||
Future<List<File>> _doList(ObjectStore store, Account account, File f) async {
|
||||
final index = store.index(AppDbFileEntry.indexName);
|
||||
final path = AppDbFileEntry.toPath(account, f);
|
||||
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
|
||||
final List results = await index.getAll(range);
|
||||
if (results.isNotEmpty == true) {
|
||||
final entries = results
|
||||
.map((e) => AppDbFileEntry.fromJson(e.cast<String, dynamic>()));
|
||||
return entries
|
||||
.map((e) {
|
||||
_log.info("[_doList] ${e.path}[${e.index}]");
|
||||
return e.data;
|
||||
})
|
||||
.reduce((value, element) => value + element)
|
||||
.where((element) => _validateFile(element))
|
||||
.toList();
|
||||
} else {
|
||||
throw CacheNotFoundException("No entry: $path");
|
||||
}
|
||||
}
|
||||
|
||||
final AppDb appDb;
|
||||
|
||||
static final _log = Logger("entity.file.data_source.FileAppDbDataSource");
|
||||
|
|
@ -423,21 +411,7 @@ class FileCachedDataSource implements FileDataSource {
|
|||
}
|
||||
|
||||
if (cache != null) {
|
||||
_syncCacheWithRemote(account, remote, cache);
|
||||
} else {
|
||||
appDb.use((db) async {
|
||||
final transaction =
|
||||
db.transaction(AppDb.fileDbStoreName, idbModeReadWrite);
|
||||
final fileDbStore = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
for (final f in remote) {
|
||||
try {
|
||||
await _upsertFileDbStoreCache(account, f, fileDbStore);
|
||||
} catch (e, stacktrace) {
|
||||
_log.shout(
|
||||
"[list] Failed while _upsertFileDbStoreCache", e, stacktrace);
|
||||
}
|
||||
}
|
||||
});
|
||||
await _cleanUpCacheWithRemote(account, remote, cache);
|
||||
}
|
||||
return remote;
|
||||
} on ApiException catch (e) {
|
||||
|
|
@ -536,109 +510,49 @@ class FileCachedDataSource implements FileDataSource {
|
|||
|
||||
Future<void> _cacheResult(Account account, File f, List<File> result) {
|
||||
return appDb.use((db) async {
|
||||
final transaction = db.transaction(AppDb.fileStoreName, idbModeReadWrite);
|
||||
final store = transaction.objectStore(AppDb.fileStoreName);
|
||||
await _cacheListResults(store, account, f, result);
|
||||
final transaction = db.transaction(
|
||||
[AppDb.dirStoreName, AppDb.file2StoreName], idbModeReadWrite);
|
||||
final dirStore = transaction.objectStore(AppDb.dirStoreName);
|
||||
final fileStore = transaction.objectStore(AppDb.file2StoreName);
|
||||
await _cacheListResults(account, f, result,
|
||||
fileStore: fileStore, dirStore: dirStore);
|
||||
});
|
||||
}
|
||||
|
||||
/// Sync the remote result and local cache
|
||||
void _syncCacheWithRemote(
|
||||
/// Remove extra entries from local cache based on remote results
|
||||
Future<void> _cleanUpCacheWithRemote(
|
||||
Account account, List<File> remote, List<File> cache) async {
|
||||
final removed =
|
||||
cache.where((c) => !remote.any((r) => r.path == c.path)).toList();
|
||||
if (removed.isEmpty) {
|
||||
return;
|
||||
}
|
||||
_log.info(
|
||||
"[_syncCacheWithRemote] Removed: ${removed.map((f) => f.path).toReadableString()}");
|
||||
"[_cleanUpCacheWithRemote] Removed: ${removed.map((f) => f.path).toReadableString()}");
|
||||
|
||||
appDb.use((db) async {
|
||||
await appDb.use((db) async {
|
||||
final transaction = db.transaction(
|
||||
[AppDb.fileStoreName, AppDb.fileDbStoreName], idbModeReadWrite);
|
||||
final fileStore = transaction.objectStore(AppDb.fileStoreName);
|
||||
final fileStoreIndex = fileStore.index(AppDbFileEntry.indexName);
|
||||
final fileDbStore = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
[AppDb.dirStoreName, AppDb.file2StoreName], idbModeReadWrite);
|
||||
final dirStore = transaction.objectStore(AppDb.dirStoreName);
|
||||
final fileStore = transaction.objectStore(AppDb.file2StoreName);
|
||||
for (final f in removed) {
|
||||
try {
|
||||
await _removeFileDbStoreCache(account, f, fileDbStore);
|
||||
} catch (e, stacktrace) {
|
||||
if (f.isCollection == true) {
|
||||
await _removeDirFromAppDb(account, f,
|
||||
dirStore: dirStore, fileStore: fileStore);
|
||||
} else {
|
||||
await _removeFileFromAppDb(account, f, fileStore: fileStore);
|
||||
}
|
||||
} catch (e, stackTrace) {
|
||||
_log.shout(
|
||||
"[_syncCacheWithRemote] Failed while _removeFileDbStoreCache",
|
||||
"[_cleanUpCacheWithRemote] Failed while removing file: ${logFilename(f.path)}",
|
||||
e,
|
||||
stacktrace);
|
||||
}
|
||||
try {
|
||||
await _removeFileStoreCache(account, f, fileStore, fileStoreIndex);
|
||||
} catch (e, stacktrace) {
|
||||
_log.shout(
|
||||
"[_syncCacheWithRemote] Failed while _removeFileStoreCache",
|
||||
e,
|
||||
stacktrace);
|
||||
}
|
||||
}
|
||||
for (final f in remote) {
|
||||
try {
|
||||
await _upsertFileDbStoreCache(account, f, fileDbStore);
|
||||
} catch (e, stacktrace) {
|
||||
_log.shout(
|
||||
"[_syncCacheWithRemote] Failed while _upsertFileDbStoreCache",
|
||||
e,
|
||||
stacktrace);
|
||||
stackTrace);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Future<void> _removeFileDbStoreCache(
|
||||
Account account, File file, ObjectStore objStore) async {
|
||||
if (file.isCollection == true) {
|
||||
final fullPath = AppDbFileDbEntry.toPrimaryKey(account, file);
|
||||
final range = KeyRange.bound("$fullPath/", "$fullPath/\uffff");
|
||||
await for (final k
|
||||
in objStore.openKeyCursor(range: range, autoAdvance: true)) {
|
||||
_log.fine(
|
||||
"[_removeFileDbStoreCache] Removing DB entry: ${k.primaryKey}");
|
||||
objStore.delete(k.primaryKey);
|
||||
}
|
||||
} else {
|
||||
await objStore.delete(AppDbFileDbEntry.toPrimaryKey(account, file));
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _upsertFileDbStoreCache(
|
||||
Account account, File file, ObjectStore objStore) async {
|
||||
if (file.isCollection == true) {
|
||||
return;
|
||||
}
|
||||
await objStore.put(AppDbFileDbEntry.fromFile(account, file).toJson(),
|
||||
AppDbFileDbEntry.toPrimaryKey(account, file));
|
||||
}
|
||||
|
||||
/// Remove dangling dir entries in the file object store
|
||||
Future<void> _removeFileStoreCache(
|
||||
Account account, File file, ObjectStore objStore, Index index) async {
|
||||
if (file.isCollection != true) {
|
||||
return;
|
||||
}
|
||||
|
||||
final path = AppDbFileEntry.toPath(account, file);
|
||||
// delete the dir itself
|
||||
final dirRange = KeyRange.bound([path, 0], [path, int_util.int32Max]);
|
||||
// delete with KeyRange is not supported in idb_shim/idb_sqflite
|
||||
// await store.delete(dirRange);
|
||||
await for (final k
|
||||
in index.openKeyCursor(range: dirRange, autoAdvance: true)) {
|
||||
_log.fine("[_removeFileStoreCache] Removing DB entry: ${k.primaryKey}");
|
||||
objStore.delete(k.primaryKey);
|
||||
}
|
||||
// then its children
|
||||
final childrenRange =
|
||||
KeyRange.bound(["$path/", 0], ["$path/\uffff", int_util.int32Max]);
|
||||
await for (final k
|
||||
in index.openKeyCursor(range: childrenRange, autoAdvance: true)) {
|
||||
_log.fine("[_removeFileStoreCache] Removing DB entry: ${k.primaryKey}");
|
||||
objStore.delete(k.primaryKey);
|
||||
}
|
||||
}
|
||||
|
||||
final AppDb appDb;
|
||||
final bool shouldCheckCache;
|
||||
|
||||
|
|
@ -656,40 +570,35 @@ class _CacheManager {
|
|||
this.shouldCheckCache = false,
|
||||
});
|
||||
|
||||
/// Return the cached results of listing a directory [f]
|
||||
/// Return the cached results of listing a directory [dir]
|
||||
///
|
||||
/// Should check [isGood] before using the cache returning by this method
|
||||
Future<List<File>?> list(Account account, File f) async {
|
||||
final trimmedRootPath = f.path.trimAny("/");
|
||||
Future<List<File>?> list(Account account, File dir) async {
|
||||
List<File>? cache;
|
||||
try {
|
||||
cache = await appDbSrc.list(account, f);
|
||||
cache = await appDbSrc.list(account, dir);
|
||||
// compare the cached root
|
||||
final cacheEtag = cache
|
||||
.firstWhere((element) => element.path.trimAny("/") == trimmedRootPath)
|
||||
.etag;
|
||||
if (cacheEtag != null) {
|
||||
// compare the etag to see if the content has been updated
|
||||
var remoteEtag = f.etag;
|
||||
// if no etag supplied, we need to query it form remote
|
||||
remoteEtag ??= (await remoteSrc.list(account, f, depth: 0)).first.etag;
|
||||
if (cacheEtag == remoteEtag) {
|
||||
_log.fine(
|
||||
"[_listCache] etag matched for ${AppDbFileEntry.toPath(account, f)}");
|
||||
if (shouldCheckCache) {
|
||||
await _checkTouchToken(account, f, cache);
|
||||
} else {
|
||||
_isGood = true;
|
||||
}
|
||||
final cacheEtag =
|
||||
cache.firstWhere((f) => f.compareServerIdentity(dir)).etag!;
|
||||
// compare the etag to see if the content has been updated
|
||||
var remoteEtag = dir.etag;
|
||||
// if no etag supplied, we need to query it form remote
|
||||
remoteEtag ??= (await remoteSrc.list(account, dir, depth: 0)).first.etag;
|
||||
if (cacheEtag == remoteEtag) {
|
||||
_log.fine(
|
||||
"[list] etag matched for ${AppDbDirEntry.toPrimaryKeyForDir(account, dir)}");
|
||||
if (shouldCheckCache) {
|
||||
await _checkTouchToken(account, dir, cache);
|
||||
} else {
|
||||
_isGood = true;
|
||||
}
|
||||
} else {
|
||||
_log.info(
|
||||
"[_list] Remote content updated for ${AppDbFileEntry.toPath(account, f)}");
|
||||
_log.info("[list] Remote content updated for ${dir.path}");
|
||||
}
|
||||
} on CacheNotFoundException catch (_) {
|
||||
// normal when there's no cache
|
||||
} catch (e, stacktrace) {
|
||||
_log.shout("[_list] Cache failure", e, stacktrace);
|
||||
} catch (e, stackTrace) {
|
||||
_log.shout("[list] Cache failure", e, stackTrace);
|
||||
}
|
||||
return cache;
|
||||
}
|
||||
|
|
@ -744,34 +653,76 @@ class _CacheManager {
|
|||
}
|
||||
|
||||
Future<void> _cacheListResults(
|
||||
ObjectStore store, Account account, File f, Iterable<File> results) async {
|
||||
final index = store.index(AppDbFileEntry.indexName);
|
||||
final path = AppDbFileEntry.toPath(account, f);
|
||||
final range = KeyRange.bound([path, 0], [path, int_util.int32Max]);
|
||||
// count number of entries for this dir
|
||||
final count = await index.count(range);
|
||||
int newCount = 0;
|
||||
for (final pair
|
||||
in partition(results, AppDbFileEntry.maxDataSize).withIndex()) {
|
||||
_log.info(
|
||||
"[_cacheListResults] Caching $path[${pair.item1}], length: ${pair.item2.length}");
|
||||
await store.put(
|
||||
AppDbFileEntry(path, pair.item1, pair.item2).toJson(),
|
||||
AppDbFileEntry.toPrimaryKey(account, f, pair.item1),
|
||||
);
|
||||
++newCount;
|
||||
Account account,
|
||||
File dir,
|
||||
List<File> results, {
|
||||
required ObjectStore fileStore,
|
||||
required ObjectStore dirStore,
|
||||
}) async {
|
||||
// add files to db
|
||||
await Future.wait(results.map((f) => fileStore.put(
|
||||
AppDbFile2Entry.fromFile(account, f).toJson(),
|
||||
AppDbFile2Entry.toPrimaryKeyForFile(account, f))));
|
||||
|
||||
// results from remote also contain the dir itself
|
||||
final resultGroup = results.groupListsBy((f) => f.compareServerIdentity(dir));
|
||||
final remoteDir = resultGroup[true]!.first;
|
||||
final remoteChildren = resultGroup[false] ?? [];
|
||||
// add dir to db
|
||||
await dirStore.put(
|
||||
AppDbDirEntry.fromFiles(account, remoteDir, remoteChildren).toJson(),
|
||||
AppDbDirEntry.toPrimaryKeyForDir(account, remoteDir));
|
||||
}
|
||||
|
||||
Future<void> _removeFileFromAppDb(
|
||||
Account account,
|
||||
File file, {
|
||||
required ObjectStore fileStore,
|
||||
}) async {
|
||||
assert(file.isCollection != true);
|
||||
await fileStore.delete(AppDbFile2Entry.toPrimaryKeyForFile(account, file));
|
||||
}
|
||||
|
||||
/// Remove a dir and all files inside from the database
|
||||
Future<void> _removeDirFromAppDb(
|
||||
Account account,
|
||||
File dir, {
|
||||
required ObjectStore dirStore,
|
||||
required ObjectStore fileStore,
|
||||
}) async {
|
||||
assert(dir.isCollection == true);
|
||||
// delete the dir itself
|
||||
await AppDbDirEntry.toPrimaryKeyForDir(account, dir).runFuture((key) async {
|
||||
_log.fine("[_removeDirFromAppDb] Removing dirStore entry: $key");
|
||||
await dirStore.delete(key);
|
||||
});
|
||||
// then its children
|
||||
final childrenRange = KeyRange.bound(
|
||||
AppDbDirEntry.toPrimaryLowerKeyForSubDirs(account, dir),
|
||||
AppDbDirEntry.toPrimaryUpperKeyForSubDirs(account, dir),
|
||||
);
|
||||
for (final key in await dirStore.getAllKeys(childrenRange)) {
|
||||
_log.fine("[_removeDirFromAppDb] Removing dirStore entry: $key");
|
||||
await dirStore.delete(key);
|
||||
}
|
||||
if (count > newCount) {
|
||||
// index is 0-based
|
||||
final rmRange = KeyRange.bound([path, newCount], [path, int_util.int32Max]);
|
||||
final rmKeys = await index
|
||||
.openKeyCursor(range: rmRange, autoAdvance: true)
|
||||
.map((cursor) => cursor.primaryKey)
|
||||
.toList();
|
||||
for (final k in rmKeys) {
|
||||
_log.fine("[_cacheListResults] Removing DB entry: $k");
|
||||
await store.delete(k);
|
||||
}
|
||||
|
||||
// delete files from fileStore
|
||||
// first the dir
|
||||
await AppDbFile2Entry.toPrimaryKeyForFile(account, dir)
|
||||
.runFuture((key) async {
|
||||
_log.fine("[_removeDirFromAppDb] Removing fileStore entry: $key");
|
||||
await fileStore.delete(key);
|
||||
});
|
||||
// then files under this dir and sub-dirs
|
||||
final range = KeyRange.bound(
|
||||
AppDbFile2Entry.toStrippedPathIndexLowerKeyForDir(account, dir),
|
||||
AppDbFile2Entry.toStrippedPathIndexUpperKeyForDir(account, dir),
|
||||
);
|
||||
final strippedPathIndex =
|
||||
fileStore.index(AppDbFile2Entry.strippedPathIndexName);
|
||||
for (final key in await strippedPathIndex.getAllKeys(range)) {
|
||||
_log.fine("[_removeDirFromAppDb] Removing fileStore entry: $key");
|
||||
await fileStore.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,15 @@ String renameConflict(String filename, int conflictCount) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Return if this file is the no media marker
|
||||
///
|
||||
/// A no media marker marks the parent dir and its sub dirs as not containing
|
||||
/// media files of interest
|
||||
bool isNoMediaMarker(File file) {
|
||||
final filename = file.filename;
|
||||
return filename == ".nomedia" || filename == ".noimage";
|
||||
}
|
||||
|
||||
final _supportedFormatMimes = [
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
|
|
|
|||
|
|
@ -15,4 +15,9 @@ extension ObjectExtension<T> on T {
|
|||
|
||||
/// Run [fn] with this, and return the results of [fn]
|
||||
U run<U>(U Function(T obj) fn) => fn(this);
|
||||
|
||||
/// Run [fn] with this, and return the results of [fn]
|
||||
Future<U> runFuture<U>(FutureOr<U> Function(T obj) fn) async {
|
||||
return await fn(this);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import 'package:nc_photos/account.dart';
|
|||
import 'package:nc_photos/app_db.dart';
|
||||
import 'package:nc_photos/di_container.dart';
|
||||
import 'package:nc_photos/entity/file.dart';
|
||||
import 'package:nc_photos/entity/file_util.dart' as file_util;
|
||||
|
||||
class FindFile {
|
||||
FindFile(this._c) : assert(require(_c));
|
||||
|
|
@ -13,19 +12,14 @@ class FindFile {
|
|||
/// Find the [File] in the DB by [fileId]
|
||||
Future<File> call(Account account, int fileId) async {
|
||||
return await _c.appDb.use((db) async {
|
||||
final transaction =
|
||||
db.transaction(AppDb.fileDbStoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
final index = store.index(AppDbFileDbEntry.indexName);
|
||||
final List dbItems = await index
|
||||
.getAll(AppDbFileDbEntry.toNamespacedFileId(account, fileId));
|
||||
|
||||
// find the one owned by us
|
||||
final dbItem = dbItems.firstWhere((element) {
|
||||
final e = AppDbFileDbEntry.fromJson(element.cast<String, dynamic>());
|
||||
return file_util.getUserDirName(e.file) == account.username;
|
||||
});
|
||||
final dbEntry = AppDbFileDbEntry.fromJson(dbItem.cast<String, dynamic>());
|
||||
final transaction = db.transaction(AppDb.file2StoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.file2StoreName);
|
||||
final dbItem = await store
|
||||
.getObject(AppDbFile2Entry.toPrimaryKey(account, fileId)) as Map?;
|
||||
if (dbItem == null) {
|
||||
throw StateError("File ID not found: $fileId");
|
||||
}
|
||||
final dbEntry = AppDbFile2Entry.fromJson(dbItem.cast<String, dynamic>());
|
||||
return dbEntry.file;
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ import 'package:nc_photos/account.dart';
|
|||
import 'package:nc_photos/app_db.dart';
|
||||
import 'package:nc_photos/entity/face.dart';
|
||||
import 'package:nc_photos/entity/file.dart';
|
||||
import 'package:nc_photos/entity/file_util.dart' as file_util;
|
||||
import 'package:nc_photos/exception.dart';
|
||||
|
||||
class PopulatePerson {
|
||||
|
|
@ -13,14 +12,12 @@ class PopulatePerson {
|
|||
/// Return a list of files of the faces
|
||||
Future<List<File>> call(Account account, List<Face> faces) async {
|
||||
return await appDb.use((db) async {
|
||||
final transaction =
|
||||
db.transaction(AppDb.fileDbStoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
final index = store.index(AppDbFileDbEntry.indexName);
|
||||
final transaction = db.transaction(AppDb.file2StoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.file2StoreName);
|
||||
final products = <File>[];
|
||||
for (final f in faces) {
|
||||
try {
|
||||
products.add(await _populateOne(account, f, store, index));
|
||||
products.add(await _populateOne(account, f, fileStore: store));
|
||||
} catch (e, stackTrace) {
|
||||
_log.severe("[call] Failed populating file of face: ${f.fileId}", e,
|
||||
stackTrace);
|
||||
|
|
@ -31,25 +28,18 @@ class PopulatePerson {
|
|||
}
|
||||
|
||||
Future<File> _populateOne(
|
||||
Account account, Face face, ObjectStore store, Index index) async {
|
||||
final List dbItems = await index
|
||||
.getAll(AppDbFileDbEntry.toNamespacedFileId(account, face.fileId));
|
||||
// find the one owned by us
|
||||
Map? dbItem;
|
||||
try {
|
||||
dbItem = dbItems.firstWhere((element) {
|
||||
final e = AppDbFileDbEntry.fromJson(element.cast<String, dynamic>());
|
||||
return file_util.getUserDirName(e.file) == account.username;
|
||||
});
|
||||
} on StateError catch (_) {
|
||||
// not found
|
||||
}
|
||||
Account account,
|
||||
Face face, {
|
||||
required ObjectStore fileStore,
|
||||
}) async {
|
||||
final dbItem = await fileStore
|
||||
.getObject(AppDbFile2Entry.toPrimaryKey(account, face.fileId)) as Map?;
|
||||
if (dbItem == null) {
|
||||
_log.warning(
|
||||
"[_populateOne] File doesn't exist in DB, removed?: '${face.fileId}'");
|
||||
throw CacheNotFoundException();
|
||||
}
|
||||
final dbEntry = AppDbFileDbEntry.fromJson(dbItem.cast<String, dynamic>());
|
||||
final dbEntry = AppDbFile2Entry.fromJson(dbItem.cast<String, dynamic>());
|
||||
return dbEntry.file;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import 'package:nc_photos/debug_util.dart';
|
|||
import 'package:nc_photos/entity/album.dart';
|
||||
import 'package:nc_photos/entity/album/item.dart';
|
||||
import 'package:nc_photos/entity/album/provider.dart';
|
||||
import 'package:nc_photos/entity/file_util.dart' as file_util;
|
||||
|
||||
/// Resync files inside an album with the file db
|
||||
class ResyncAlbum {
|
||||
|
|
@ -19,15 +18,15 @@ class ResyncAlbum {
|
|||
"Resync only make sense for static albums: ${album.name}");
|
||||
}
|
||||
return await appDb.use((db) async {
|
||||
final transaction =
|
||||
db.transaction(AppDb.fileDbStoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
final index = store.index(AppDbFileDbEntry.indexName);
|
||||
final transaction = db.transaction(AppDb.file2StoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.file2StoreName);
|
||||
final index = store.index(AppDbFile2Entry.strippedPathIndexName);
|
||||
final newItems = <AlbumItem>[];
|
||||
for (final item in AlbumStaticProvider.of(album).items) {
|
||||
if (item is AlbumFileItem) {
|
||||
try {
|
||||
newItems.add(await _syncOne(account, item, store, index));
|
||||
newItems.add(await _syncOne(account, item,
|
||||
fileStore: store, fileStoreStrippedPathIndex: index));
|
||||
} catch (e, stacktrace) {
|
||||
_log.shout(
|
||||
"[call] Failed syncing file in album: ${logFilename(item.file.path)}",
|
||||
|
|
@ -43,31 +42,27 @@ class ResyncAlbum {
|
|||
});
|
||||
}
|
||||
|
||||
Future<AlbumFileItem> _syncOne(Account account, AlbumFileItem item,
|
||||
ObjectStore objStore, Index index) async {
|
||||
Future<AlbumFileItem> _syncOne(
|
||||
Account account,
|
||||
AlbumFileItem item, {
|
||||
required ObjectStore fileStore,
|
||||
required Index fileStoreStrippedPathIndex,
|
||||
}) async {
|
||||
Map? dbItem;
|
||||
if (item.file.fileId != null) {
|
||||
final List dbItems = await index.getAll(
|
||||
AppDbFileDbEntry.toNamespacedFileId(account, item.file.fileId!));
|
||||
// find the one owned by us
|
||||
try {
|
||||
dbItem = dbItems.firstWhere((element) {
|
||||
final e = AppDbFileDbEntry.fromJson(element.cast<String, dynamic>());
|
||||
return file_util.getUserDirName(e.file) == account.username;
|
||||
});
|
||||
} on StateError catch (_) {
|
||||
// not found
|
||||
}
|
||||
dbItem = await fileStore.getObject(
|
||||
AppDbFile2Entry.toPrimaryKeyForFile(account, item.file)) as Map?;
|
||||
} else {
|
||||
dbItem = await objStore
|
||||
.getObject(AppDbFileDbEntry.toPrimaryKey(account, item.file)) as Map?;
|
||||
dbItem = await fileStoreStrippedPathIndex.get(
|
||||
AppDbFile2Entry.toStrippedPathIndexKeyForFile(account, item.file))
|
||||
as Map?;
|
||||
}
|
||||
if (dbItem == null) {
|
||||
_log.warning(
|
||||
"[_syncOne] File doesn't exist in DB, removed?: '${item.file.path}'");
|
||||
return item;
|
||||
}
|
||||
final dbEntry = AppDbFileDbEntry.fromJson(dbItem.cast<String, dynamic>());
|
||||
final dbEntry = AppDbFile2Entry.fromJson(dbItem.cast<String, dynamic>());
|
||||
return item.copyWith(
|
||||
file: dbEntry.file,
|
||||
);
|
||||
|
|
|
|||
66
lib/use_case/scan_dir_offline.dart
Normal file
66
lib/use_case/scan_dir_offline.dart
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import 'package:idb_shim/idb_client.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:nc_photos/account.dart';
|
||||
import 'package:nc_photos/app_db.dart';
|
||||
import 'package:nc_photos/di_container.dart';
|
||||
import 'package:nc_photos/entity/file.dart';
|
||||
import 'package:nc_photos/entity/file_util.dart' as file_util;
|
||||
import 'package:nc_photos/iterable_extension.dart';
|
||||
import 'package:path/path.dart' as path_lib;
|
||||
|
||||
class ScanDirOffline {
|
||||
ScanDirOffline(this._c) : assert(require(_c));
|
||||
|
||||
static bool require(DiContainer c) => DiContainer.has(c, DiType.appDb);
|
||||
|
||||
/// List all files under a dir recursively from the local DB
|
||||
///
|
||||
/// Dirs with a .nomedia/.noimage file will be ignored.
|
||||
///
|
||||
/// If [isSupportedFileOnly] == true, the returned files will be filtered by
|
||||
/// [file_util.isSupportedFormat]
|
||||
Future<List<File>> call(
|
||||
Account account,
|
||||
File root, {
|
||||
bool isSupportedFileOnly = true,
|
||||
}) async {
|
||||
final skipDirs = <File>[];
|
||||
final files = await _c.appDb.use((db) async {
|
||||
final transaction = db.transaction(AppDb.file2StoreName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(AppDb.file2StoreName);
|
||||
final index = store.index(AppDbFile2Entry.strippedPathIndexName);
|
||||
final range = KeyRange.bound(
|
||||
AppDbFile2Entry.toStrippedPathIndexLowerKeyForDir(account, root),
|
||||
AppDbFile2Entry.toStrippedPathIndexUpperKeyForDir(account, root),
|
||||
);
|
||||
final files = <File>[];
|
||||
await for (final f in index
|
||||
.openCursor(range: range, autoAdvance: true)
|
||||
.map((c) => c.value)
|
||||
.cast<Map>()
|
||||
.map((e) =>
|
||||
AppDbFile2Entry.fromJson(e.cast<String, dynamic>()).file)) {
|
||||
if (file_util.isNoMediaMarker(f)) {
|
||||
skipDirs.add(File(path: path_lib.dirname(f.path)));
|
||||
} else if (file_util.isSupportedFormat(f)) {
|
||||
files.add(f);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
});
|
||||
|
||||
_log.info(
|
||||
"[call] Skip dirs: ${skipDirs.map((d) => d.strippedPath).toReadableString()}");
|
||||
if (skipDirs.isEmpty) {
|
||||
return files;
|
||||
} else {
|
||||
return files
|
||||
.where((f) => !skipDirs.any((d) => file_util.isUnderDir(f, d)))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
final DiContainer _c;
|
||||
|
||||
static final _log = Logger("use_case.scan_dir_offline.ScanDirOffline");
|
||||
}
|
||||
|
|
@ -69,6 +69,34 @@ class MockAlbumMemoryRepo extends MockAlbumRepo {
|
|||
|
||||
/// Each MockAppDb instance contains a unique memory database
|
||||
class MockAppDb implements AppDb {
|
||||
static Future<MockAppDb> create({
|
||||
bool hasAlbumStore = true,
|
||||
bool hasFileDb2Store = true,
|
||||
bool hasDirStore = true,
|
||||
// compat
|
||||
bool hasFileStore = false,
|
||||
bool hasFileDbStore = false,
|
||||
}) async {
|
||||
final inst = MockAppDb();
|
||||
final db = await inst._dbFactory.open(
|
||||
"test.db",
|
||||
version: 1,
|
||||
onUpgradeNeeded: (event) async {
|
||||
final db = event.database;
|
||||
_createDb(
|
||||
db,
|
||||
hasAlbumStore: hasAlbumStore,
|
||||
hasFileDb2Store: hasFileDb2Store,
|
||||
hasDirStore: hasDirStore,
|
||||
hasFileStore: hasFileStore,
|
||||
hasFileDbStore: hasFileDbStore,
|
||||
);
|
||||
},
|
||||
);
|
||||
db.close();
|
||||
return inst;
|
||||
}
|
||||
|
||||
@override
|
||||
Future<T> use<T>(FutureOr<T> Function(Database) fn) async {
|
||||
final db = await _dbFactory.open(
|
||||
|
|
@ -76,15 +104,7 @@ class MockAppDb implements AppDb {
|
|||
version: 1,
|
||||
onUpgradeNeeded: (event) async {
|
||||
final db = event.database;
|
||||
final albumStore = db.createObjectStore(AppDb.albumStoreName);
|
||||
albumStore.createIndex(
|
||||
AppDbAlbumEntry.indexName, AppDbAlbumEntry.keyPath);
|
||||
final fileDbStore = db.createObjectStore(AppDb.fileDbStoreName);
|
||||
fileDbStore.createIndex(
|
||||
AppDbFileDbEntry.indexName, AppDbFileDbEntry.keyPath,
|
||||
unique: false);
|
||||
final fileStore = db.createObjectStore(AppDb.fileStoreName);
|
||||
fileStore.createIndex(AppDbFileEntry.indexName, AppDbFileEntry.keyPath);
|
||||
_createDb(db);
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -95,7 +115,50 @@ class MockAppDb implements AppDb {
|
|||
}
|
||||
}
|
||||
|
||||
static void _createDb(
|
||||
Database db, {
|
||||
bool hasAlbumStore = true,
|
||||
bool hasFileDb2Store = true,
|
||||
bool hasDirStore = true,
|
||||
// compat
|
||||
bool hasFileStore = false,
|
||||
bool hasFileDbStore = false,
|
||||
}) {
|
||||
if (hasAlbumStore) {
|
||||
final albumStore = db.createObjectStore(AppDb.albumStoreName);
|
||||
albumStore.createIndex(
|
||||
AppDbAlbumEntry.indexName, AppDbAlbumEntry.keyPath);
|
||||
}
|
||||
if (hasFileDb2Store) {
|
||||
final file2Store = db.createObjectStore(AppDb.file2StoreName);
|
||||
file2Store.createIndex(AppDbFile2Entry.strippedPathIndexName,
|
||||
AppDbFile2Entry.strippedPathKeyPath);
|
||||
}
|
||||
if (hasDirStore) {
|
||||
db.createObjectStore(AppDb.dirStoreName);
|
||||
}
|
||||
|
||||
// compat
|
||||
if (hasFileStore) {
|
||||
final fileStore = db.createObjectStore(_fileStoreName);
|
||||
fileStore.createIndex(_fileIndexName, _fileKeyPath);
|
||||
}
|
||||
if (hasFileDbStore) {
|
||||
final fileDbStore = db.createObjectStore(_fileDbStoreName);
|
||||
fileDbStore.createIndex(_fileDbIndexName, _fileDbKeyPath, unique: false);
|
||||
}
|
||||
}
|
||||
|
||||
late final _dbFactory = newIdbFactoryMemory();
|
||||
|
||||
// compat only
|
||||
static const _fileDbStoreName = "filesDb";
|
||||
static const _fileDbIndexName = "fileDbStore_namespacedFileId";
|
||||
static const _fileDbKeyPath = "namespacedFileId";
|
||||
|
||||
static const _fileStoreName = "files";
|
||||
static const _fileIndexName = "fileStore_path_index";
|
||||
static const _fileKeyPath = ["path", "index"];
|
||||
}
|
||||
|
||||
/// EventBus that ignore all events
|
||||
|
|
|
|||
|
|
@ -17,5 +17,10 @@ void main() {
|
|||
const obj = Object();
|
||||
expect(obj.run((obj) => 1), 1);
|
||||
});
|
||||
|
||||
test("runFuture", () async {
|
||||
const obj = Object();
|
||||
expect(await obj.runFuture((obj) => Future.value(1)), 1);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,7 @@ import 'package:nc_photos/entity/file.dart';
|
|||
import 'package:nc_photos/entity/share.dart';
|
||||
import 'package:nc_photos/entity/sharee.dart';
|
||||
import 'package:nc_photos/iterable_extension.dart';
|
||||
|
||||
import 'mock_type.dart';
|
||||
import 'package:nc_photos/type.dart';
|
||||
|
||||
class FilesBuilder {
|
||||
FilesBuilder({
|
||||
|
|
@ -341,13 +340,37 @@ Sharee buildSharee({
|
|||
);
|
||||
|
||||
Future<void> fillAppDb(
|
||||
MockAppDb appDb, Account account, Iterable<File> files) async {
|
||||
AppDb appDb, Account account, Iterable<File> files) async {
|
||||
await appDb.use((db) async {
|
||||
final transaction = db.transaction(AppDb.fileDbStoreName, idbModeReadWrite);
|
||||
final store = transaction.objectStore(AppDb.fileDbStoreName);
|
||||
final transaction = db.transaction(AppDb.file2StoreName, idbModeReadWrite);
|
||||
final file2Store = transaction.objectStore(AppDb.file2StoreName);
|
||||
for (final f in files) {
|
||||
await store.put(AppDbFileDbEntry.fromFile(account, f).toJson(),
|
||||
AppDbFileDbEntry.toPrimaryKey(account, f));
|
||||
await file2Store.put(AppDbFile2Entry.fromFile(account, f).toJson(),
|
||||
AppDbFile2Entry.toPrimaryKeyForFile(account, f));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Future<void> fillAppDbDir(
|
||||
AppDb appDb, Account account, File dir, List<File> children) async {
|
||||
await appDb.use((db) async {
|
||||
final transaction = db.transaction(AppDb.dirStoreName, idbModeReadWrite);
|
||||
final dirStore = transaction.objectStore(AppDb.dirStoreName);
|
||||
await dirStore.put(AppDbDirEntry.fromFiles(account, dir, children).toJson(),
|
||||
AppDbDirEntry.toPrimaryKeyForDir(account, dir));
|
||||
});
|
||||
}
|
||||
|
||||
Future<List<T>> listAppDb<T>(
|
||||
AppDb appDb, String storeName, T Function(JsonObj) transform) {
|
||||
return appDb.use((db) async {
|
||||
final transaction = db.transaction(storeName, idbModeReadOnly);
|
||||
final store = transaction.objectStore(storeName);
|
||||
return await store
|
||||
.openCursor(autoAdvance: true)
|
||||
.map((c) => c.value)
|
||||
.cast<Map>()
|
||||
.map((e) => transform(e.cast<String, dynamic>()))
|
||||
.toList();
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue