part of 'database.dart'; const maxByFileIdsSize = 30000; class CompleteFile { const CompleteFile( this.file, this.accountFile, this.image, this.imageLocation, this.trash); final File file; final AccountFile accountFile; final Image? image; final ImageLocation? imageLocation; final Trash? trash; } class CompleteFileCompanion { const CompleteFileCompanion( this.file, this.accountFile, this.image, this.imageLocation, this.trash); final FilesCompanion file; final AccountFilesCompanion accountFile; final ImagesCompanion? image; final ImageLocationsCompanion? imageLocation; final TrashesCompanion? trash; } extension CompleteFileListExtension on List { Future> convertToAppFile(app.Account account) { return map((f) => { "userId": account.userId.toString(), "completeFile": f, }).computeAll(_covertSqliteDbFile); } } extension FileListExtension on List { Future> convertToFileCompanion(Account? account) { return map((f) => { "account": account, "file": f, }).computeAll(_convertAppFile); } } class FileDescriptor { const FileDescriptor({ required this.relativePath, required this.fileId, required this.contentType, required this.isArchived, required this.isFavorite, required this.bestDateTime, }); final String relativePath; final int fileId; final String? contentType; final bool? isArchived; final bool? isFavorite; final DateTime bestDateTime; } extension FileDescriptorListExtension on List { List convertToAppFileDescriptor(app.Account account) { return map((f) => SqliteFileDescriptorConverter.fromSql(account.userId.toString(), f)) .toList(); } } class AlbumWithShare { const AlbumWithShare(this.album, this.share); final Album album; final AlbumShare? share; } class CompleteAlbumCompanion { const CompleteAlbumCompanion(this.album, this.albumShares); final AlbumsCompanion album; final List albumShares; } class AccountFileRowIds { const AccountFileRowIds( this.accountFileRowId, this.accountRowId, this.fileRowId); final int accountFileRowId; final int accountRowId; final int fileRowId; } class AccountFileRowIdsWithFileId { const AccountFileRowIdsWithFileId( this.accountFileRowId, this.accountRowId, this.fileRowId, this.fileId); final int accountFileRowId; final int accountRowId; final int fileRowId; final int fileId; } class ByAccount { const ByAccount.sql(Account account) : this._(sqlAccount: account); const ByAccount.app(app.Account account) : this._(appAccount: account); const ByAccount._({ this.sqlAccount, this.appAccount, }) : assert((sqlAccount != null) != (appAccount != null)); final Account? sqlAccount; final app.Account? appAccount; } extension SqliteDbExtension on SqliteDb { /// Start a transaction and run [block] /// /// The [db] argument passed to [block] is identical to this /// /// Do NOT call this when using [isolate], call [useInIsolate] instead Future use(Future Function(SqliteDb db) block) async { return await platform.Lock.synchronized(k.appDbLockId, () async { return await transaction(() async { return await block(this); }); }); } /// Run [block] after acquiring the database /// /// The [db] argument passed to [block] is identical to this /// /// This function does not start a transaction, see [use] instead Future useNoTransaction(Future Function(SqliteDb db) block) async { return await platform.Lock.synchronized(k.appDbLockId, () async { return await block(this); }); } /// Start an isolate and run [callback] there, with access to the /// SQLite database Future isolate(T args, ComputeWithDbCallback callback) async { // we need to acquire the lock here as method channel is not supported in // background isolates return await platform.Lock.synchronized(k.appDbLockId, () async { // in unit tests we use an in-memory db, which mean there's no way to // access it in other isolates if (platform_k.isUnitTest) { return await callback(this, args); } else { return await computeWithDb(callback, args); } }); } /// Start a transaction and run [block], this version is suitable to be called /// in [isolate] /// /// See: [use] Future useInIsolate(Future Function(SqliteDb db) block) async { return await transaction(() async { return await block(this); }); } Future insertAccountOf(app.Account account) async { Server dbServer; try { dbServer = await into(servers).insertReturning( ServersCompanion.insert( address: account.url, ), mode: InsertMode.insertOrIgnore, ); } on StateError catch (_) { // already exists final query = select(servers) ..where((t) => t.address.equals(account.url)); dbServer = await query.getSingle(); } await into(accounts).insert( AccountsCompanion.insert( server: dbServer.rowId, userId: account.userId.toCaseInsensitiveString(), ), mode: InsertMode.insertOrIgnore, ); } Future accountOf(app.Account account) { final query = select(accounts).join([ innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false) ]) ..where(servers.address.equals(account.url)) ..where(accounts.userId.equals(account.userId.toCaseInsensitiveString())) ..limit(1); return query.map((r) => r.readTable(accounts)).getSingle(); } /// Delete Account by app Account /// /// If the deleted Account is the last one associated with a Server, then the /// Server will also be deleted Future deleteAccountOf(app.Account account) async { final dbAccount = await accountOf(account); _log.info("[deleteAccountOf] Remove account: ${dbAccount.rowId}"); await (delete(accounts)..where((t) => t.rowId.equals(dbAccount.rowId))) .go(); final accountCountExp = accounts.rowId.count(filter: accounts.server.equals(dbAccount.server)); final accountCountQuery = selectOnly(accounts) ..addColumns([accountCountExp]); final accountCount = await accountCountQuery.map((r) => r.read(accountCountExp)).getSingle(); _log.info("[deleteAccountOf] Remaining accounts in server: $accountCount"); if (accountCount == 0) { _log.info("[deleteAccountOf] Remove server: ${dbAccount.server}"); await (delete(servers)..where((t) => t.rowId.equals(dbAccount.server))) .go(); } await cleanUpDanglingFiles(); } /// Delete Files without a corresponding entry in AccountFiles Future cleanUpDanglingFiles() async { final query = selectOnly(files).join([ leftOuterJoin(accountFiles, accountFiles.file.equalsExp(files.rowId), useColumns: false), ]) ..addColumns([files.rowId]) ..where(accountFiles.relativePath.isNull()); final fileRowIds = await query.map((r) => r.read(files.rowId)!).get(); if (fileRowIds.isNotEmpty) { _log.info("[cleanUpDanglingFiles] Delete ${fileRowIds.length} files"); await fileRowIds.withPartitionNoReturn((sublist) async { await (delete(files)..where((t) => t.rowId.isIn(sublist))).go(); }, maxByFileIdsSize); } } FilesQueryBuilder queryFiles() => FilesQueryBuilder(this); /// Query File by app File /// /// Only one of [sqlAccount] and [appAccount] must be passed Future fileOf( app.File file, { Account? sqlAccount, app.Account? appAccount, }) { assert((sqlAccount != null) != (appAccount != null)); final query = queryFiles().run((q) { q.setQueryMode(FilesQueryMode.file); if (sqlAccount != null) { q.setSqlAccount(sqlAccount); } else { q.setAppAccount(appAccount!); } if (file.fileId != null) { q.byFileId(file.fileId!); } else { q.byRelativePath(file.strippedPathWithEmpty); } return q.build()..limit(1); }); return query.map((r) => r.readTable(files)).getSingle(); } /// Query AccountFiles, Accounts and Files row ID by app File /// /// Only one of [sqlAccount] and [appAccount] must be passed Future accountFileRowIdsOfOrNull( app.FileDescriptor file, { Account? sqlAccount, app.Account? appAccount, }) { assert((sqlAccount != null) != (appAccount != null)); final query = queryFiles().run((q) { q.setQueryMode(FilesQueryMode.expression, expressions: [ accountFiles.rowId, accountFiles.account, accountFiles.file, ]); if (sqlAccount != null) { q.setSqlAccount(sqlAccount); } else { q.setAppAccount(appAccount!); } try { q.byFileId(file.fdId); } catch (_) { q.byRelativePath(file.strippedPathWithEmpty); } return q.build()..limit(1); }); return query .map((r) => AccountFileRowIds( r.read(accountFiles.rowId)!, r.read(accountFiles.account)!, r.read(accountFiles.file)!, )) .getSingleOrNull(); } /// See [accountFileRowIdsOfOrNull] Future accountFileRowIdsOf( app.FileDescriptor file, { Account? sqlAccount, app.Account? appAccount, }) => accountFileRowIdsOfOrNull(file, sqlAccount: sqlAccount, appAccount: appAccount) .notNull(); /// Query AccountFiles, Accounts and Files row ID by fileIds /// /// Returned files are NOT guaranteed to be sorted as [fileIds] Future> accountFileRowIdsByFileIds( ByAccount account, Iterable fileIds) { return fileIds.withPartition((sublist) { final query = queryFiles().run((q) { q.setQueryMode(FilesQueryMode.expression, expressions: [ accountFiles.rowId, accountFiles.account, accountFiles.file, files.fileId, ]); if (account.sqlAccount != null) { q.setSqlAccount(account.sqlAccount!); } else { q.setAppAccount(account.appAccount!); } q.byFileIds(sublist); return q.build(); }); return query .map((r) => AccountFileRowIdsWithFileId( r.read(accountFiles.rowId)!, r.read(accountFiles.account)!, r.read(accountFiles.file)!, r.read(files.fileId)!, )) .get(); }, maxByFileIdsSize); } /// Query CompleteFile by fileId /// /// Returned files are NOT guaranteed to be sorted as [fileIds] Future> completeFilesByFileIds( Iterable fileIds, { Account? sqlAccount, app.Account? appAccount, }) { assert((sqlAccount != null) != (appAccount != null)); return fileIds.withPartition((sublist) { final query = queryFiles().run((q) { q.setQueryMode(FilesQueryMode.completeFile); if (sqlAccount != null) { q.setSqlAccount(sqlAccount); } else { q.setAppAccount(appAccount!); } q.byFileIds(sublist); return q.build(); }); return query .map((r) => CompleteFile( r.readTable(files), r.readTable(accountFiles), r.readTableOrNull(images), r.readTableOrNull(imageLocations), r.readTableOrNull(trashes), )) .get(); }, maxByFileIdsSize); } Future> completeFilesByDirRowId( int dirRowId, { Account? sqlAccount, app.Account? appAccount, }) { assert((sqlAccount != null) != (appAccount != null)); final query = queryFiles().run((q) { q.setQueryMode(FilesQueryMode.completeFile); if (sqlAccount != null) { q.setSqlAccount(sqlAccount); } else { q.setAppAccount(appAccount!); } q.byDirRowId(dirRowId); return q.build(); }); return query .map((r) => CompleteFile( r.readTable(files), r.readTable(accountFiles), r.readTableOrNull(images), r.readTableOrNull(imageLocations), r.readTableOrNull(trashes), )) .get(); } /// Query CompleteFile by favorite Future> completeFilesByFavorite({ Account? sqlAccount, app.Account? appAccount, }) { assert((sqlAccount != null) != (appAccount != null)); final query = queryFiles().run((q) { q.setQueryMode(FilesQueryMode.completeFile); if (sqlAccount != null) { q.setSqlAccount(sqlAccount); } else { q.setAppAccount(appAccount!); } q.byFavorite(true); return q.build(); }); return query .map((r) => CompleteFile( r.readTable(files), r.readTable(accountFiles), r.readTableOrNull(images), r.readTableOrNull(imageLocations), r.readTableOrNull(trashes), )) .get(); } /// Query [FileDescriptor]s by fileId /// /// Returned files are NOT guaranteed to be sorted as [fileIds] Future> fileDescriptorsByFileIds( ByAccount account, Iterable fileIds) { return fileIds.withPartition((sublist) { final query = queryFiles().run((q) { q.setQueryMode( FilesQueryMode.expression, expressions: [ accountFiles.relativePath, files.fileId, files.contentType, accountFiles.isArchived, accountFiles.isFavorite, accountFiles.bestDateTime, ], ); if (account.sqlAccount != null) { q.setSqlAccount(account.sqlAccount!); } else { q.setAppAccount(account.appAccount!); } q.byFileIds(sublist); return q.build(); }); return query .map((r) => FileDescriptor( relativePath: r.read(accountFiles.relativePath)!, fileId: r.read(files.fileId)!, contentType: r.read(files.contentType), isArchived: r.read(accountFiles.isArchived), isFavorite: r.read(accountFiles.isFavorite), bestDateTime: r.read(accountFiles.bestDateTime)!, )) .get(); }, maxByFileIdsSize); } Future moveFileByFileId( ByAccount account, int fileId, String destinationRelativePath) async { final rowId = (await accountFileRowIdsByFileIds(account, [fileId])).first; final q = update(accountFiles) ..where((t) => t.rowId.equals(rowId.accountFileRowId)); await q.write(AccountFilesCompanion( relativePath: Value(destinationRelativePath), )); } Future> allTags({ Account? sqlAccount, app.Account? appAccount, }) { assert((sqlAccount != null) != (appAccount != null)); if (sqlAccount != null) { final query = select(tags) ..where((t) => t.server.equals(sqlAccount.server)); return query.get(); } else { final query = select(tags).join([ innerJoin(servers, servers.rowId.equalsExp(tags.server), useColumns: false), ]) ..where(servers.address.equals(appAccount!.url)); return query.map((r) => r.readTable(tags)).get(); } } Future tagByDisplayName({ Account? sqlAccount, app.Account? appAccount, required String displayName, }) { assert((sqlAccount != null) != (appAccount != null)); if (sqlAccount != null) { final query = select(tags) ..where((t) => t.server.equals(sqlAccount.server)) ..where((t) => t.displayName.like(displayName)) ..limit(1); return query.getSingleOrNull(); } else { final query = select(tags).join([ innerJoin(servers, servers.rowId.equalsExp(tags.server), useColumns: false), ]) ..where(servers.address.equals(appAccount!.url)) ..where(tags.displayName.like(displayName)) ..limit(1); return query.map((r) => r.readTable(tags)).getSingleOrNull(); } } Future> allFaceRecognitionPersons({ required ByAccount account, }) { assert((account.sqlAccount != null) != (account.appAccount != null)); if (account.sqlAccount != null) { final query = select(faceRecognitionPersons) ..where((t) => t.account.equals(account.sqlAccount!.rowId)); return query.get(); } else { final query = select(faceRecognitionPersons).join([ innerJoin( accounts, accounts.rowId.equalsExp(faceRecognitionPersons.account), useColumns: false), innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false), ]) ..where(servers.address.equals(account.appAccount!.url)) ..where(accounts.userId .equals(account.appAccount!.userId.toCaseInsensitiveString())); return query.map((r) => r.readTable(faceRecognitionPersons)).get(); } } Future> faceRecognitionPersonsByName({ Account? sqlAccount, app.Account? appAccount, required String name, }) { assert((sqlAccount != null) != (appAccount != null)); if (sqlAccount != null) { final query = select(faceRecognitionPersons) ..where((t) => t.account.equals(sqlAccount.rowId)) ..where((t) => t.name.like(name) | t.name.like("% $name") | t.name.like("$name %")); return query.get(); } else { final query = select(faceRecognitionPersons).join([ innerJoin( accounts, accounts.rowId.equalsExp(faceRecognitionPersons.account), useColumns: false), innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false), ]) ..where(servers.address.equals(appAccount!.url)) ..where( accounts.userId.equals(appAccount.userId.toCaseInsensitiveString())) ..where(faceRecognitionPersons.name.like(name) | faceRecognitionPersons.name.like("% $name") | faceRecognitionPersons.name.like("$name %")); return query.map((r) => r.readTable(faceRecognitionPersons)).get(); } } Future> allRecognizeFaces({ required ByAccount account, }) { assert((account.sqlAccount != null) != (account.appAccount != null)); if (account.sqlAccount != null) { final query = select(recognizeFaces) ..where((t) => t.account.equals(account.sqlAccount!.rowId)); return query.get(); } else { final query = select(recognizeFaces).join([ innerJoin(accounts, accounts.rowId.equalsExp(recognizeFaces.account), useColumns: false), innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false), ]) ..where(servers.address.equals(account.appAccount!.url)) ..where(accounts.userId .equals(account.appAccount!.userId.toCaseInsensitiveString())); return query.map((r) => r.readTable(recognizeFaces)).get(); } } Future recognizeFaceByLabel({ required ByAccount account, required String label, }) { assert((account.sqlAccount != null) != (account.appAccount != null)); if (account.sqlAccount != null) { final query = select(recognizeFaces) ..where((t) => t.account.equals(account.sqlAccount!.rowId)) ..where((t) => t.label.equals(label)); return query.getSingle(); } else { final query = select(recognizeFaces).join([ innerJoin(accounts, accounts.rowId.equalsExp(recognizeFaces.account), useColumns: false), innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false), ]) ..where(servers.address.equals(account.appAccount!.url)) ..where(accounts.userId .equals(account.appAccount!.userId.toCaseInsensitiveString())) ..where(recognizeFaces.label.equals(label)); return query.map((r) => r.readTable(recognizeFaces)).getSingle(); } } Future> recognizeFaceItemsByParentLabel({ required ByAccount account, required String label, List? orderBy, int? limit, int? offset, }) { assert((account.sqlAccount != null) != (account.appAccount != null)); final query = select(recognizeFaceItems).join([ innerJoin(recognizeFaces, recognizeFaces.rowId.equalsExp(recognizeFaceItems.parent), useColumns: false), ]); if (account.sqlAccount != null) { query ..where(recognizeFaces.account.equals(account.sqlAccount!.rowId)) ..where(recognizeFaces.label.equals(label)); } else { query ..join([ innerJoin(accounts, accounts.rowId.equalsExp(recognizeFaces.account), useColumns: false), innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false), ]) ..where(servers.address.equals(account.appAccount!.url)) ..where(accounts.userId .equals(account.appAccount!.userId.toCaseInsensitiveString())) ..where(recognizeFaces.label.equals(label)); } if (orderBy != null) { query.orderBy(orderBy); if (limit != null) { query.limit(limit, offset: offset); } } return query.map((r) => r.readTable(recognizeFaceItems)).get(); } Future countMissingMetadataByFileIds({ Account? sqlAccount, app.Account? appAccount, required List fileIds, }) async { assert((sqlAccount != null) != (appAccount != null)); if (fileIds.isEmpty) { return 0; } final counts = await fileIds.withPartition((sublist) async { final count = countAll( filter: images.lastUpdated.isNull() | imageLocations.version.isNull()); final query = selectOnly(files).join([ innerJoin(accountFiles, accountFiles.file.equalsExp(files.rowId), useColumns: false), if (appAccount != null) ...[ innerJoin(accounts, accounts.rowId.equalsExp(accountFiles.account), useColumns: false), innerJoin(servers, servers.rowId.equalsExp(accounts.server), useColumns: false), ], leftOuterJoin(images, images.accountFile.equalsExp(accountFiles.rowId), useColumns: false), leftOuterJoin(imageLocations, imageLocations.accountFile.equalsExp(accountFiles.rowId), useColumns: false), ]); query.addColumns([count]); if (sqlAccount != null) { query.where(accountFiles.account.equals(sqlAccount.rowId)); } else if (appAccount != null) { query ..where(servers.address.equals(appAccount.url)) ..where(accounts.userId .equals(appAccount.userId.toCaseInsensitiveString())); } query ..where(files.fileId.isIn(sublist)) ..where(whereFileIsSupportedImageMime()); return [await query.map((r) => r.read(count)!).getSingle()]; }, maxByFileIdsSize); return counts.reduce((value, element) => value + element); } Future truncate() async { await delete(servers).go(); // technically deleting Servers table is enough to clear the followings, but // just in case await delete(accounts).go(); await delete(files).go(); await delete(images).go(); await delete(imageLocations).go(); await delete(trashes).go(); await delete(accountFiles).go(); await delete(dirFiles).go(); await delete(albums).go(); await delete(albumShares).go(); await delete(tags).go(); await delete(faceRecognitionPersons).go(); await delete(ncAlbums).go(); await delete(ncAlbumItems).go(); await delete(recognizeFaces).go(); await delete(recognizeFaceItems).go(); // reset the auto increment counter await customStatement("UPDATE sqlite_sequence SET seq=0;"); } Expression whereFileIsSupportedMime() { return file_util.supportedFormatMimes .map>((m) => files.contentType.equals(m)) .reduce((value, element) => value | element); } Expression whereFileIsSupportedImageMime() { return file_util.supportedImageFormatMimes .map>((m) => files.contentType.equals(m)) .reduce((value, element) => value | element); } } app.File _covertSqliteDbFile(Map map) { final userId = map["userId"] as String; final file = map["completeFile"] as CompleteFile; return SqliteFileConverter.fromSql(userId, file); } CompleteFileCompanion _convertAppFile(Map map) { final account = map["account"] as Account?; final file = map["file"] as app.File; return SqliteFileConverter.toSql(account, file); }