Improve usability of diff functions

This commit is contained in:
Ming Ming 2023-02-14 23:40:30 +08:00
parent 427b43fcb8
commit c68761b8c5
6 changed files with 63 additions and 54 deletions

View file

@ -138,16 +138,16 @@ class FileSqliteCacheUpdater {
final dirFiles = await dirFileQuery.get(); final dirFiles = await dirFileQuery.get();
final diff = list_util.diff(dirFiles.map((e) => e.child), final diff = list_util.diff(dirFiles.map((e) => e.child),
_childRowIds.sorted(Comparable.compare)); _childRowIds.sorted(Comparable.compare));
if (diff.item1.isNotEmpty) { if (diff.onlyInB.isNotEmpty) {
await db.batch((batch) { await db.batch((batch) {
// insert new children // insert new children
batch.insertAll(db.dirFiles, batch.insertAll(db.dirFiles,
diff.item1.map((k) => sql.DirFile(dir: _dirRowId!, child: k))); diff.onlyInB.map((k) => sql.DirFile(dir: _dirRowId!, child: k)));
}); });
} }
if (diff.item2.isNotEmpty) { if (diff.onlyInA.isNotEmpty) {
// remove entries from the DirFiles table first // remove entries from the DirFiles table first
await diff.item2.withPartitionNoReturn((sublist) async { await diff.onlyInA.withPartitionNoReturn((sublist) async {
final deleteQuery = db.delete(db.dirFiles) final deleteQuery = db.delete(db.dirFiles)
..where((t) => t.child.isIn(sublist)) ..where((t) => t.child.isIn(sublist))
..where((t) => ..where((t) =>
@ -157,7 +157,7 @@ class FileSqliteCacheUpdater {
// select files having another dir parent under this account (i.e., // select files having another dir parent under this account (i.e.,
// moved files) // moved files)
final moved = await diff.item2.withPartition((sublist) async { final moved = await diff.onlyInA.withPartition((sublist) async {
final query = db.selectOnly(db.dirFiles).join([ final query = db.selectOnly(db.dirFiles).join([
sql.innerJoin(db.accountFiles, sql.innerJoin(db.accountFiles,
db.accountFiles.file.equalsExp(db.dirFiles.dir)), db.accountFiles.file.equalsExp(db.dirFiles.dir)),
@ -168,7 +168,7 @@ class FileSqliteCacheUpdater {
..where(db.dirFiles.child.isIn(sublist)); ..where(db.dirFiles.child.isIn(sublist));
return query.map((r) => r.read(db.dirFiles.child)!).get(); return query.map((r) => r.read(db.dirFiles.child)!).get();
}, sql.maxByFileIdsSize); }, sql.maxByFileIdsSize);
final removed = diff.item2.where((e) => !moved.contains(e)).toList(); final removed = diff.onlyInA.where((e) => !moved.contains(e)).toList();
if (removed.isNotEmpty) { if (removed.isNotEmpty) {
// delete obsolete children // delete obsolete children
await _removeSqliteFiles(db, dbAccount, removed); await _removeSqliteFiles(db, dbAccount, removed);

View file

@ -1,14 +1,24 @@
import 'package:nc_photos/iterator_extension.dart'; import 'package:nc_photos/iterator_extension.dart';
import 'package:tuple/tuple.dart';
/// Contain results from the diff functions
///
/// [onlyInA] contains items exist in a but not b, [onlyInB] contains items
/// exist in b but not a
class DiffResult<T> {
const DiffResult({
required this.onlyInA,
required this.onlyInB,
});
final List<T> onlyInB;
final List<T> onlyInA;
}
/// Return the difference between two sorted lists, [a] and [b] /// Return the difference between two sorted lists, [a] and [b]
/// ///
/// [a] and [b] MUST be sorted in ascending order, otherwise the result is /// [a] and [b] MUST be sorted in ascending order, otherwise the result is
/// undefined. /// undefined
/// DiffResult<T> diffWith<T>(
/// The first returned list contains items exist in [b] but not [a], the second
/// returned list contains items exist in [a] but not [b]
Tuple2<List<T>, List<T>> diffWith<T>(
Iterable<T> a, Iterable<T> b, int Function(T a, T b) comparator) { Iterable<T> a, Iterable<T> b, int Function(T a, T b) comparator) {
final aIt = a.iterator, bIt = b.iterator; final aIt = a.iterator, bIt = b.iterator;
final aMissing = <T>[], bMissing = <T>[]; final aMissing = <T>[], bMissing = <T>[];
@ -16,47 +26,46 @@ Tuple2<List<T>, List<T>> diffWith<T>(
if (!aIt.moveNext()) { if (!aIt.moveNext()) {
// no more elements in a // no more elements in a
bIt.iterate((obj) => aMissing.add(obj)); bIt.iterate((obj) => aMissing.add(obj));
return Tuple2(aMissing, bMissing); return DiffResult(onlyInB: aMissing, onlyInA: bMissing);
} }
if (!bIt.moveNext()) { if (!bIt.moveNext()) {
// no more elements in b // no more elements in b
// needed because aIt has already advanced // needed because aIt has already advanced
bMissing.add(aIt.current); bMissing.add(aIt.current);
aIt.iterate((obj) => bMissing.add(obj)); aIt.iterate((obj) => bMissing.add(obj));
return Tuple2(aMissing, bMissing); return DiffResult(onlyInB: aMissing, onlyInA: bMissing);
} }
final result = _diffUntilEqual(aIt, bIt, comparator); final result = _diffUntilEqual(aIt, bIt, comparator);
aMissing.addAll(result.item1); aMissing.addAll(result.onlyInB);
bMissing.addAll(result.item2); bMissing.addAll(result.onlyInA);
} }
} }
Tuple2<List<T>, List<T>> diff<T extends Comparable>( DiffResult<T> diff<T extends Comparable>(Iterable<T> a, Iterable<T> b) =>
Iterable<T> a, Iterable<T> b) =>
diffWith(a, b, Comparable.compare); diffWith(a, b, Comparable.compare);
Tuple2<List<T>, List<T>> _diffUntilEqual<T>( DiffResult<T> _diffUntilEqual<T>(
Iterator<T> aIt, Iterator<T> bIt, int Function(T a, T b) comparator) { Iterator<T> aIt, Iterator<T> bIt, int Function(T a, T b) comparator) {
final a = aIt.current, b = bIt.current; final a = aIt.current, b = bIt.current;
final diff = comparator(a, b); final diff = comparator(a, b);
if (diff < 0) { if (diff < 0) {
// a < b // a < b
if (!aIt.moveNext()) { if (!aIt.moveNext()) {
return Tuple2([b] + bIt.toList(), [a]); return DiffResult(onlyInB: [b] + bIt.toList(), onlyInA: [a]);
} else { } else {
final result = _diffUntilEqual(aIt, bIt, comparator); final result = _diffUntilEqual(aIt, bIt, comparator);
return Tuple2(result.item1, [a] + result.item2); return DiffResult(onlyInB: result.onlyInB, onlyInA: [a] + result.onlyInA);
} }
} else if (diff > 0) { } else if (diff > 0) {
// a > b // a > b
if (!bIt.moveNext()) { if (!bIt.moveNext()) {
return Tuple2([b], [a] + aIt.toList()); return DiffResult(onlyInB: [b], onlyInA: [a] + aIt.toList());
} else { } else {
final result = _diffUntilEqual(aIt, bIt, comparator); final result = _diffUntilEqual(aIt, bIt, comparator);
return Tuple2([b] + result.item1, result.item2); return DiffResult(onlyInB: [b] + result.onlyInB, onlyInA: result.onlyInA);
} }
} else { } else {
// a == b // a == b
return const Tuple2([], []); return const DiffResult(onlyInB: [], onlyInA: []);
} }
} }

View file

@ -34,9 +34,9 @@ class CacheFavorite {
Map.fromEntries(cache.map((e) => MapEntry(e.fileId, e.rowId))); Map.fromEntries(cache.map((e) => MapEntry(e.fileId, e.rowId)));
final diff = final diff =
list_util.diff(cacheMap.keys.sorted(Comparable.compare), remote); list_util.diff(cacheMap.keys.sorted(Comparable.compare), remote);
final newFileIds = diff.item1; final newFileIds = diff.onlyInB;
_log.info("[call] New favorites: ${newFileIds.toReadableString()}"); _log.info("[call] New favorites: ${newFileIds.toReadableString()}");
final removedFildIds = diff.item2; final removedFildIds = diff.onlyInA;
_log.info( _log.info(
"[call] Removed favorites: ${removedFildIds.toReadableString()}"); "[call] Removed favorites: ${removedFildIds.toReadableString()}");

View file

@ -39,9 +39,9 @@ class SyncPerson {
final cache = await _c.personRepoLocal.list(account); final cache = await _c.personRepoLocal.list(account);
int personSorter(Person a, Person b) => a.name.compareTo(b.name); int personSorter(Person a, Person b) => a.name.compareTo(b.name);
final diff = list_util.diffWith<Person>(cache, remote, personSorter); final diff = list_util.diffWith<Person>(cache, remote, personSorter);
final inserts = diff.item1; final inserts = diff.onlyInB;
_log.info("[call] New people: ${inserts.toReadableString()}"); _log.info("[call] New people: ${inserts.toReadableString()}");
final deletes = diff.item2; final deletes = diff.onlyInA;
_log.info("[call] Removed people: ${deletes.toReadableString()}"); _log.info("[call] Removed people: ${deletes.toReadableString()}");
final updates = remote.where((r) { final updates = remote.where((r) {
final c = cache.firstWhereOrNull((c) => c.name == r.name); final c = cache.firstWhereOrNull((c) => c.name == r.name);

View file

@ -28,9 +28,9 @@ class SyncTag {
final remote = (await _c.tagRepoRemote.list(account))..sort(tagSorter); final remote = (await _c.tagRepoRemote.list(account))..sort(tagSorter);
final cache = (await _c.tagRepoLocal.list(account))..sort(tagSorter); final cache = (await _c.tagRepoLocal.list(account))..sort(tagSorter);
final diff = list_util.diffWith<Tag>(cache, remote, tagSorter); final diff = list_util.diffWith<Tag>(cache, remote, tagSorter);
final inserts = diff.item1; final inserts = diff.onlyInB;
_log.info("[call] New tags: ${inserts.toReadableString()}"); _log.info("[call] New tags: ${inserts.toReadableString()}");
final deletes = diff.item2; final deletes = diff.onlyInA;
_log.info("[call] Removed tags: ${deletes.toReadableString()}"); _log.info("[call] Removed tags: ${deletes.toReadableString()}");
final updates = remote.where((r) { final updates = remote.where((r) {
final c = cache.firstWhereOrNull((c) => c.id == r.id); final c = cache.firstWhereOrNull((c) => c.id == r.id);

View file

@ -27,8 +27,8 @@ void main() {
/// Expect: [1, 2], [] /// Expect: [1, 2], []
void _diffExtraBBegin() { void _diffExtraBBegin() {
final diff = list_util.diff([3, 4, 5], [1, 2, 3, 4, 5]); final diff = list_util.diff([3, 4, 5], [1, 2, 3, 4, 5]);
expect(diff.item1, [1, 2]); expect(diff.onlyInB, [1, 2]);
expect(diff.item2, []); expect(diff.onlyInA, []);
} }
/// Diff with extra elements at the end of list b /// Diff with extra elements at the end of list b
@ -38,8 +38,8 @@ void _diffExtraBBegin() {
/// Expect: [4, 5], [] /// Expect: [4, 5], []
void _diffExtraBEnd() { void _diffExtraBEnd() {
final diff = list_util.diff([1, 2, 3], [1, 2, 3, 4, 5]); final diff = list_util.diff([1, 2, 3], [1, 2, 3, 4, 5]);
expect(diff.item1, [4, 5]); expect(diff.onlyInB, [4, 5]);
expect(diff.item2, []); expect(diff.onlyInA, []);
} }
/// Diff with extra elements in the middle of list b /// Diff with extra elements in the middle of list b
@ -49,8 +49,8 @@ void _diffExtraBEnd() {
/// Expect: [3, 4], [] /// Expect: [3, 4], []
void _diffExtraBMid() { void _diffExtraBMid() {
final diff = list_util.diff([1, 2, 5], [1, 2, 3, 4, 5]); final diff = list_util.diff([1, 2, 5], [1, 2, 3, 4, 5]);
expect(diff.item1, [3, 4]); expect(diff.onlyInB, [3, 4]);
expect(diff.item2, []); expect(diff.onlyInA, []);
} }
/// Diff with list a being empty /// Diff with list a being empty
@ -60,8 +60,8 @@ void _diffExtraBMid() {
/// Expect: [1, 2, 3], [] /// Expect: [1, 2, 3], []
void _diffAEmpty() { void _diffAEmpty() {
final diff = list_util.diff(<int>[], [1, 2, 3]); final diff = list_util.diff(<int>[], [1, 2, 3]);
expect(diff.item1, [1, 2, 3]); expect(diff.onlyInB, [1, 2, 3]);
expect(diff.item2, []); expect(diff.onlyInA, []);
} }
/// Diff with extra elements at the beginning of list a /// Diff with extra elements at the beginning of list a
@ -71,8 +71,8 @@ void _diffAEmpty() {
/// Expect: [], [1, 2] /// Expect: [], [1, 2]
void _diffExtraABegin() { void _diffExtraABegin() {
final diff = list_util.diff([1, 2, 3, 4, 5], [3, 4, 5]); final diff = list_util.diff([1, 2, 3, 4, 5], [3, 4, 5]);
expect(diff.item1, []); expect(diff.onlyInB, []);
expect(diff.item2, [1, 2]); expect(diff.onlyInA, [1, 2]);
} }
/// Diff with extra elements at the end of list a /// Diff with extra elements at the end of list a
@ -82,8 +82,8 @@ void _diffExtraABegin() {
/// Expect: [], [4, 5] /// Expect: [], [4, 5]
void _diffExtraAEnd() { void _diffExtraAEnd() {
final diff = list_util.diff([1, 2, 3, 4, 5], [1, 2, 3]); final diff = list_util.diff([1, 2, 3, 4, 5], [1, 2, 3]);
expect(diff.item1, []); expect(diff.onlyInB, []);
expect(diff.item2, [4, 5]); expect(diff.onlyInA, [4, 5]);
} }
/// Diff with extra elements in the middle of list a /// Diff with extra elements in the middle of list a
@ -93,8 +93,8 @@ void _diffExtraAEnd() {
/// Expect: [], [3, 4] /// Expect: [], [3, 4]
void _diffExtraAMid() { void _diffExtraAMid() {
final diff = list_util.diff([1, 2, 3, 4, 5], [1, 2, 5]); final diff = list_util.diff([1, 2, 3, 4, 5], [1, 2, 5]);
expect(diff.item1, []); expect(diff.onlyInB, []);
expect(diff.item2, [3, 4]); expect(diff.onlyInA, [3, 4]);
} }
/// Diff with list b being empty /// Diff with list b being empty
@ -104,8 +104,8 @@ void _diffExtraAMid() {
/// Expect: [], [1, 2, 3] /// Expect: [], [1, 2, 3]
void _diffBEmpty() { void _diffBEmpty() {
final diff = list_util.diff([1, 2, 3], <int>[]); final diff = list_util.diff([1, 2, 3], <int>[]);
expect(diff.item1, []); expect(diff.onlyInB, []);
expect(diff.item2, [1, 2, 3]); expect(diff.onlyInA, [1, 2, 3]);
} }
/// Diff with no matches between list a and b /// Diff with no matches between list a and b
@ -115,8 +115,8 @@ void _diffBEmpty() {
/// Expect: [2, 4], [1, 3, 5] /// Expect: [2, 4], [1, 3, 5]
void _diffNoMatches() { void _diffNoMatches() {
final diff = list_util.diff([1, 3, 5], [2, 4]); final diff = list_util.diff([1, 3, 5], [2, 4]);
expect(diff.item1, [2, 4]); expect(diff.onlyInB, [2, 4]);
expect(diff.item2, [1, 3, 5]); expect(diff.onlyInA, [1, 3, 5]);
} }
/// Diff between list a and b with repeated elements /// Diff between list a and b with repeated elements
@ -126,8 +126,8 @@ void _diffNoMatches() {
/// Expect: [2], [] /// Expect: [2], []
void _diffRepeatedElements() { void _diffRepeatedElements() {
final diff = list_util.diff([1, 2, 3], [1, 2, 2, 3]); final diff = list_util.diff([1, 2, 3], [1, 2, 2, 3]);
expect(diff.item1, [2]); expect(diff.onlyInB, [2]);
expect(diff.item2, []); expect(diff.onlyInA, []);
} }
/// Diff between list a and b with repeated elements /// Diff between list a and b with repeated elements
@ -137,8 +137,8 @@ void _diffRepeatedElements() {
/// Expect: [2, 2], [4, 4] /// Expect: [2, 2], [4, 4]
void _diffRepeatedElements2() { void _diffRepeatedElements2() {
final diff = list_util.diff([1, 3, 4, 4, 5], [1, 2, 2, 3, 5]); final diff = list_util.diff([1, 3, 4, 4, 5], [1, 2, 2, 3, 5]);
expect(diff.item1, [2, 2]); expect(diff.onlyInB, [2, 2]);
expect(diff.item2, [4, 4]); expect(diff.onlyInA, [4, 4]);
} }
/// Diff between list a and b /// Diff between list a and b
@ -148,6 +148,6 @@ void _diffRepeatedElements2() {
/// Expect: [1, 4, 8, 13, 14], [2, 7, 10, 11, 12] /// Expect: [1, 4, 8, 13, 14], [2, 7, 10, 11, 12]
void _diffMix() { void _diffMix() {
final diff = list_util.diff([2, 3, 7, 10, 11, 12], [1, 3, 4, 8, 13, 14]); final diff = list_util.diff([2, 3, 7, 10, 11, 12], [1, 3, 4, 8, 13, 14]);
expect(diff.item1, [1, 4, 8, 13, 14]); expect(diff.onlyInB, [1, 4, 8, 13, 14]);
expect(diff.item2, [2, 7, 10, 11, 12]); expect(diff.onlyInA, [2, 7, 10, 11, 12]);
} }