Reload no longer clear timeline stream

app/lib/controller/files_controller.dart

@@ -530,9 +530,23 @@ class FilesController {
       files: _toFileMap(newFiles),
       hasNext: false,
     ));
-    await _reloadSummary();
+    final diff = await _reloadSummary();
-    _timelineStreamController
+    final dropDates = [
-        .addWithValue((value) => value.copyWith(data: const {}));
+      ...diff.onlyInThis.keys,
+      ...diff.onlyInOther.keys,
+      ...diff.updated.keys,
+    ];
+    if (dropDates.isNotEmpty) {
+      _timelineStreamController.addWithValue((value) {
+        final next = <int, FileDescriptor>{};
+        for (final e in value.data.entries) {
+          if (!dropDates.contains(e.value.fdDateTime.toLocal().toDate())) {
+            next[e.key] = e.value;
+          }
+        }
+        return value.copyWith(data: next);
+      });
+    }
   }
 
   Map<int, FileDescriptor> _toFileMap(List<FileDescriptor> results) {
@@ -541,7 +555,9 @@ class FilesController {
     };
   }
 
-  Future<void> _reloadSummary() async {
+  Future<DbFilesSummaryDiff> _reloadSummary() async {
+    final original = _summaryStreamController.valueOrNull?.summary ??
+        const DbFilesSummary(items: {});
     final results = await _c.npDb.getFilesSummary(
       account: account.toDb(),
       includeRelativeRoots: account.roots
@@ -551,7 +567,9 @@ class FilesController {
       excludeRelativeRoots: [remote_storage_util.remoteStorageDirRelativePath],
       mimes: file_util.supportedFormatMimes,
     );
+    final diff = original.diff(results);
     _summaryStreamController.add(FilesSummaryStreamEvent(summary: results));
+    return diff;
   }
 
   _MockResult _mockRemove({

np_collection/lib/src/iterator_extension.dart

@@ -11,3 +11,11 @@ extension IteratorExtionsion<T> on Iterator<T> {
     return list;
   }
 }
+
+extension IteratorMapEntryExtionsion<T, U> on Iterator<MapEntry<T, U>> {
+  Map<T, U> toMap() {
+    final result = <T, U>{};
+    iterate((obj) => result[obj.key] = obj.value);
+    return result;
+  }
+}

np_db/lib/np_db.dart

@@ -3,3 +3,4 @@ library np_db;
 export 'src/api.dart';
 export 'src/entity.dart';
 export 'src/exception.dart';
+export 'src/util.dart';

np_db/lib/src/api.dart

@@ -133,7 +133,7 @@ class DbLocationGroupResult {
 
 @genCopyWith
 @toString
-class DbFilesSummaryItem {
+class DbFilesSummaryItem with EquatableMixin {
   const DbFilesSummaryItem({
     required this.count,
   });
@@ -141,6 +141,11 @@ class DbFilesSummaryItem {
   @override
   String toString() => _$toString();
 
+  @override
+  List<Object?> get props => [
+        count,
+      ];
+
   final int count;
 }
 

np_db/lib/src/util.dart

@@ -0,0 +1,119 @@
+import 'dart:collection';
+
+import 'package:equatable/equatable.dart';
+import 'package:np_collection/np_collection.dart';
+import 'package:np_datetime/np_datetime.dart';
+import 'package:np_db/src/api.dart';
+
+class DbFilesSummaryDiff with EquatableMixin {
+  const DbFilesSummaryDiff({
+    required this.onlyInThis,
+    required this.onlyInOther,
+    required this.updated,
+  });
+
+  @override
+  List<Object?> get props => [
+        onlyInThis,
+        onlyInOther,
+        updated,
+      ];
+
+  final Map<Date, DbFilesSummaryItem> onlyInThis;
+  final Map<Date, DbFilesSummaryItem> onlyInOther;
+  final Map<Date, DbFilesSummaryItem> updated;
+}
+
+extension DbFilesSummaryExtension on DbFilesSummary {
+  DbFilesSummaryDiff diff(DbFilesSummary other) {
+    final thisIt = items.entries.toList().reversed.iterator;
+    final otherIt = other.items.entries.toList().reversed.iterator;
+    final thisMissing = <Date, DbFilesSummaryItem>{};
+    final otherMissing = <Date, DbFilesSummaryItem>{};
+    final updated = <Date, DbFilesSummaryItem>{};
+    while (true) {
+      if (!thisIt.moveNext()) {
+        // no more elements in this
+        otherIt.iterate((obj) {
+          thisMissing[obj.key] = obj.value;
+        });
+        return DbFilesSummaryDiff(
+          onlyInOther:
+              LinkedHashMap.fromEntries(thisMissing.entries.toList().reversed),
+          onlyInThis:
+              LinkedHashMap.fromEntries(otherMissing.entries.toList().reversed),
+          updated: LinkedHashMap.fromEntries(updated.entries.toList().reversed),
+        );
+      }
+      if (!otherIt.moveNext()) {
+        // no more elements in other
+        // needed because thisIt has already advanced
+        otherMissing[thisIt.current.key] = thisIt.current.value;
+        thisIt.iterate((obj) {
+          otherMissing[obj.key] = obj.value;
+        });
+        return DbFilesSummaryDiff(
+          onlyInOther:
+              LinkedHashMap.fromEntries(thisMissing.entries.toList().reversed),
+          onlyInThis:
+              LinkedHashMap.fromEntries(otherMissing.entries.toList().reversed),
+          updated: LinkedHashMap.fromEntries(updated.entries.toList().reversed),
+        );
+      }
+      final result = _diffUntilEqual(thisIt, otherIt);
+      thisMissing.addAll(result.onlyInOther);
+      otherMissing.addAll(result.onlyInThis);
+      updated.addAll(result.updated);
+    }
+  }
+
+  DbFilesSummaryDiff _diffUntilEqual(
+    Iterator<MapEntry<Date, DbFilesSummaryItem>> thisIt,
+    Iterator<MapEntry<Date, DbFilesSummaryItem>> otherIt,
+  ) {
+    final thisObj = thisIt.current, otherObj = otherIt.current;
+    final diff = thisObj.key.compareTo(otherObj.key);
+    if (diff < 0) {
+      // this < other
+      if (!thisIt.moveNext()) {
+        return DbFilesSummaryDiff(
+          onlyInOther: Map.fromEntries([otherObj])..addAll(otherIt.toMap()),
+          onlyInThis: Map.fromEntries([thisObj]),
+          updated: const {},
+        );
+      } else {
+        final result = _diffUntilEqual(thisIt, otherIt);
+        return DbFilesSummaryDiff(
+          onlyInOther: result.onlyInOther,
+          onlyInThis: Map.fromEntries([thisObj])..addAll(result.onlyInThis),
+          updated: const {},
+        );
+      }
+    } else if (diff > 0) {
+      // this > other
+      if (!otherIt.moveNext()) {
+        return DbFilesSummaryDiff(
+          onlyInOther: Map.fromEntries([otherObj]),
+          onlyInThis: Map.fromEntries([thisObj])..addAll(thisIt.toMap()),
+          updated: const {},
+        );
+      } else {
+        final result = _diffUntilEqual(thisIt, otherIt);
+        return DbFilesSummaryDiff(
+          onlyInOther: Map.fromEntries([otherObj])..addAll(result.onlyInOther),
+          onlyInThis: result.onlyInThis,
+          updated: const {},
+        );
+      }
+    } else {
+      // this == other
+      return DbFilesSummaryDiff(
+        onlyInOther: const {},
+        onlyInThis: const {},
+        updated: thisObj.value == otherObj.value
+            ? const {}
+            : Map.fromEntries([otherObj]),
+      );
+    }
+  }
+}

np_db/pubspec.yaml

@@ -20,6 +20,8 @@ dependencies:
   logging: ^1.1.1
   np_codegen:
     path: ../codegen
+  np_collection:
+    path: ../np_collection
   np_common:
     path: ../np_common
   np_datetime:

np_db/test/util_test.dart

@@ -0,0 +1,281 @@
+import 'package:np_datetime/np_datetime.dart';
+import 'package:np_db/np_db.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group("DbFilesSummaryExtension", () {
+    group("diff", () {
+      test("extra other begin", _diffExtraOtherBegin);
+      test("extra other end", _diffExtraOtherEnd);
+      test("extra other mid", _diffExtraOtherMid);
+      test("empty this", _diffThisEmpty);
+      test("extra this begin", _diffExtraThisBegin);
+      test("extra this end", _diffExtraThisEnd);
+      test("extra this mid", _diffExtraThisMid);
+      test("empty other", _diffOtherEmpty);
+      test("no matches", _diffNoMatches);
+    });
+  });
+}
+
+/// Diff with extra elements at the beginning of other list
+///
+/// this: {13/1/2024: 5, 12/1/2024: 4}
+/// other: {15/1/2024: 7 ,14/1/2024: 6, 13/1/2024: 5, 12/1/2024: 4}
+/// Expect: {}, {15/1/2024: 7 ,14/1/2024: 6}, {}
+void _diffExtraOtherBegin() {
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+  });
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 15): const DbFilesSummaryItem(count: 7),
+    Date(2024, 1, 14): const DbFilesSummaryItem(count: 6),
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: const {},
+      onlyInOther: {
+        Date(2024, 1, 15): const DbFilesSummaryItem(count: 7),
+        Date(2024, 1, 14): const DbFilesSummaryItem(count: 6),
+      },
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with extra elements at the end of other list
+///
+/// this: {13/1/2024: 5, 12/1/2024: 4}
+/// other: {13/1/2024: 5, 12/1/2024: 4, 11/1/2024: 3, 10/1/2024: 2}
+/// Expect: {}, {11/1/2024: 3, 10/1/2024: 2}, {}
+void _diffExtraOtherEnd() {
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+  });
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+    Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: const {},
+      onlyInOther: {
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+        Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+      },
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with extra elements in the middle of other list
+///
+/// this: {13/1/2024: 5, 12/1/2024: 4, 9/1/2024: 1}
+/// other: {13/1/2024: 5, 12/1/2024: 4, 11/1/2024: 3, 10/1/2024: 2, 9/1/2024: 1}
+/// Expect: {}, {11/1/2024: 3, 10/1/2024: 2}, {}
+void _diffExtraOtherMid() {
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 9): const DbFilesSummaryItem(count: 1),
+  });
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+    Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+    Date(2024, 1, 9): const DbFilesSummaryItem(count: 1),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: const {},
+      onlyInOther: {
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+        Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+      },
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with this being empty
+///
+/// this: {}
+/// other: {13/1/2024: 5, 12/1/2024: 4, 11/1/2024: 3, 10/1/2024: 2, 9/1/2024: 1}
+/// Expect: {}, {11/1/2024: 3, 10/1/2024: 2}, {}
+void _diffThisEmpty() {
+  const obj = DbFilesSummary(items: {});
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: const {},
+      onlyInOther: {
+        Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+        Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+      },
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with extra elements at the beginning of this list
+///
+/// this: {15/1/2024: 7 ,14/1/2024: 6, 13/1/2024: 5, 12/1/2024: 4}
+/// other: {13/1/2024: 5, 12/1/2024: 4}
+/// Expect: {15/1/2024: 7 ,14/1/2024: 6}, {}, {}
+void _diffExtraThisBegin() {
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+  });
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 15): const DbFilesSummaryItem(count: 7),
+    Date(2024, 1, 14): const DbFilesSummaryItem(count: 6),
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: {
+        Date(2024, 1, 15): const DbFilesSummaryItem(count: 7),
+        Date(2024, 1, 14): const DbFilesSummaryItem(count: 6),
+      },
+      onlyInOther: const {},
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with extra elements at the end of this list
+///
+/// this: {13/1/2024: 5, 12/1/2024: 4, 11/1/2024: 3, 10/1/2024: 2}
+/// other: {13/1/2024: 5, 12/1/2024: 4}
+/// Expect: {11/1/2024: 3, 10/1/2024: 2}, {}, {}
+void _diffExtraThisEnd() {
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+  });
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+    Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: {
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+        Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+      },
+      onlyInOther: const {},
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with extra elements in the middle of this list
+///
+/// this: {13/1/2024: 5, 12/1/2024: 4, 11/1/2024: 3, 10/1/2024: 2, 9/1/2024: 1}
+/// other: {13/1/2024: 5, 12/1/2024: 4, 9/1/2024: 1}
+/// Expect: {11/1/2024: 3, 10/1/2024: 2}, {}, {}
+void _diffExtraThisMid() {
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 9): const DbFilesSummaryItem(count: 1),
+  });
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+    Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+    Date(2024, 1, 9): const DbFilesSummaryItem(count: 1),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: {
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+        Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+      },
+      onlyInOther: const {},
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with other being empty
+///
+/// this: {13/1/2024: 5, 12/1/2024: 4, 11/1/2024: 3, 10/1/2024: 2, 9/1/2024: 1}
+/// other: {}
+/// Expect: {11/1/2024: 3, 10/1/2024: 2}, {}, {}
+void _diffOtherEmpty() {
+  const other = DbFilesSummary(items: {});
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: {
+        Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+        Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+      },
+      onlyInOther: const {},
+      updated: const {},
+    ),
+  );
+}
+
+/// Diff with no matches between this and other
+///
+/// this: {13/1/2024: 5, 11/1/2024: 3, 9/1/2024: 1}
+/// other: {12/1/2024: 4, 10/1/2024: 2}
+/// Expect: [2, 4], [1, 3, 5]
+void _diffNoMatches() {
+  final other = DbFilesSummary(items: {
+    Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+    Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+    Date(2024, 1, 9): const DbFilesSummaryItem(count: 1),
+  });
+  final obj = DbFilesSummary(items: {
+    Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+    Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+  });
+  expect(
+    obj.diff(other),
+    DbFilesSummaryDiff(
+      onlyInThis: {
+        Date(2024, 1, 12): const DbFilesSummaryItem(count: 4),
+        Date(2024, 1, 10): const DbFilesSummaryItem(count: 2),
+      },
+      onlyInOther: {
+        Date(2024, 1, 13): const DbFilesSummaryItem(count: 5),
+        Date(2024, 1, 11): const DbFilesSummaryItem(count: 3),
+        Date(2024, 1, 9): const DbFilesSummaryItem(count: 1),
+      },
+      updated: const {},
+    ),
+  );
+}