mirror of
https://gitlab.com/nkming2/nc-photos.git
synced 2025-01-22 08:46:18 +01:00
Remove erratic metadata from heic files w/o exif
This commit is contained in:
parent
576d8dafc8
commit
c19003e5b9
4 changed files with 133 additions and 11 deletions
|
@ -124,6 +124,7 @@ class Metadata with EquatableMixin {
|
|||
JsonObj json, {
|
||||
required MetadataUpgraderV1? upgraderV1,
|
||||
required MetadataUpgraderV2? upgraderV2,
|
||||
required MetadataUpgraderV3? upgraderV3,
|
||||
}) {
|
||||
final jsonVersion = json["version"];
|
||||
JsonObj? result = json;
|
||||
|
@ -141,6 +142,13 @@ class Metadata with EquatableMixin {
|
|||
return null;
|
||||
}
|
||||
}
|
||||
if (jsonVersion < 4) {
|
||||
result = upgraderV3?.call(result);
|
||||
if (result == null) {
|
||||
_log.info("[fromJson] Version $jsonVersion not compatible");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return Metadata(
|
||||
lastUpdated: result["lastUpdated"] == null
|
||||
? null
|
||||
|
@ -219,7 +227,7 @@ class Metadata with EquatableMixin {
|
|||
final Exif? exif;
|
||||
|
||||
/// versioning of this class, use to upgrade old persisted metadata
|
||||
static const version = 3;
|
||||
static const version = 4;
|
||||
|
||||
static final _log = Logger("entity.file.Metadata");
|
||||
}
|
||||
|
@ -287,6 +295,35 @@ class MetadataUpgraderV2 implements MetadataUpgrader {
|
|||
static final _log = Logger("entity.file.MetadataUpgraderV2");
|
||||
}
|
||||
|
||||
/// Upgrade v3 Metadata to v4
|
||||
class MetadataUpgraderV3 implements MetadataUpgrader {
|
||||
const MetadataUpgraderV3({
|
||||
required this.fileContentType,
|
||||
this.logFilePath,
|
||||
});
|
||||
|
||||
@override
|
||||
JsonObj? call(JsonObj json) {
|
||||
if (fileContentType == "image/heic") {
|
||||
// Version 3 metadata for heic may incorrectly have exif as null due to a
|
||||
// bug in exifdart
|
||||
if (json["exif"] == null) {
|
||||
_log.fine("[call] Remove v3 metadata for file: $logFilePath");
|
||||
// return null to let the app parse the file again
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
final String? fileContentType;
|
||||
|
||||
/// File path for logging only
|
||||
final String? logFilePath;
|
||||
|
||||
static final _log = Logger("entity.file.MetadataUpgraderV3");
|
||||
}
|
||||
|
||||
class File with EquatableMixin implements FileDescriptor {
|
||||
File({
|
||||
required String path,
|
||||
|
@ -357,6 +394,10 @@ class File with EquatableMixin implements FileDescriptor {
|
|||
fileContentType: json["contentType"],
|
||||
logFilePath: json["path"],
|
||||
),
|
||||
upgraderV3: MetadataUpgraderV3(
|
||||
fileContentType: json["contentType"],
|
||||
logFilePath: json["path"],
|
||||
),
|
||||
),
|
||||
isArchived: json["isArchived"],
|
||||
overrideDateTime: json["overrideDateTime"] == null
|
||||
|
|
|
@ -379,6 +379,10 @@ class _FilePropParser {
|
|||
fileContentType: _contentType,
|
||||
logFilePath: logFilePath,
|
||||
),
|
||||
upgraderV3: MetadataUpgraderV3(
|
||||
fileContentType: _contentType,
|
||||
logFilePath: logFilePath,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -18,7 +18,8 @@ class CompatV55 {
|
|||
final count = await countQ.map((r) => r.read<int>(countExp)).getSingle();
|
||||
onProgress?.call(0, count);
|
||||
|
||||
final needUpdates = <Tuple2<int, DateTime>>[];
|
||||
final dateTimeUpdates = <Tuple2<int, DateTime>>[];
|
||||
final imageRemoves = <int>[];
|
||||
for (var i = 0; i < count; i += 1000) {
|
||||
final q = db.select(db.files).join([
|
||||
sql.innerJoin(
|
||||
|
@ -51,19 +52,28 @@ class CompatV55 {
|
|||
);
|
||||
if (f.accountFile.bestDateTime != bestDateTime) {
|
||||
// need update
|
||||
needUpdates.add(Tuple2(f.accountFile.rowId, bestDateTime));
|
||||
dateTimeUpdates.add(Tuple2(f.accountFile.rowId, bestDateTime));
|
||||
}
|
||||
|
||||
if (f.file.contentType == "image/heic" &&
|
||||
f.image != null &&
|
||||
f.image!.exifRaw == null) {
|
||||
imageRemoves.add(f.accountFile.rowId);
|
||||
}
|
||||
}
|
||||
onProgress?.call(i, count);
|
||||
}
|
||||
|
||||
_log.info("[migrateDb] ${needUpdates.length} rows require updating");
|
||||
_log.info(
|
||||
"[migrateDb] ${dateTimeUpdates.length} rows require updating, ${imageRemoves.length} rows require removing");
|
||||
if (kDebugMode) {
|
||||
_log.fine(
|
||||
"[migrateDb] ${needUpdates.map((e) => e.item1).toReadableString()}");
|
||||
"[migrateDb] dateTimeUpdates: ${dateTimeUpdates.map((e) => e.item1).toReadableString()}");
|
||||
_log.fine(
|
||||
"[migrateDb] imageRemoves: ${imageRemoves.map((e) => e).toReadableString()}");
|
||||
}
|
||||
await db.batch((batch) {
|
||||
for (final pair in needUpdates) {
|
||||
for (final pair in dateTimeUpdates) {
|
||||
batch.update(
|
||||
db.accountFiles,
|
||||
sql.AccountFilesCompanion(
|
||||
|
@ -73,6 +83,12 @@ class CompatV55 {
|
|||
table.rowId.equals(pair.item1),
|
||||
);
|
||||
}
|
||||
for (final r in imageRemoves) {
|
||||
batch.deleteWhere(
|
||||
db.images,
|
||||
(sql.$ImagesTable table) => table.accountFile.equals(r),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
@ -111,7 +111,9 @@ void main() {
|
|||
"version": Metadata.version,
|
||||
"lastUpdated": "2020-01-02T03:04:05.678901Z",
|
||||
};
|
||||
expect(Metadata.fromJson(json, upgraderV1: null, upgraderV2: null),
|
||||
expect(
|
||||
Metadata.fromJson(json,
|
||||
upgraderV1: null, upgraderV2: null, upgraderV3: null),
|
||||
Metadata(lastUpdated: DateTime.utc(2020, 1, 2, 3, 4, 5, 678, 901)));
|
||||
});
|
||||
|
||||
|
@ -122,7 +124,8 @@ void main() {
|
|||
"fileEtag": "8a3e0799b6f0711c23cc2d93950eceb5",
|
||||
};
|
||||
expect(
|
||||
Metadata.fromJson(json, upgraderV1: null, upgraderV2: null),
|
||||
Metadata.fromJson(json,
|
||||
upgraderV1: null, upgraderV2: null, upgraderV3: null),
|
||||
Metadata(
|
||||
lastUpdated: DateTime.utc(2020, 1, 2, 3, 4, 5, 678, 901),
|
||||
fileEtag: "8a3e0799b6f0711c23cc2d93950eceb5",
|
||||
|
@ -136,7 +139,8 @@ void main() {
|
|||
"imageWidth": 1024,
|
||||
};
|
||||
expect(
|
||||
Metadata.fromJson(json, upgraderV1: null, upgraderV2: null),
|
||||
Metadata.fromJson(json,
|
||||
upgraderV1: null, upgraderV2: null, upgraderV3: null),
|
||||
Metadata(
|
||||
lastUpdated: DateTime.utc(2020, 1, 2, 3, 4, 5, 678, 901),
|
||||
imageWidth: 1024,
|
||||
|
@ -150,7 +154,8 @@ void main() {
|
|||
"imageHeight": 768,
|
||||
};
|
||||
expect(
|
||||
Metadata.fromJson(json, upgraderV1: null, upgraderV2: null),
|
||||
Metadata.fromJson(json,
|
||||
upgraderV1: null, upgraderV2: null, upgraderV3: null),
|
||||
Metadata(
|
||||
lastUpdated: DateTime.utc(2020, 1, 2, 3, 4, 5, 678, 901),
|
||||
imageHeight: 768,
|
||||
|
@ -166,7 +171,8 @@ void main() {
|
|||
},
|
||||
};
|
||||
expect(
|
||||
Metadata.fromJson(json, upgraderV1: null, upgraderV2: null),
|
||||
Metadata.fromJson(json,
|
||||
upgraderV1: null, upgraderV2: null, upgraderV3: null),
|
||||
Metadata(
|
||||
lastUpdated: DateTime.utc(2020, 1, 2, 3, 4, 5, 678, 901),
|
||||
exif: Exif({
|
||||
|
@ -328,6 +334,61 @@ void main() {
|
|||
});
|
||||
});
|
||||
|
||||
group("MetadataUpgraderV3", () {
|
||||
test("exif null", () {
|
||||
final json = <String, dynamic>{
|
||||
"version": 3,
|
||||
"exif": null,
|
||||
"imageWidth": 1024,
|
||||
"imageHeight": 768,
|
||||
};
|
||||
expect(
|
||||
const MetadataUpgraderV3(fileContentType: "image/heic")(json),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("exif non-null", () {
|
||||
final json = <String, dynamic>{
|
||||
"version": 3,
|
||||
"exif": <String, dynamic>{
|
||||
"Make": "Amazing",
|
||||
},
|
||||
"imageWidth": 1024,
|
||||
"imageHeight": 768,
|
||||
};
|
||||
expect(
|
||||
const MetadataUpgraderV3(fileContentType: "image/heic")(json),
|
||||
<String, dynamic>{
|
||||
"version": 3,
|
||||
"exif": <String, dynamic>{
|
||||
"Make": "Amazing",
|
||||
},
|
||||
"imageWidth": 1024,
|
||||
"imageHeight": 768,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test("non-heic", () {
|
||||
final json = <String, dynamic>{
|
||||
"version": 3,
|
||||
"exif": null,
|
||||
"imageWidth": 1024,
|
||||
"imageHeight": 768,
|
||||
};
|
||||
expect(
|
||||
const MetadataUpgraderV3(fileContentType: "image/jpeg")(json),
|
||||
<String, dynamic>{
|
||||
"version": 3,
|
||||
"exif": null,
|
||||
"imageWidth": 1024,
|
||||
"imageHeight": 768,
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
group("File", () {
|
||||
group("constructor", () {
|
||||
test("path trim slash", () {
|
||||
|
|
Loading…
Reference in a new issue