Mirrors/nc-photos
1
0
Fork 0
mirror of https://gitlab.com/nkming2/nc-photos.git synced 2025-01-22 16:56:19 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add color pop enhancement

Browse source
This commit is contained in:
Ming Ming 2022-09-10 19:24:14 +08:00
parent 443e7c61ef
commit 348802efc5
12 changed files with 356 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
app/lib/help_utils.dart
View file

@ -8,4 +8,5 @@ const enhanceZeroDceUrl = "https://bit.ly/3wKJcm9";
const enhanceDeepLabPortraitBlurUrl = "https://bit.ly/3wIuXy6";
const enhanceEsrganUrl = "https://bit.ly/3wO0NJP";
const enhanceStyleTransferUrl = "https://bit.ly/3agpTcF";
const enhanceDeepLabColorPopUrl = "https://bit.ly/3Rx0YCD";
const editPhotosUrl = "https://bit.ly/3v82oKA";

8
app/lib/l10n/app_en.arb
View file

@ -1266,6 +1266,14 @@
"@enhanceStyleTransferStyleDialogTitle": {
"description": "Pick a reference image for the style transfer algorithm"
},
"enhanceColorPopTitle": "Color pop",
"@enhanceColorPopTitle": {
"description": "Desaturate the background of a photo"
},
"enhanceGenericParamWeightLabel": "Weight",
"@enhanceGenericParamWeightLabel": {
"description": "This generic parameter sets the weight of the applied effect. The effect will be more obvious when the weight is high."
},
"doubleTapExitNotification": "Tap again to exit",
"@doubleTapExitNotification": {
"description": "If double tap to exit is enabled in settings, shown when users tap the back button"

22
app/lib/l10n/untranslated-messages.txt
View file

@ -118,6 +118,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -293,6 +295,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -359,6 +363,8 @@
"shareMethodOriginalFileDescription",
"collectionEditedPhotosLabel",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -419,6 +425,8 @@
"shareMethodOriginalFileTitle",
"shareMethodOriginalFileDescription",
"collectionEditedPhotosLabel",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"imageEditToolbarColorLabel",
"imageEditToolbarTransformLabel",
"imageEditTransformOrientation",
@ -447,6 +455,8 @@
"shareMethodOriginalFileTitle",
"shareMethodOriginalFileDescription",
"collectionEditedPhotosLabel",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"imageEditToolbarColorLabel",
"imageEditToolbarTransformLabel",
"imageEditTransformOrientation",
@ -501,6 +511,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -603,6 +615,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -684,6 +698,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -765,6 +781,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -846,6 +864,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -927,6 +947,8 @@
"enhanceSuperResolution4xTitle",
"enhanceStyleTransferTitle",
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",

77
app/lib/widget/handler/enhance_handler.dart
View file

@ -115,6 +115,20 @@ class EnhanceHandler {
isSaveToServer: isSaveToServer,
);
break;
case _Algorithm.deepLab3ColorPop:
await ImageProcessor.deepLab3ColorPop(
"${account.url}/${file.path}",
file.filename,
Pref().getEnhanceMaxWidthOr(),
Pref().getEnhanceMaxHeightOr(),
args["weight"],
headers: {
"Authorization": Api.getAuthorizationHeaderValue(account),
},
isSaveToServer: isSaveToServer,
);
break;
}
}
@ -180,6 +194,13 @@ class EnhanceHandler {
);
List<_Option> _getOptions() => [
if (platform_k.isAndroid)
_Option(
title: L10n.global().enhanceColorPopTitle,
subtitle: "DeepLap v3",
link: enhanceDeepLabColorPopUrl,
algorithm: _Algorithm.deepLab3ColorPop,
),
if (platform_k.isAndroid)
_Option(
title: L10n.global().enhanceLowLightTitle,
@ -223,6 +244,9 @@ class EnhanceHandler {
case _Algorithm.arbitraryStyleTransfer:
return _getArbitraryStyleTransferArgs(context);
case _Algorithm.deepLab3ColorPop:
return _getDeepLab3ColorPopArgs(context);
}
}
@ -347,6 +371,58 @@ class EnhanceHandler {
}
}
Future<Map<String, dynamic>?> _getDeepLab3ColorPopArgs(
BuildContext context) async {
var current = 1.0;
final weight = await showDialog<double>(
context: context,
builder: (context) => AppTheme(
child: AlertDialog(
title: Text(L10n.global().enhanceGenericParamWeightLabel),
contentPadding: const EdgeInsets.fromLTRB(24.0, 20.0, 24.0, 0),
content: Column(
crossAxisAlignment: CrossAxisAlignment.start,
mainAxisSize: MainAxisSize.min,
children: [
Row(
mainAxisSize: MainAxisSize.max,
children: [
Icon(
Icons.water_drop,
size: 20,
color: AppTheme.getSecondaryTextColor(context),
),
Expanded(
child: StatefulSlider(
initialValue: current,
onChangeEnd: (value) {
current = value;
},
),
),
Icon(
Icons.water_drop_outlined,
color: AppTheme.getSecondaryTextColor(context),
),
],
),
],
),
actions: [
TextButton(
onPressed: () {
Navigator.of(context).pop(current);
},
child: Text(L10n.global().enhanceButtonLabel),
),
],
),
),
);
_log.info("[_getDeepLab3ColorPopArgs] weight: $weight");
return weight?.run((it) => {"weight": it});
}
bool isAtLeast4GbRam() {
// We can't compare with 4096 directly as some RAM are preserved
return AndroidInfo().totalMemMb > 3584;
@ -368,6 +444,7 @@ enum _Algorithm {
deepLab3Portrait,
esrgan,
arbitraryStyleTransfer,
deepLab3ColorPop,
}
class _Option {

161
plugin/android/src/main/cpp/deep_lap_3.cpp
View file

@ -13,6 +13,8 @@
#include <jni.h>
#include <tensorflow/lite/c/c_api.h>
#include "./filter/saturation.h"
using namespace plugin;
using namespace renderscript;
using namespace std;
@ -24,6 +26,7 @@ constexpr const char *MODEL = "tf/lite-model_mobilenetv2-dm05-coco_dr_1.tflite";
constexpr size_t WIDTH = 513;
constexpr size_t HEIGHT = 513;
constexpr unsigned LABEL_COUNT = 21;
constexpr const char *TAG = "deep_lap_3";
enum struct Label {
BACKGROUND = 0,
@ -72,18 +75,6 @@ public:
const size_t height, const unsigned radius);
private:
/**
* Post-process the segment map.
*
* The resulting segment map will:
* 1. Contain only the most significant label (the one with the most pixel)
* 2. The label value set to 255
* 3. The background set to 0
*
* @param segmentMap
*/
void postProcessSegmentMap(std::vector<uint8_t> *segmentMap);
std::vector<uint8_t> enhance(const uint8_t *image, const size_t width,
const size_t height,
const std::vector<uint8_t> &segmentMap,
@ -94,6 +85,36 @@ private:
static constexpr const char *TAG = "DeepLab3Portrait";
};
class DeepLab3ColorPop {
public:
explicit DeepLab3ColorPop(DeepLab3 &&deepLab);
std::vector<uint8_t> infer(const uint8_t *image, const size_t width,
const size_t height, const float weight);
private:
std::vector<uint8_t> enhance(const uint8_t *image, const size_t width,
const size_t height,
const std::vector<uint8_t> &segmentMap,
const float weight);
DeepLab3 deepLab;
static constexpr const char *TAG = "DeepLab3ColorPop";
};
/**
* Post-process the segment map.
*
* The resulting segment map will:
* 1. Contain only the most significant label (the one with the most pixel)
* 2. The label value set to 255
* 3. The background set to 0
*
* @param segmentMap
*/
void postProcessSegmentMap(std::vector<uint8_t> *segmentMap);
} // namespace
extern "C" JNIEXPORT jbyteArray JNICALL
@ -121,6 +142,31 @@ Java_com_nkming_nc_1photos_plugin_image_1processor_DeepLab3Portrait_inferNative(
}
}
extern "C" JNIEXPORT jbyteArray JNICALL
Java_com_nkming_nc_1photos_plugin_image_1processor_DeepLab3ColorPop_inferNative(
JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
jint width, jint height, jfloat weight) {
try {
initOpenMp();
auto aam = AAssetManager_fromJava(env, assetManager);
DeepLab3ColorPop model(DeepLab3{aam});
RaiiContainer<jbyte> cImage(
[&]() { return env->GetByteArrayElements(image, nullptr); },
[&](jbyte *obj) {
env->ReleaseByteArrayElements(image, obj, JNI_ABORT);
});
const auto result = model.infer(reinterpret_cast<uint8_t *>(cImage.get()),
width, height, weight);
auto resultAry = env->NewByteArray(result.size());
env->SetByteArrayRegion(resultAry, 0, result.size(),
reinterpret_cast<const int8_t *>(result.data()));
return resultAry;
} catch (const exception &e) {
throwJavaException(env, e.what());
return nullptr;
}
}
namespace {
DeepLab3::DeepLab3(AAssetManager *const aam) : model(Asset(aam, MODEL)) {}
@ -168,31 +214,6 @@ vector<uint8_t> DeepLab3Portrait::infer(const uint8_t *image,
return enhance(image, width, height, segmentMap, radius);
}
void DeepLab3Portrait::postProcessSegmentMap(vector<uint8_t> *segmentMap) {
// keep only the largest segment
vector<uint8_t> &segmentMapRef = *segmentMap;
vector<int> count(LABEL_COUNT);
for (size_t i = 0; i < segmentMapRef.size(); ++i) {
assert(segmentMapRef[i] < LABEL_COUNT);
const auto label = std::min<unsigned>(segmentMapRef[i], LABEL_COUNT);
if (label != static_cast<int>(Label::BACKGROUND)) {
++count[label];
}
}
const auto keep = distance(
count.data(), max_element(count.data(), count.data() + count.size()));
LOGI(TAG, "[postProcessSegmentMap] Label to keep: %d",
static_cast<int>(keep));
#pragma omp parallel for
for (size_t i = 0; i < segmentMapRef.size(); ++i) {
if (segmentMapRef[i] == keep) {
segmentMapRef[i] = 0xFF;
} else {
segmentMapRef[i] = 0;
}
}
}
vector<uint8_t> DeepLab3Portrait::enhance(const uint8_t *image,
const size_t width,
const size_t height,
@ -222,4 +243,70 @@ vector<uint8_t> DeepLab3Portrait::enhance(const uint8_t *image,
return rgba8ToRgb8(blur.data(), width, height);
}
DeepLab3ColorPop::DeepLab3ColorPop(DeepLab3 &&deepLab)
: deepLab(move(deepLab)) {}
vector<uint8_t> DeepLab3ColorPop::infer(const uint8_t *image,
const size_t width, const size_t height,
const float weight) {
auto segmentMap = deepLab.infer(image, width, height);
postProcessSegmentMap(&segmentMap);
return enhance(image, width, height, segmentMap, weight);
}
vector<uint8_t> DeepLab3ColorPop::enhance(const uint8_t *image,
const size_t width,
const size_t height,
const vector<uint8_t> &segmentMap,
const float weight) {
LOGI(TAG, "[enhance] Enhancing image");
// resize alpha to input size
vector<uint8_t> alpha(width * height);
base::ResampleImage<1>(segmentMap.data(), WIDTH, HEIGHT, alpha.data(), width,
height, base::KernelTypeLanczos3);
// smoothen the edge
vector<uint8_t> alphaFiltered(width * height);
getToolkitInst().blur(alpha.data(), alphaFiltered.data(), width, height, 1,
4);
alpha.clear();
// desaturate input
auto rgba8 = rgb8ToRgba8(image, width, height);
vector<uint8_t> desaturate(width * height * 4);
plugin::filter::Saturation saturation;
desaturate = saturation.apply(rgba8.data(), width, height, -1 * weight);
// draw input on top of blurred image, with alpha map
replaceChannel<4>(rgba8.data(), alphaFiltered.data(), width, height, 3);
alphaFiltered.clear();
alphaBlend(rgba8.data(), desaturate.data(), width, height);
rgba8.clear();
return rgba8ToRgb8(desaturate.data(), width, height);
}
void postProcessSegmentMap(vector<uint8_t> *segmentMap) {
// keep only the largest segment
vector<uint8_t> &segmentMapRef = *segmentMap;
vector<int> count(LABEL_COUNT);
for (size_t i = 0; i < segmentMapRef.size(); ++i) {
assert(segmentMapRef[i] < LABEL_COUNT);
const auto label = std::min<unsigned>(segmentMapRef[i], LABEL_COUNT);
if (label != static_cast<int>(Label::BACKGROUND)) {
++count[label];
}
}
const auto keep = distance(
count.data(), max_element(count.data(), count.data() + count.size()));
LOGI(TAG, "[postProcessSegmentMap] Label to keep: %d",
static_cast<int>(keep));
#pragma omp parallel for
for (size_t i = 0; i < segmentMapRef.size(); ++i) {
if (segmentMapRef[i] == keep) {
segmentMapRef[i] = 0xFF;
} else {
segmentMapRef[i] = 0;
}
}
}
} // namespace

5
plugin/android/src/main/cpp/deep_lap_3.h
View file

@ -11,6 +11,11 @@ Java_com_nkming_nc_1photos_plugin_image_1processor_DeepLab3Portrait_inferNative(
JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
jint width, jint height, jint radius);
JNIEXPORT jbyteArray JNICALL
Java_com_nkming_nc_1photos_plugin_image_1processor_DeepLab3ColorPop_inferNative(
JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
jint width, jint height, jfloat weight);
#ifdef __cplusplus
}
#endif

20
plugin/android/src/main/cpp/filter/saturation.cpp
View file

@ -11,23 +11,11 @@
#include "../math_util.h"
#include "../util.h"
#include "./hslhsv.h"
#include "./saturation.h"
using namespace plugin;
using namespace std;
namespace {
class Saturation {
public:
std::vector<uint8_t> apply(const uint8_t *rgba8, const size_t width,
const size_t height, const float weight);
private:
static constexpr const char *TAG = "Saturation";
};
} // namespace
extern "C" JNIEXPORT jbyteArray JNICALL
Java_com_nkming_nc_1photos_plugin_image_1processor_Saturation_applyNative(
JNIEnv *env, jobject *thiz, jbyteArray rgba8, jint width, jint height,
@ -39,7 +27,7 @@ Java_com_nkming_nc_1photos_plugin_image_1processor_Saturation_applyNative(
[&](jbyte *obj) {
env->ReleaseByteArrayElements(rgba8, obj, JNI_ABORT);
});
const auto result = Saturation().apply(
const auto result = filter::Saturation().apply(
reinterpret_cast<uint8_t *>(cRgba8.get()), width, height, value);
auto resultAry = env->NewByteArray(result.size());
env->SetByteArrayRegion(resultAry, 0, result.size(),
@ -51,7 +39,8 @@ Java_com_nkming_nc_1photos_plugin_image_1processor_Saturation_applyNative(
}
}
namespace {
namespace plugin {
namespace filter {
vector<uint8_t> Saturation::apply(const uint8_t *rgba8, const size_t width,
const size_t height, const float weight) {
@ -75,3 +64,4 @@ vector<uint8_t> Saturation::apply(const uint8_t *rgba8, const size_t width,
}
} // namespace
}

17
plugin/android/src/main/cpp/filter/saturation.h Normal file
View file

@ -0,0 +1,17 @@
#include <cstdint>
#include <vector>
namespace plugin {
namespace filter {
class Saturation {
public:
std::vector<uint8_t> apply(const uint8_t *rgba8, const size_t width,
const size_t height, const float weight);
private:
static constexpr const char *TAG = "Saturation";
};
} // namespace filter
} // namespace plugin

29
plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt
View file

@ -92,6 +92,24 @@ class ImageProcessorChannelHandler(context: Context) :
}
}
"deepLab3ColorPop" -> {
try {
deepLab3ColorPop(
call.argument("fileUrl")!!,
call.argument("headers"),
call.argument("filename")!!,
call.argument("maxWidth")!!,
call.argument("maxHeight")!!,
call.argument<Boolean>("isSaveToServer")!!,
call.argument("weight")!!,
result
)
} catch (e: Throwable) {
logE(TAG, "Uncaught exception", e)
result.error("systemException", e.toString(), null)
}
}
"filter" -> {
try {
filter(
@ -181,6 +199,17 @@ class ImageProcessorChannelHandler(context: Context) :
}
)
private fun deepLab3ColorPop(
fileUrl: String, headers: Map<String, String>?, filename: String,
maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean, weight: Float,
result: MethodChannel.Result
) = method(
fileUrl, headers, filename, maxWidth, maxHeight, isSaveToServer,
ImageProcessorService.METHOD_DEEP_LAP_COLOR_POP, result, onIntent = {
it.putExtra(ImageProcessorService.EXTRA_WEIGHT, weight)
}
)
private fun filter(
fileUrl: String, headers: Map<String, String>?, filename: String,
maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean,

17
plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt
View file

@ -31,6 +31,7 @@ class ImageProcessorService : Service() {
const val METHOD_DEEP_LAP_PORTRAIT = "DeepLab3Portrait"
const val METHOD_ESRGAN = "Esrgan"
const val METHOD_ARBITRARY_STYLE_TRANSFER = "ArbitraryStyleTransfer"
const val METHOD_DEEP_LAP_COLOR_POP = "DeepLab3ColorPop"
const val METHOD_FILTER = "Filter"
const val EXTRA_FILE_URL = "fileUrl"
const val EXTRA_HEADERS = "headers"
@ -49,6 +50,7 @@ class ImageProcessorService : Service() {
METHOD_DEEP_LAP_PORTRAIT,
METHOD_ESRGAN,
METHOD_ARBITRARY_STYLE_TRANSFER,
METHOD_DEEP_LAP_COLOR_POP,
)
val EDIT_METHODS = listOf(
METHOD_FILTER,
@ -128,6 +130,9 @@ class ImageProcessorService : Service() {
METHOD_ARBITRARY_STYLE_TRANSFER -> onArbitraryStyleTransfer(
startId, intent.extras!!
)
METHOD_DEEP_LAP_COLOR_POP -> onDeepLapColorPop(
startId, intent.extras!!
)
METHOD_FILTER -> onFilter(startId, intent.extras!!)
else -> {
logE(TAG, "Unknown method: $method")
@ -172,6 +177,14 @@ class ImageProcessorService : Service() {
)
}
private fun onDeepLapColorPop(startId: Int, extras: Bundle) {
return onMethod(
startId, extras, METHOD_DEEP_LAP_COLOR_POP, args = mapOf(
"weight" to extras.getFloat(EXTRA_WEIGHT)
)
)
}
private fun onFilter(startId: Int, extras: Bundle) {
val filters = extras.getSerializable(EXTRA_FILTERS)!!
.asType<ArrayList<Serializable>>()
@ -465,6 +478,10 @@ private class ImageProcessorEnhanceCommand(
args["weight"] as Float
).infer(fileUri)
ImageProcessorService.METHOD_DEEP_LAP_COLOR_POP -> DeepLab3ColorPop(
context, maxWidth, maxHeight, args["weight"] as Float
).infer(fileUri)
else -> throw IllegalArgumentException("Unknown method: $method")
}
}

32
plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/DeepLab3.kt
View file

@ -45,3 +45,35 @@ class DeepLab3Portrait(
private val maxHeight = maxHeight
private val radius = radius
}
class DeepLab3ColorPop(
context: Context, maxWidth: Int, maxHeight: Int, weight: Float
) {
fun infer(imageUri: Uri): Bitmap {
val width: Int
val height: Int
val rgb8Image = BitmapUtil.loadImage(
context, imageUri, maxWidth, maxHeight, BitmapResizeMethod.FIT,
isAllowSwapSide = true, shouldUpscale = false
).use {
width = it.width
height = it.height
TfLiteHelper.bitmapToRgb8Array(it)
}
val am = context.assets
return inferNative(am, rgb8Image, width, height, weight).let {
TfLiteHelper.rgb8ArrayToBitmap(it, width, height)
}
}
private external fun inferNative(
am: AssetManager, image: ByteArray, width: Int, height: Int,
weight: Float
): ByteArray
private val context = context
private val maxWidth = maxWidth
private val maxHeight = maxHeight
private val weight = weight
}

19
plugin/lib/src/image_processor.dart
View file

@ -151,6 +151,25 @@ class ImageProcessor {
"isSaveToServer": isSaveToServer,
});
static Future<void> deepLab3ColorPop(
String fileUrl,
String filename,
int maxWidth,
int maxHeight,
double weight, {
Map<String, String>? headers,
required bool isSaveToServer,
}) =>
_methodChannel.invokeMethod("deepLab3ColorPop", <String, dynamic>{
"fileUrl": fileUrl,
"headers": headers,
"filename": filename,
"maxWidth": maxWidth,
"maxHeight": maxHeight,
"weight": weight,
"isSaveToServer": isSaveToServer,
});
static Future<void> filter(
String fileUrl,
String filename,