Add auto retouch enhancement

app/lib/help_utils.dart

@@ -9,4 +9,5 @@ const enhanceDeepLabPortraitBlurUrl = "https://bit.ly/3wIuXy6";
 const enhanceEsrganUrl = "https://bit.ly/3wO0NJP";
 const enhanceStyleTransferUrl = "https://bit.ly/3agpTcF";
 const enhanceDeepLabColorPopUrl = "https://bit.ly/3Rx0YCD";
+const enhanceRetouchUrl = "https://bit.ly/3Ds2cea";
 const editPhotosUrl = "https://bit.ly/3v82oKA";

app/lib/l10n/app_en.arb

@@ -1274,6 +1274,10 @@
 	"@enhanceGenericParamWeightLabel": {
 		"description": "This generic parameter sets the weight of the applied effect. The effect will be more obvious when the weight is high."
 	},
+	"enhanceRetouchTitle": "Auto retouch",
+	"@enhanceRetouchTitle": {
+		"description": "Automatically improve your photo"
+	},
 	"doubleTapExitNotification": "Tap again to exit",
 	"@doubleTapExitNotification": {
 		"description": "If double tap to exit is enabled in settings, shown when users tap the back button"

app/lib/l10n/untranslated-messages.txt

@@ -120,6 +120,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -301,6 +302,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -373,6 +375,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -439,6 +442,7 @@
     "collectionEditedPhotosLabel",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "imageEditToolbarColorLabel",
     "imageEditToolbarTransformLabel",
     "imageEditTransformOrientation",
@@ -473,6 +477,7 @@
     "collectionEditedPhotosLabel",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "imageEditToolbarColorLabel",
     "imageEditToolbarTransformLabel",
     "imageEditTransformOrientation",
@@ -533,6 +538,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -641,6 +647,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -728,6 +735,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -815,6 +823,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -902,6 +911,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",
@@ -989,6 +999,7 @@
     "enhanceStyleTransferStyleDialogTitle",
     "enhanceColorPopTitle",
     "enhanceGenericParamWeightLabel",
+    "enhanceRetouchTitle",
     "doubleTapExitNotification",
     "imageEditDiscardDialogTitle",
     "imageEditDiscardDialogContent",

app/lib/widget/handler/enhance_handler.dart

@@ -133,6 +133,19 @@ class EnhanceHandler {
           isSaveToServer: isSaveToServer,
         );
         break;
+
+      case _Algorithm.neurOp:
+        await ImageProcessor.neurOp(
+          "${account.url}/${file.path}",
+          file.filename,
+          Pref().getEnhanceMaxWidthOr(),
+          Pref().getEnhanceMaxHeightOr(),
+          headers: {
+            "Authorization": Api.getAuthorizationHeaderValue(account),
+          },
+          isSaveToServer: isSaveToServer,
+        );
+        break;
     }
   }
 
@@ -207,6 +220,12 @@ class EnhanceHandler {
       );
 
   List<_Option> _getOptions() => [
+        if (platform_k.isAndroid)
+          _Option(
+            title: L10n.global().enhanceRetouchTitle,
+            link: enhanceRetouchUrl,
+            algorithm: _Algorithm.neurOp,
+          ),
         if (platform_k.isAndroid)
           _Option(
             title: L10n.global().enhanceColorPopTitle,
@@ -260,6 +279,9 @@ class EnhanceHandler {
 
       case _Algorithm.deepLab3ColorPop:
         return _getDeepLab3ColorPopArgs(context);
+
+      case _Algorithm.neurOp:
+        return {};
     }
   }
 
@@ -458,6 +480,7 @@ enum _Algorithm {
   esrgan,
   arbitraryStyleTransfer,
   deepLab3ColorPop,
+  neurOp,
 }
 
 class _Option {

plugin/android/src/main/cpp/CMakeLists.txt

@@ -50,6 +50,7 @@ add_library( # Sets the name of the library.
              esrgan.cpp
              exception.cpp
              image_splitter.cpp
+             neur_op.cpp
              stopwatch.cpp
              tflite_wrapper.cpp
              util.cpp

plugin/android/src/main/cpp/neur_op.cpp

@@ -0,0 +1,95 @@
+#include "exception.h"
+#include "log.h"
+#include "stopwatch.h"
+#include "tflite_wrapper.h"
+#include "util.h"
+#include <android/asset_manager.h>
+#include <android/asset_manager_jni.h>
+#include <cassert>
+#include <exception>
+#include <jni.h>
+#include <tensorflow/lite/c/c_api.h>
+
+using namespace plugin;
+using namespace std;
+using namespace tflite;
+
+namespace {
+
+constexpr const char *MODEL = "tf/neurop_fivek_lite.tflite";
+
+class NeurOp {
+public:
+  explicit NeurOp(AAssetManager *const aam);
+
+  std::vector<uint8_t> infer(const uint8_t *image, const size_t width,
+                             const size_t height);
+
+private:
+  Model model;
+
+  static constexpr const char *TAG = "NeurOp";
+};
+
+} // namespace
+
+extern "C" JNIEXPORT jbyteArray JNICALL
+Java_com_nkming_nc_1photos_plugin_image_1processor_NeurOp_inferNative(
+    JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
+    jint width, jint height) {
+  try {
+    initOpenMp();
+    auto aam = AAssetManager_fromJava(env, assetManager);
+    NeurOp model(aam);
+    RaiiContainer<jbyte> cImage(
+        [&]() { return env->GetByteArrayElements(image, nullptr); },
+        [&](jbyte *obj) {
+          env->ReleaseByteArrayElements(image, obj, JNI_ABORT);
+        });
+    const auto result =
+        model.infer(reinterpret_cast<uint8_t *>(cImage.get()), width, height);
+    auto resultAry = env->NewByteArray(result.size());
+    env->SetByteArrayRegion(resultAry, 0, result.size(),
+                            reinterpret_cast<const int8_t *>(result.data()));
+    return resultAry;
+  } catch (const exception &e) {
+    throwJavaException(env, e.what());
+    return nullptr;
+  }
+}
+
+namespace {
+
+NeurOp::NeurOp(AAssetManager *const aam) : model(Asset(aam, MODEL)) {}
+
+vector<uint8_t> NeurOp::infer(const uint8_t *image, const size_t width,
+                              const size_t height) {
+  InterpreterOptions options;
+  options.setNumThreads(getNumberOfProcessors());
+  Interpreter interpreter(model, options);
+  const int dims[] = {1, static_cast<int>(height), static_cast<int>(width), 3};
+  interpreter.resizeInputTensor(0, dims, 4);
+  interpreter.allocateTensors();
+
+  LOGI(TAG, "[infer] Convert bitmap to input");
+  auto input = rgb8ToRgbFloat(image, width * height * 3, true);
+  auto inputTensor = interpreter.getInputTensor(0);
+  assert(TfLiteTensorByteSize(inputTensor) == input.size() * sizeof(float));
+  TfLiteTensorCopyFromBuffer(inputTensor, input.data(),
+                             input.size() * sizeof(float));
+  input.clear();
+
+  LOGI(TAG, "[infer] Inferring");
+  Stopwatch stopwatch;
+  interpreter.invoke();
+  LOGI(TAG, "[infer] Elapsed: %.3fs", stopwatch.getMs() / 1000.0f);
+
+  auto outputTensor = interpreter.getOutputTensor(0);
+  vector<float> output(width * height * 3);
+  assert(TfLiteTensorByteSize(outputTensor) == output.size() * sizeof(float));
+  TfLiteTensorCopyToBuffer(outputTensor, output.data(),
+                           output.size() * sizeof(float));
+  return rgbFloatToRgb8(output.data(), output.size(), true);
+}
+
+} // namespace

plugin/android/src/main/cpp/neur_op.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include <jni.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+JNIEXPORT jbyteArray JNICALL
+Java_com_nkming_nc_1photos_plugin_image_1processor_NeurOp_inferNative(
+    JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
+    jint width, jint height);
+
+#ifdef __cplusplus
+}
+#endif

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt

@@ -110,6 +110,23 @@ class ImageProcessorChannelHandler(context: Context) :
 				}
 			}
 
+			"neurOp" -> {
+				try {
+					neurOp(
+						call.argument("fileUrl")!!,
+						call.argument("headers"),
+						call.argument("filename")!!,
+						call.argument("maxWidth")!!,
+						call.argument("maxHeight")!!,
+						call.argument<Boolean>("isSaveToServer")!!,
+						result
+					)
+				} catch (e: Throwable) {
+					logE(TAG, "Uncaught exception", e)
+					result.error("systemException", e.toString(), null)
+				}
+			}
+
 			"filter" -> {
 				try {
 					filter(
@@ -210,6 +227,15 @@ class ImageProcessorChannelHandler(context: Context) :
 		}
 	)
 
+	private fun neurOp(
+		fileUrl: String, headers: Map<String, String>?, filename: String,
+		maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean,
+		result: MethodChannel.Result
+	) = method(
+		fileUrl, headers, filename, maxWidth, maxHeight, isSaveToServer,
+		ImageProcessorService.METHOD_NEUR_OP, result
+	)
+
 	private fun filter(
 		fileUrl: String, headers: Map<String, String>?, filename: String,
 		maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean,

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt

@@ -32,6 +32,7 @@ class ImageProcessorService : Service() {
 		const val METHOD_ESRGAN = "Esrgan"
 		const val METHOD_ARBITRARY_STYLE_TRANSFER = "ArbitraryStyleTransfer"
 		const val METHOD_DEEP_LAP_COLOR_POP = "DeepLab3ColorPop"
+		const val METHOD_NEUR_OP = "NeurOp"
 		const val METHOD_FILTER = "Filter"
 		const val EXTRA_FILE_URL = "fileUrl"
 		const val EXTRA_HEADERS = "headers"
@@ -122,6 +123,7 @@ class ImageProcessorService : Service() {
 			METHOD_DEEP_LAP_COLOR_POP -> onDeepLapColorPop(
 				startId, intent.extras!!
 			)
+			METHOD_NEUR_OP -> onNeurOp(startId, intent.extras!!)
 			METHOD_FILTER -> onFilter(startId, intent.extras!!)
 			else -> {
 				logE(TAG, "Unknown method: $method")
@@ -186,6 +188,12 @@ class ImageProcessorService : Service() {
 		)
 	}
 
+	private fun onNeurOp(startId: Int, extras: Bundle) {
+		return onMethod(
+			startId, extras, { params -> ImageProcessorNeurOpCommand(params) },
+		)
+	}
+
 	private fun onFilter(startId: Int, extras: Bundle) {
 		val filters = extras.getSerializable(EXTRA_FILTERS)!!
 			.asType<ArrayList<Serializable>>()
@@ -543,6 +551,16 @@ private class ImageProcessorDeepLapColorPopCommand(
 	override fun isEnhanceCommand() = true
 }
 
+private class ImageProcessorNeurOpCommand(
+	params: Params,
+) : ImageProcessorImageCommand(params) {
+	override fun apply(context: Context, fileUri: Uri): Bitmap {
+		return NeurOp(context, maxWidth, maxHeight).infer(fileUri)
+	}
+
+	override fun isEnhanceCommand() = true
+}
+
 private class ImageProcessorFilterCommand(
 	params: Params,
 	val filters: List<ImageFilter>,

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/NeurOp.kt

@@ -0,0 +1,38 @@
+package com.nkming.nc_photos.plugin.image_processor
+
+import android.content.Context
+import android.content.res.AssetManager
+import android.graphics.Bitmap
+import android.net.Uri
+import com.nkming.nc_photos.plugin.BitmapResizeMethod
+import com.nkming.nc_photos.plugin.BitmapUtil
+import com.nkming.nc_photos.plugin.use
+
+class NeurOp(context: Context, maxWidth: Int, maxHeight: Int) {
+	fun infer(imageUri: Uri): Bitmap {
+		val width: Int
+		val height: Int
+		val rgb8Image = BitmapUtil.loadImage(
+			context, imageUri, maxWidth, maxHeight, BitmapResizeMethod.FIT,
+			isAllowSwapSide = true, shouldUpscale = false,
+			shouldFixOrientation = true
+		).use {
+			width = it.width
+			height = it.height
+			TfLiteHelper.bitmapToRgb8Array(it)
+		}
+		val am = context.assets
+
+		return inferNative(am, rgb8Image, width, height).let {
+			TfLiteHelper.rgb8ArrayToBitmap(it, width, height)
+		}
+	}
+
+	private external fun inferNative(
+		am: AssetManager, image: ByteArray, width: Int, height: Int
+	): ByteArray
+
+	private val context = context
+	private val maxWidth = maxWidth
+	private val maxHeight = maxHeight
+}

plugin/lib/src/image_processor.dart

@@ -170,6 +170,23 @@ class ImageProcessor {
         "isSaveToServer": isSaveToServer,
       });
 
+  static Future<void> neurOp(
+    String fileUrl,
+    String filename,
+    int maxWidth,
+    int maxHeight, {
+    Map<String, String>? headers,
+    required bool isSaveToServer,
+  }) =>
+      _methodChannel.invokeMethod("neurOp", <String, dynamic>{
+        "fileUrl": fileUrl,
+        "headers": headers,
+        "filename": filename,
+        "maxWidth": maxWidth,
+        "maxHeight": maxHeight,
+        "isSaveToServer": isSaveToServer,
+      });
+
   static Future<void> filter(
     String fileUrl,
     String filename,