diff --git a/app/lib/help_utils.dart b/app/lib/help_utils.dart index c48ee3a4..d921d460 100644 --- a/app/lib/help_utils.dart +++ b/app/lib/help_utils.dart @@ -9,4 +9,5 @@ const enhanceDeepLabPortraitBlurUrl = "https://bit.ly/3wIuXy6"; const enhanceEsrganUrl = "https://bit.ly/3wO0NJP"; const enhanceStyleTransferUrl = "https://bit.ly/3agpTcF"; const enhanceDeepLabColorPopUrl = "https://bit.ly/3Rx0YCD"; +const enhanceRetouchUrl = "https://bit.ly/3Ds2cea"; const editPhotosUrl = "https://bit.ly/3v82oKA"; diff --git a/app/lib/l10n/app_en.arb b/app/lib/l10n/app_en.arb index c145056c..1d7658cd 100644 --- a/app/lib/l10n/app_en.arb +++ b/app/lib/l10n/app_en.arb @@ -1274,6 +1274,10 @@ "@enhanceGenericParamWeightLabel": { "description": "This generic parameter sets the weight of the applied effect. The effect will be more obvious when the weight is high." }, + "enhanceRetouchTitle": "Auto retouch", + "@enhanceRetouchTitle": { + "description": "Automatically improve your photo" + }, "doubleTapExitNotification": "Tap again to exit", "@doubleTapExitNotification": { "description": "If double tap to exit is enabled in settings, shown when users tap the back button" diff --git a/app/lib/l10n/untranslated-messages.txt b/app/lib/l10n/untranslated-messages.txt index 7e924eab..d250685a 100644 --- a/app/lib/l10n/untranslated-messages.txt +++ b/app/lib/l10n/untranslated-messages.txt @@ -120,6 +120,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -301,6 +302,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -373,6 +375,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -439,6 +442,7 @@ "collectionEditedPhotosLabel", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "imageEditToolbarColorLabel", "imageEditToolbarTransformLabel", "imageEditTransformOrientation", @@ -473,6 +477,7 @@ "collectionEditedPhotosLabel", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "imageEditToolbarColorLabel", "imageEditToolbarTransformLabel", "imageEditTransformOrientation", @@ -533,6 +538,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -641,6 +647,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -728,6 +735,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -815,6 +823,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -902,6 +911,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", @@ -989,6 +999,7 @@ "enhanceStyleTransferStyleDialogTitle", "enhanceColorPopTitle", "enhanceGenericParamWeightLabel", + "enhanceRetouchTitle", "doubleTapExitNotification", "imageEditDiscardDialogTitle", "imageEditDiscardDialogContent", diff --git a/app/lib/widget/handler/enhance_handler.dart b/app/lib/widget/handler/enhance_handler.dart index 5e7edf48..6f40e17c 100644 --- a/app/lib/widget/handler/enhance_handler.dart +++ b/app/lib/widget/handler/enhance_handler.dart @@ -133,6 +133,19 @@ class EnhanceHandler { isSaveToServer: isSaveToServer, ); break; + + case _Algorithm.neurOp: + await ImageProcessor.neurOp( + "${account.url}/${file.path}", + file.filename, + Pref().getEnhanceMaxWidthOr(), + Pref().getEnhanceMaxHeightOr(), + headers: { + "Authorization": Api.getAuthorizationHeaderValue(account), + }, + isSaveToServer: isSaveToServer, + ); + break; } } @@ -207,6 +220,12 @@ class EnhanceHandler { ); List<_Option> _getOptions() => [ + if (platform_k.isAndroid) + _Option( + title: L10n.global().enhanceRetouchTitle, + link: enhanceRetouchUrl, + algorithm: _Algorithm.neurOp, + ), if (platform_k.isAndroid) _Option( title: L10n.global().enhanceColorPopTitle, @@ -260,6 +279,9 @@ class EnhanceHandler { case _Algorithm.deepLab3ColorPop: return _getDeepLab3ColorPopArgs(context); + + case _Algorithm.neurOp: + return {}; } } @@ -458,6 +480,7 @@ enum _Algorithm { esrgan, arbitraryStyleTransfer, deepLab3ColorPop, + neurOp, } class _Option { diff --git a/plugin/android/src/main/assets/tf/neurop_fivek_lite.tflite b/plugin/android/src/main/assets/tf/neurop_fivek_lite.tflite new file mode 100644 index 00000000..09d2b0ac Binary files /dev/null and b/plugin/android/src/main/assets/tf/neurop_fivek_lite.tflite differ diff --git a/plugin/android/src/main/cpp/CMakeLists.txt b/plugin/android/src/main/cpp/CMakeLists.txt index c1c6c6f6..834e8ded 100644 --- a/plugin/android/src/main/cpp/CMakeLists.txt +++ b/plugin/android/src/main/cpp/CMakeLists.txt @@ -50,6 +50,7 @@ add_library( # Sets the name of the library. esrgan.cpp exception.cpp image_splitter.cpp + neur_op.cpp stopwatch.cpp tflite_wrapper.cpp util.cpp diff --git a/plugin/android/src/main/cpp/neur_op.cpp b/plugin/android/src/main/cpp/neur_op.cpp new file mode 100644 index 00000000..52543c71 --- /dev/null +++ b/plugin/android/src/main/cpp/neur_op.cpp @@ -0,0 +1,95 @@ +#include "exception.h" +#include "log.h" +#include "stopwatch.h" +#include "tflite_wrapper.h" +#include "util.h" +#include +#include +#include +#include +#include +#include + +using namespace plugin; +using namespace std; +using namespace tflite; + +namespace { + +constexpr const char *MODEL = "tf/neurop_fivek_lite.tflite"; + +class NeurOp { +public: + explicit NeurOp(AAssetManager *const aam); + + std::vector infer(const uint8_t *image, const size_t width, + const size_t height); + +private: + Model model; + + static constexpr const char *TAG = "NeurOp"; +}; + +} // namespace + +extern "C" JNIEXPORT jbyteArray JNICALL +Java_com_nkming_nc_1photos_plugin_image_1processor_NeurOp_inferNative( + JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image, + jint width, jint height) { + try { + initOpenMp(); + auto aam = AAssetManager_fromJava(env, assetManager); + NeurOp model(aam); + RaiiContainer cImage( + [&]() { return env->GetByteArrayElements(image, nullptr); }, + [&](jbyte *obj) { + env->ReleaseByteArrayElements(image, obj, JNI_ABORT); + }); + const auto result = + model.infer(reinterpret_cast(cImage.get()), width, height); + auto resultAry = env->NewByteArray(result.size()); + env->SetByteArrayRegion(resultAry, 0, result.size(), + reinterpret_cast(result.data())); + return resultAry; + } catch (const exception &e) { + throwJavaException(env, e.what()); + return nullptr; + } +} + +namespace { + +NeurOp::NeurOp(AAssetManager *const aam) : model(Asset(aam, MODEL)) {} + +vector NeurOp::infer(const uint8_t *image, const size_t width, + const size_t height) { + InterpreterOptions options; + options.setNumThreads(getNumberOfProcessors()); + Interpreter interpreter(model, options); + const int dims[] = {1, static_cast(height), static_cast(width), 3}; + interpreter.resizeInputTensor(0, dims, 4); + interpreter.allocateTensors(); + + LOGI(TAG, "[infer] Convert bitmap to input"); + auto input = rgb8ToRgbFloat(image, width * height * 3, true); + auto inputTensor = interpreter.getInputTensor(0); + assert(TfLiteTensorByteSize(inputTensor) == input.size() * sizeof(float)); + TfLiteTensorCopyFromBuffer(inputTensor, input.data(), + input.size() * sizeof(float)); + input.clear(); + + LOGI(TAG, "[infer] Inferring"); + Stopwatch stopwatch; + interpreter.invoke(); + LOGI(TAG, "[infer] Elapsed: %.3fs", stopwatch.getMs() / 1000.0f); + + auto outputTensor = interpreter.getOutputTensor(0); + vector output(width * height * 3); + assert(TfLiteTensorByteSize(outputTensor) == output.size() * sizeof(float)); + TfLiteTensorCopyToBuffer(outputTensor, output.data(), + output.size() * sizeof(float)); + return rgbFloatToRgb8(output.data(), output.size(), true); +} + +} // namespace diff --git a/plugin/android/src/main/cpp/neur_op.h b/plugin/android/src/main/cpp/neur_op.h new file mode 100644 index 00000000..4c4c504e --- /dev/null +++ b/plugin/android/src/main/cpp/neur_op.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +JNIEXPORT jbyteArray JNICALL +Java_com_nkming_nc_1photos_plugin_image_1processor_NeurOp_inferNative( + JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image, + jint width, jint height); + +#ifdef __cplusplus +} +#endif diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt index 35fa34f5..93493434 100644 --- a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt @@ -110,6 +110,23 @@ class ImageProcessorChannelHandler(context: Context) : } } + "neurOp" -> { + try { + neurOp( + call.argument("fileUrl")!!, + call.argument("headers"), + call.argument("filename")!!, + call.argument("maxWidth")!!, + call.argument("maxHeight")!!, + call.argument("isSaveToServer")!!, + result + ) + } catch (e: Throwable) { + logE(TAG, "Uncaught exception", e) + result.error("systemException", e.toString(), null) + } + } + "filter" -> { try { filter( @@ -210,6 +227,15 @@ class ImageProcessorChannelHandler(context: Context) : } ) + private fun neurOp( + fileUrl: String, headers: Map?, filename: String, + maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean, + result: MethodChannel.Result + ) = method( + fileUrl, headers, filename, maxWidth, maxHeight, isSaveToServer, + ImageProcessorService.METHOD_NEUR_OP, result + ) + private fun filter( fileUrl: String, headers: Map?, filename: String, maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean, diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt index 739a8429..a836f875 100644 --- a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt @@ -32,6 +32,7 @@ class ImageProcessorService : Service() { const val METHOD_ESRGAN = "Esrgan" const val METHOD_ARBITRARY_STYLE_TRANSFER = "ArbitraryStyleTransfer" const val METHOD_DEEP_LAP_COLOR_POP = "DeepLab3ColorPop" + const val METHOD_NEUR_OP = "NeurOp" const val METHOD_FILTER = "Filter" const val EXTRA_FILE_URL = "fileUrl" const val EXTRA_HEADERS = "headers" @@ -122,6 +123,7 @@ class ImageProcessorService : Service() { METHOD_DEEP_LAP_COLOR_POP -> onDeepLapColorPop( startId, intent.extras!! ) + METHOD_NEUR_OP -> onNeurOp(startId, intent.extras!!) METHOD_FILTER -> onFilter(startId, intent.extras!!) else -> { logE(TAG, "Unknown method: $method") @@ -186,6 +188,12 @@ class ImageProcessorService : Service() { ) } + private fun onNeurOp(startId: Int, extras: Bundle) { + return onMethod( + startId, extras, { params -> ImageProcessorNeurOpCommand(params) }, + ) + } + private fun onFilter(startId: Int, extras: Bundle) { val filters = extras.getSerializable(EXTRA_FILTERS)!! .asType>() @@ -543,6 +551,16 @@ private class ImageProcessorDeepLapColorPopCommand( override fun isEnhanceCommand() = true } +private class ImageProcessorNeurOpCommand( + params: Params, +) : ImageProcessorImageCommand(params) { + override fun apply(context: Context, fileUri: Uri): Bitmap { + return NeurOp(context, maxWidth, maxHeight).infer(fileUri) + } + + override fun isEnhanceCommand() = true +} + private class ImageProcessorFilterCommand( params: Params, val filters: List, diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/NeurOp.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/NeurOp.kt new file mode 100644 index 00000000..cab579c6 --- /dev/null +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/NeurOp.kt @@ -0,0 +1,38 @@ +package com.nkming.nc_photos.plugin.image_processor + +import android.content.Context +import android.content.res.AssetManager +import android.graphics.Bitmap +import android.net.Uri +import com.nkming.nc_photos.plugin.BitmapResizeMethod +import com.nkming.nc_photos.plugin.BitmapUtil +import com.nkming.nc_photos.plugin.use + +class NeurOp(context: Context, maxWidth: Int, maxHeight: Int) { + fun infer(imageUri: Uri): Bitmap { + val width: Int + val height: Int + val rgb8Image = BitmapUtil.loadImage( + context, imageUri, maxWidth, maxHeight, BitmapResizeMethod.FIT, + isAllowSwapSide = true, shouldUpscale = false, + shouldFixOrientation = true + ).use { + width = it.width + height = it.height + TfLiteHelper.bitmapToRgb8Array(it) + } + val am = context.assets + + return inferNative(am, rgb8Image, width, height).let { + TfLiteHelper.rgb8ArrayToBitmap(it, width, height) + } + } + + private external fun inferNative( + am: AssetManager, image: ByteArray, width: Int, height: Int + ): ByteArray + + private val context = context + private val maxWidth = maxWidth + private val maxHeight = maxHeight +} diff --git a/plugin/lib/src/image_processor.dart b/plugin/lib/src/image_processor.dart index 1106b3c4..23722481 100644 --- a/plugin/lib/src/image_processor.dart +++ b/plugin/lib/src/image_processor.dart @@ -170,6 +170,23 @@ class ImageProcessor { "isSaveToServer": isSaveToServer, }); + static Future neurOp( + String fileUrl, + String filename, + int maxWidth, + int maxHeight, { + Map? headers, + required bool isSaveToServer, + }) => + _methodChannel.invokeMethod("neurOp", { + "fileUrl": fileUrl, + "headers": headers, + "filename": filename, + "maxWidth": maxWidth, + "maxHeight": maxHeight, + "isSaveToServer": isSaveToServer, + }); + static Future filter( String fileUrl, String filename,