Mirrors/nc-photos
1
0
Fork 0
mirror of https://gitlab.com/nkming2/nc-photos.git synced 2025-02-02 06:46:22 +01:00
Code Issues Projects Releases Packages Wiki Activity

Add auto retouch enhancement

Browse source
This commit is contained in:
Ming Ming 2022-09-15 12:59:52 +08:00
parent fee907c89e
commit a43aa4cbbd
12 changed files with 250 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
app/lib/help_utils.dart
View file

@ -9,4 +9,5 @@ const enhanceDeepLabPortraitBlurUrl = "https://bit.ly/3wIuXy6";
const enhanceEsrganUrl = "https://bit.ly/3wO0NJP";
const enhanceStyleTransferUrl = "https://bit.ly/3agpTcF";
const enhanceDeepLabColorPopUrl = "https://bit.ly/3Rx0YCD";
const enhanceRetouchUrl = "https://bit.ly/3Ds2cea";
const editPhotosUrl = "https://bit.ly/3v82oKA";

4
app/lib/l10n/app_en.arb
View file

@ -1274,6 +1274,10 @@
"@enhanceGenericParamWeightLabel": {
"description": "This generic parameter sets the weight of the applied effect. The effect will be more obvious when the weight is high."
},
"enhanceRetouchTitle": "Auto retouch",
"@enhanceRetouchTitle": {
"description": "Automatically improve your photo"
},
"doubleTapExitNotification": "Tap again to exit",
"@doubleTapExitNotification": {
"description": "If double tap to exit is enabled in settings, shown when users tap the back button"

11
app/lib/l10n/untranslated-messages.txt
View file

@ -120,6 +120,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -301,6 +302,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -373,6 +375,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -439,6 +442,7 @@
"collectionEditedPhotosLabel",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"imageEditToolbarColorLabel",
"imageEditToolbarTransformLabel",
"imageEditTransformOrientation",
@ -473,6 +477,7 @@
"collectionEditedPhotosLabel",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"imageEditToolbarColorLabel",
"imageEditToolbarTransformLabel",
"imageEditTransformOrientation",
@ -533,6 +538,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -641,6 +647,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -728,6 +735,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -815,6 +823,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -902,6 +911,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",
@ -989,6 +999,7 @@
"enhanceStyleTransferStyleDialogTitle",
"enhanceColorPopTitle",
"enhanceGenericParamWeightLabel",
"enhanceRetouchTitle",
"doubleTapExitNotification",
"imageEditDiscardDialogTitle",
"imageEditDiscardDialogContent",

23
app/lib/widget/handler/enhance_handler.dart
View file

@ -133,6 +133,19 @@ class EnhanceHandler {
isSaveToServer: isSaveToServer,
);
break;
case _Algorithm.neurOp:
await ImageProcessor.neurOp(
"${account.url}/${file.path}",
file.filename,
Pref().getEnhanceMaxWidthOr(),
Pref().getEnhanceMaxHeightOr(),
headers: {
"Authorization": Api.getAuthorizationHeaderValue(account),
},
isSaveToServer: isSaveToServer,
);
break;
}
}
@ -207,6 +220,12 @@ class EnhanceHandler {
);
List<_Option> _getOptions() => [
if (platform_k.isAndroid)
_Option(
title: L10n.global().enhanceRetouchTitle,
link: enhanceRetouchUrl,
algorithm: _Algorithm.neurOp,
),
if (platform_k.isAndroid)
_Option(
title: L10n.global().enhanceColorPopTitle,
@ -260,6 +279,9 @@ class EnhanceHandler {
case _Algorithm.deepLab3ColorPop:
return _getDeepLab3ColorPopArgs(context);
case _Algorithm.neurOp:
return {};
}
}
@ -458,6 +480,7 @@ enum _Algorithm {
esrgan,
arbitraryStyleTransfer,
deepLab3ColorPop,
neurOp,
}
class _Option {

BIN
plugin/android/src/main/assets/tf/neurop_fivek_lite.tflite Normal file
View file

Binary file not shown.

1
plugin/android/src/main/cpp/CMakeLists.txt
View file

@ -50,6 +50,7 @@ add_library( # Sets the name of the library.
esrgan.cpp
exception.cpp
image_splitter.cpp
neur_op.cpp
stopwatch.cpp
tflite_wrapper.cpp
util.cpp

95
plugin/android/src/main/cpp/neur_op.cpp Normal file
View file

@ -0,0 +1,95 @@
#include "exception.h"
#include "log.h"
#include "stopwatch.h"
#include "tflite_wrapper.h"
#include "util.h"
#include <android/asset_manager.h>
#include <android/asset_manager_jni.h>
#include <cassert>
#include <exception>
#include <jni.h>
#include <tensorflow/lite/c/c_api.h>
using namespace plugin;
using namespace std;
using namespace tflite;
namespace {
constexpr const char *MODEL = "tf/neurop_fivek_lite.tflite";
class NeurOp {
public:
explicit NeurOp(AAssetManager *const aam);
std::vector<uint8_t> infer(const uint8_t *image, const size_t width,
const size_t height);
private:
Model model;
static constexpr const char *TAG = "NeurOp";
};
} // namespace
extern "C" JNIEXPORT jbyteArray JNICALL
Java_com_nkming_nc_1photos_plugin_image_1processor_NeurOp_inferNative(
JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
jint width, jint height) {
try {
initOpenMp();
auto aam = AAssetManager_fromJava(env, assetManager);
NeurOp model(aam);
RaiiContainer<jbyte> cImage(
[&]() { return env->GetByteArrayElements(image, nullptr); },
[&](jbyte *obj) {
env->ReleaseByteArrayElements(image, obj, JNI_ABORT);
});
const auto result =
model.infer(reinterpret_cast<uint8_t *>(cImage.get()), width, height);
auto resultAry = env->NewByteArray(result.size());
env->SetByteArrayRegion(resultAry, 0, result.size(),
reinterpret_cast<const int8_t *>(result.data()));
return resultAry;
} catch (const exception &e) {
throwJavaException(env, e.what());
return nullptr;
}
}
namespace {
NeurOp::NeurOp(AAssetManager *const aam) : model(Asset(aam, MODEL)) {}
vector<uint8_t> NeurOp::infer(const uint8_t *image, const size_t width,
const size_t height) {
InterpreterOptions options;
options.setNumThreads(getNumberOfProcessors());
Interpreter interpreter(model, options);
const int dims[] = {1, static_cast<int>(height), static_cast<int>(width), 3};
interpreter.resizeInputTensor(0, dims, 4);
interpreter.allocateTensors();
LOGI(TAG, "[infer] Convert bitmap to input");
auto input = rgb8ToRgbFloat(image, width * height * 3, true);
auto inputTensor = interpreter.getInputTensor(0);
assert(TfLiteTensorByteSize(inputTensor) == input.size() * sizeof(float));
TfLiteTensorCopyFromBuffer(inputTensor, input.data(),
input.size() * sizeof(float));
input.clear();
LOGI(TAG, "[infer] Inferring");
Stopwatch stopwatch;
interpreter.invoke();
LOGI(TAG, "[infer] Elapsed: %.3fs", stopwatch.getMs() / 1000.0f);
auto outputTensor = interpreter.getOutputTensor(0);
vector<float> output(width * height * 3);
assert(TfLiteTensorByteSize(outputTensor) == output.size() * sizeof(float));
TfLiteTensorCopyToBuffer(outputTensor, output.data(),
output.size() * sizeof(float));
return rgbFloatToRgb8(output.data(), output.size(), true);
}
} // namespace

16
plugin/android/src/main/cpp/neur_op.h Normal file
View file

@ -0,0 +1,16 @@
#pragma once
#include <jni.h>
#ifdef __cplusplus
extern "C" {
#endif
JNIEXPORT jbyteArray JNICALL
Java_com_nkming_nc_1photos_plugin_image_1processor_NeurOp_inferNative(
JNIEnv *env, jobject *thiz, jobject assetManager, jbyteArray image,
jint width, jint height);
#ifdef __cplusplus
}
#endif

26
plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt
View file

@ -110,6 +110,23 @@ class ImageProcessorChannelHandler(context: Context) :
}
}
"neurOp" -> {
try {
neurOp(
call.argument("fileUrl")!!,
call.argument("headers"),
call.argument("filename")!!,
call.argument("maxWidth")!!,
call.argument("maxHeight")!!,
call.argument<Boolean>("isSaveToServer")!!,
result
)
} catch (e: Throwable) {
logE(TAG, "Uncaught exception", e)
result.error("systemException", e.toString(), null)
}
}
"filter" -> {
try {
filter(
@ -210,6 +227,15 @@ class ImageProcessorChannelHandler(context: Context) :
}
)
private fun neurOp(
fileUrl: String, headers: Map<String, String>?, filename: String,
maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean,
result: MethodChannel.Result
) = method(
fileUrl, headers, filename, maxWidth, maxHeight, isSaveToServer,
ImageProcessorService.METHOD_NEUR_OP, result
)
private fun filter(
fileUrl: String, headers: Map<String, String>?, filename: String,
maxWidth: Int, maxHeight: Int, isSaveToServer: Boolean,

18
plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt
View file

@ -32,6 +32,7 @@ class ImageProcessorService : Service() {
const val METHOD_ESRGAN = "Esrgan"
const val METHOD_ARBITRARY_STYLE_TRANSFER = "ArbitraryStyleTransfer"
const val METHOD_DEEP_LAP_COLOR_POP = "DeepLab3ColorPop"
const val METHOD_NEUR_OP = "NeurOp"
const val METHOD_FILTER = "Filter"
const val EXTRA_FILE_URL = "fileUrl"
const val EXTRA_HEADERS = "headers"
@ -122,6 +123,7 @@ class ImageProcessorService : Service() {
METHOD_DEEP_LAP_COLOR_POP -> onDeepLapColorPop(
startId, intent.extras!!
)
METHOD_NEUR_OP -> onNeurOp(startId, intent.extras!!)
METHOD_FILTER -> onFilter(startId, intent.extras!!)
else -> {
logE(TAG, "Unknown method: $method")
@ -186,6 +188,12 @@ class ImageProcessorService : Service() {
)
}
private fun onNeurOp(startId: Int, extras: Bundle) {
return onMethod(
startId, extras, { params -> ImageProcessorNeurOpCommand(params) },
)
}
private fun onFilter(startId: Int, extras: Bundle) {
val filters = extras.getSerializable(EXTRA_FILTERS)!!
.asType<ArrayList<Serializable>>()
@ -543,6 +551,16 @@ private class ImageProcessorDeepLapColorPopCommand(
override fun isEnhanceCommand() = true
}
private class ImageProcessorNeurOpCommand(
params: Params,
) : ImageProcessorImageCommand(params) {
override fun apply(context: Context, fileUri: Uri): Bitmap {
return NeurOp(context, maxWidth, maxHeight).infer(fileUri)
}
override fun isEnhanceCommand() = true
}
private class ImageProcessorFilterCommand(
params: Params,
val filters: List<ImageFilter>,

38
plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/NeurOp.kt Normal file
View file

@ -0,0 +1,38 @@
package com.nkming.nc_photos.plugin.image_processor
import android.content.Context
import android.content.res.AssetManager
import android.graphics.Bitmap
import android.net.Uri
import com.nkming.nc_photos.plugin.BitmapResizeMethod
import com.nkming.nc_photos.plugin.BitmapUtil
import com.nkming.nc_photos.plugin.use
class NeurOp(context: Context, maxWidth: Int, maxHeight: Int) {
fun infer(imageUri: Uri): Bitmap {
val width: Int
val height: Int
val rgb8Image = BitmapUtil.loadImage(
context, imageUri, maxWidth, maxHeight, BitmapResizeMethod.FIT,
isAllowSwapSide = true, shouldUpscale = false,
shouldFixOrientation = true
).use {
width = it.width
height = it.height
TfLiteHelper.bitmapToRgb8Array(it)
}
val am = context.assets
return inferNative(am, rgb8Image, width, height).let {
TfLiteHelper.rgb8ArrayToBitmap(it, width, height)
}
}
private external fun inferNative(
am: AssetManager, image: ByteArray, width: Int, height: Int
): ByteArray
private val context = context
private val maxWidth = maxWidth
private val maxHeight = maxHeight
}

17
plugin/lib/src/image_processor.dart
View file

@ -170,6 +170,23 @@ class ImageProcessor {
"isSaveToServer": isSaveToServer,
});
static Future<void> neurOp(
String fileUrl,
String filename,
int maxWidth,
int maxHeight, {
Map<String, String>? headers,
required bool isSaveToServer,
}) =>
_methodChannel.invokeMethod("neurOp", <String, dynamic>{
"fileUrl": fileUrl,
"headers": headers,
"filename": filename,
"maxWidth": maxWidth,
"maxHeight": maxHeight,
"isSaveToServer": isSaveToServer,
});
static Future<void> filter(
String fileUrl,
String filename,