diff --git a/app/lib/help_utils.dart b/app/lib/help_utils.dart index 7325b91a..7e53e54a 100644 --- a/app/lib/help_utils.dart +++ b/app/lib/help_utils.dart @@ -8,3 +8,5 @@ const homeFolderNotFoundUrl = "https://gitlab.com/nkming2/nc-photos/-/wikis/help/home-folder-not-found"; const enhanceZeroDceUrl = "https://gitlab.com/nkming2/nc-photos/-/wikis/help/enhance/zero-dce"; +const enhanceDeepLabPortraitBlurUrl = + "https://gitlab.com/nkming2/nc-photos/-/wikis/help/enhance/portrait-blur-(deeplab)"; diff --git a/app/lib/l10n/app_en.arb b/app/lib/l10n/app_en.arb index 283fc0ec..2d4afcc2 100644 --- a/app/lib/l10n/app_en.arb +++ b/app/lib/l10n/app_en.arb @@ -1187,6 +1187,10 @@ "@deletePermanentlyLocalConfirmationDialogContent": { "description": "Make sure the user wants to delete the items from the current device" }, + "enhancePortraitBlurTitle": "Portrait blur", + "@enhancePortraitBlurTitle": { + "description": "Blur the background of a photo" + }, "errorUnauthenticated": "Unauthenticated access. Please sign-in again if the problem continues", "@errorUnauthenticated": { diff --git a/app/lib/l10n/untranslated-messages.txt b/app/lib/l10n/untranslated-messages.txt index f28e5354..debbf9c2 100644 --- a/app/lib/l10n/untranslated-messages.txt +++ b/app/lib/l10n/untranslated-messages.txt @@ -88,6 +88,7 @@ "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle", "errorAlbumDowngrade" ], @@ -194,6 +195,7 @@ "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle", "errorAlbumDowngrade" ], @@ -355,6 +357,7 @@ "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle", "errorAlbumDowngrade" ], @@ -366,14 +369,16 @@ "enhanceTooltip", "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", - "deletePermanentlyLocalConfirmationDialogContent" + "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle" ], "fi": [ "enhanceTooltip", "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", - "deletePermanentlyLocalConfirmationDialogContent" + "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle" ], "fr": [ @@ -384,7 +389,8 @@ "enhanceTooltip", "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", - "deletePermanentlyLocalConfirmationDialogContent" + "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle" ], "pl": [ @@ -412,20 +418,23 @@ "enhanceTooltip", "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", - "deletePermanentlyLocalConfirmationDialogContent" + "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle" ], "pt": [ "enhanceTooltip", "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", - "deletePermanentlyLocalConfirmationDialogContent" + "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle" ], "ru": [ "enhanceTooltip", "enhanceLowLightTitle", "collectionEnhancedPhotosLabel", - "deletePermanentlyLocalConfirmationDialogContent" + "deletePermanentlyLocalConfirmationDialogContent", + "enhancePortraitBlurTitle" ] } diff --git a/app/lib/widget/handler/enhance_handler.dart b/app/lib/widget/handler/enhance_handler.dart index 420ae6f5..40428944 100644 --- a/app/lib/widget/handler/enhance_handler.dart +++ b/app/lib/widget/handler/enhance_handler.dart @@ -72,6 +72,16 @@ class EnhanceHandler { }, ); break; + + case _Algorithm.deepLab3Portrait: + await ImageProcessor.deepLab3Portrait( + "${account.url}/${file.path}", + file.filename, + headers: { + "Authorization": Api.getAuthorizationHeaderValue(account), + }, + ); + break; } } @@ -105,6 +115,13 @@ class EnhanceHandler { link: enhanceZeroDceUrl, algorithm: _Algorithm.zeroDce, ), + if (platform_k.isAndroid) + _Option( + title: L10n.global().enhancePortraitBlurTitle, + subtitle: "DeepLap v3", + link: enhanceDeepLabPortraitBlurUrl, + algorithm: _Algorithm.deepLab3Portrait, + ), ]; final Account account; @@ -115,6 +132,7 @@ class EnhanceHandler { enum _Algorithm { zeroDce, + deepLab3Portrait, } class _Option { diff --git a/plugin/android/build.gradle b/plugin/android/build.gradle index ffc3918a..fd2a22a4 100644 --- a/plugin/android/build.gradle +++ b/plugin/android/build.gradle @@ -18,6 +18,7 @@ rootProject.allprojects { repositories { google() mavenCentral() + maven { url 'https://jitpack.io' } } } @@ -55,6 +56,7 @@ dependencies { implementation "androidx.annotation:annotation:1.3.0" implementation "androidx.core:core-ktx:1.7.0" implementation "androidx.exifinterface:exifinterface:1.3.3" + implementation 'com.github.android:renderscript-intrinsics-replacement-toolkit:b6363490c3' implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" implementation 'org.tensorflow:tensorflow-lite:2.8.0' } diff --git a/plugin/android/src/main/assets/lite-model_mobilenetv2-dm05-coco_dr_1.tflite b/plugin/android/src/main/assets/lite-model_mobilenetv2-dm05-coco_dr_1.tflite new file mode 100644 index 00000000..aab88a35 Binary files /dev/null and b/plugin/android/src/main/assets/lite-model_mobilenetv2-dm05-coco_dr_1.tflite differ diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt index 22348686..377accf8 100644 --- a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt @@ -30,6 +30,19 @@ class ImageProcessorChannelHandler(context: Context) : } } + "deepLab3Portrait" -> { + try { + deepLab3Portrait( + call.argument("fileUrl")!!, + call.argument("headers"), + call.argument("filename")!!, + result + ) + } catch (e: Throwable) { + result.error("systemException", e.toString(), null) + } + } + else -> result.notImplemented() } } @@ -45,12 +58,25 @@ class ImageProcessorChannelHandler(context: Context) : private fun zeroDce( fileUrl: String, headers: Map?, filename: String, result: MethodChannel.Result + ) = method( + fileUrl, headers, filename, ImageProcessorService.METHOD_ZERO_DCE, + result + ) + + private fun deepLab3Portrait( + fileUrl: String, headers: Map?, filename: String, + result: MethodChannel.Result + ) = method( + fileUrl, headers, filename, + ImageProcessorService.METHOD_DEEL_LAP_PORTRAIT, result + ) + + private fun method( + fileUrl: String, headers: Map?, filename: String, + method: String, result: MethodChannel.Result ) { val intent = Intent(context, ImageProcessorService::class.java).apply { - putExtra( - ImageProcessorService.EXTRA_METHOD, - ImageProcessorService.METHOD_ZERO_DCE - ) + putExtra(ImageProcessorService.EXTRA_METHOD, method) putExtra(ImageProcessorService.EXTRA_FILE_URL, fileUrl) putExtra( ImageProcessorService.EXTRA_HEADERS, diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt index 713e72da..7a4d6b30 100644 --- a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt @@ -18,6 +18,7 @@ import androidx.core.app.NotificationChannelCompat import androidx.core.app.NotificationCompat import androidx.core.app.NotificationManagerCompat import androidx.exifinterface.media.ExifInterface +import com.nkming.nc_photos.plugin.image_processor.DeepLab3Portrait import com.nkming.nc_photos.plugin.image_processor.ZeroDce import java.io.File import java.net.HttpURLConnection @@ -27,6 +28,7 @@ class ImageProcessorService : Service() { companion object { const val EXTRA_METHOD = "method" const val METHOD_ZERO_DCE = "zero-dce" + const val METHOD_DEEL_LAP_PORTRAIT = "DeepLab3Portrait" const val EXTRA_FILE_URL = "fileUrl" const val EXTRA_HEADERS = "headers" const val EXTRA_FILENAME = "filename" @@ -91,6 +93,9 @@ class ImageProcessorService : Service() { val method = intent.getStringExtra(EXTRA_METHOD) when (method) { METHOD_ZERO_DCE -> onZeroDce(startId, intent.extras!!) + METHOD_DEEL_LAP_PORTRAIT -> onDeepLapPortrait( + startId, intent.extras!! + ) else -> { Log.e(TAG, "Unknown method: $method") // we can't call stopSelf here as it'll stop the service even if @@ -100,7 +105,20 @@ class ImageProcessorService : Service() { } } - private fun onZeroDce(startId: Int, extras: Bundle) { + private fun onZeroDce(startId: Int, extras: Bundle) = + onMethod(startId, extras, METHOD_ZERO_DCE) + + private fun onDeepLapPortrait(startId: Int, extras: Bundle) = + onMethod(startId, extras, METHOD_DEEL_LAP_PORTRAIT) + + /** + * Handle methods without arguments + * + * @param startId + * @param extras + * @param method + */ + private fun onMethod(startId: Int, extras: Bundle, method: String) { val fileUrl = extras.getString(EXTRA_FILE_URL)!! @Suppress("Unchecked_cast") @@ -109,7 +127,7 @@ class ImageProcessorService : Service() { val filename = extras.getString(EXTRA_FILENAME)!! addCommand( ImageProcessorCommand( - startId, METHOD_ZERO_DCE, fileUrl, headers, filename + startId, method, fileUrl, headers, filename ) ) } @@ -396,6 +414,9 @@ private open class ImageProcessorCommandTask(context: Context) : ImageProcessorService.METHOD_ZERO_DCE -> ZeroDce(context).infer( fileUri ) + ImageProcessorService.METHOD_DEEL_LAP_PORTRAIT -> DeepLab3Portrait( + context + ).infer(fileUri) else -> throw IllegalArgumentException( "Unknown method: ${cmd.method}" ) diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/Util.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/Util.kt index fa72fd3e..99b189da 100644 --- a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/Util.kt +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/Util.kt @@ -22,3 +22,7 @@ inline fun HttpURLConnection.use(block: (HttpURLConnection) -> T): T { disconnect() } } + +inline fun ByteArray.transform(transform: (Byte) -> Byte) { + forEachIndexed{ i, v -> this[i] = transform(v) } +} diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/DeepLab3.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/DeepLab3.kt new file mode 100644 index 00000000..3ff8dd05 --- /dev/null +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/DeepLab3.kt @@ -0,0 +1,150 @@ +package com.nkming.nc_photos.plugin.image_processor + +import android.content.Context +import android.graphics.* +import android.net.Uri +import android.util.Log +import com.google.android.renderscript.Toolkit +import com.nkming.nc_photos.plugin.BitmapResizeMethod +import com.nkming.nc_photos.plugin.BitmapUtil +import com.nkming.nc_photos.plugin.transform +import org.tensorflow.lite.Interpreter +import java.io.File +import java.nio.ByteBuffer +import java.nio.FloatBuffer + +/** + * DeepLab is a state-of-art deep learning model for semantic image + * segmentation, where the goal is to assign semantic labels (e.g., person, dog, + * cat and so on) to every pixel in the input image + * + * See: https://github.com/tensorflow/models/tree/master/research/deeplab + */ +private class DeepLab3(context: Context) { + companion object { + private const val MODEL = "lite-model_mobilenetv2-dm05-coco_dr_1.tflite" + const val WIDTH = 513 + const val HEIGHT = 513 + + private const val TAG = "DeepLab3" + } + + enum class Label(val value: Int) { + BACKGROUND(0), + AEROPLANE(1), + BICYCLE(2), + BIRD(3), + BOAT(4), + BOTTLE(5), + BUS(6), + CAR(7), + CAT(8), + CHAIR(9), + COW(10), + DINING_TABLE(11), + DOG(12), + HORSE(13), + MOTORBIKE(14), + PERSON(15), + POTTED_PLANT(16), + SHEEP(17), + SOFA(18), + TRAIN(19), + TV(20), + } + + fun infer(imageUri: Uri): ByteBuffer { + val interpreter = + Interpreter(TfLiteHelper.loadModelFromAsset(context, MODEL)) + interpreter.allocateTensors() + + Log.i(TAG, "Converting bitmap to input") + val inputBitmap = + BitmapUtil.loadImageFixed(context, imageUri, WIDTH, HEIGHT) + val input = TfLiteHelper.bitmapToRgbFloatArray(inputBitmap) + val output = FloatBuffer.allocate(WIDTH * HEIGHT * Label.values().size) + Log.i(TAG, "Inferring") + interpreter.run(input, output) + return TfLiteHelper.argmax(output, WIDTH, HEIGHT, Label.values().size) + } + + private val context = context +} + +class DeepLab3Portrait(context: Context) { + companion object { + private const val RADIUS = 16 + private const val MAX_WIDTH = 2048 + private const val MAX_HEIGHT = 1536 + + private const val TAG = "DeepLab3Portrait" + } + + fun infer(imageUri: Uri): Bitmap { + val segmentMap = deepLab.infer(imageUri).also { + postProcessSegmentMap(it) + } + return enhance(imageUri, segmentMap, RADIUS) + } + + /** + * Post-process the segment map. + * + * The resulting segment map will: + * 1. Contain only the most significant label (the one with the most pixel) + * 2. The label value set to 255 + * 3. The background set to 0 + * + * @param segmentMap + */ + private fun postProcessSegmentMap(segmentMap: ByteBuffer) { + // keep only the largest segment + val count = mutableMapOf() + segmentMap.array().forEach { + if (it != DeepLab3.Label.BACKGROUND.value.toByte()) { + count[it] = (count[it] ?: 0) + 1 + } + } + val keep = count.maxByOrNull { it.value }?.key + segmentMap.array().transform { if (it == keep) 0xFF.toByte() else 0 } + } + + private fun enhance( + imageUri: Uri, segmentMap: ByteBuffer, radius: Int + ): Bitmap { + Log.i(TAG, "[enhance] Enhancing image") + // downscale original to prevent OOM + val orig = BitmapUtil.loadImage( + context, imageUri, MAX_WIDTH, MAX_HEIGHT, BitmapResizeMethod.FIT, + isAllowSwapSide = true, shouldUpscale = false + ) + val bg = Toolkit.blur(orig, radius) + + var alpha = Bitmap.createBitmap( + DeepLab3.WIDTH, DeepLab3.HEIGHT, Bitmap.Config.ALPHA_8 + ) + alpha.copyPixelsFromBuffer(segmentMap) + alpha = Bitmap.createScaledBitmap(alpha, orig.width, orig.height, true) + // blur the mask to smoothen the edge + alpha = Toolkit.blur(alpha, 16) + File(context.filesDir, "alpha.png").outputStream().use { + alpha.compress(Bitmap.CompressFormat.PNG, 50, it) + } + + val shader = ComposeShader( + BitmapShader(orig, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP), + BitmapShader(alpha, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP), + PorterDuff.Mode.DST_ATOP + ) + val paint = Paint().apply { + setShader(shader) + } + Canvas(bg).apply { + drawRect(0f, 0f, orig.width.toFloat(), orig.height.toFloat(), paint) + } + return bg + } + + private val context = context + private val deepLab = DeepLab3(context) +} diff --git a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/TfLiteHelper.kt b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/TfLiteHelper.kt index 15337768..7e46b5f2 100644 --- a/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/TfLiteHelper.kt +++ b/plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/TfLiteHelper.kt @@ -88,5 +88,20 @@ interface TfLiteHelper { outputBitmap.copyPixelsFromBuffer(buffer) return outputBitmap } + + fun argmax( + output: FloatBuffer, width: Int, height: Int, channel: Int + ): ByteBuffer { + val product = ByteBuffer.allocate(width * height) + val array = output.array() + var j = 0 + for (i in 0 until width * height) { + val pixel = array.slice(j until j + channel) + val max = pixel.indices.maxByOrNull { pixel[it] }!! + product.put(i, max.toByte()) + j += channel + } + return product + } } } diff --git a/plugin/lib/src/image_processor.dart b/plugin/lib/src/image_processor.dart index d2d69c6f..5385e459 100644 --- a/plugin/lib/src/image_processor.dart +++ b/plugin/lib/src/image_processor.dart @@ -15,6 +15,17 @@ class ImageProcessor { "filename": filename, }); + static Future deepLab3Portrait( + String fileUrl, + String filename, { + Map? headers, + }) => + _methodChannel.invokeMethod("deepLab3Portrait", { + "fileUrl": fileUrl, + "headers": headers, + "filename": filename, + }); + static const _methodChannel = MethodChannel("${k.libId}/image_processor_method"); }