Add enhancement: portrait blur with deeplab

app/lib/help_utils.dart

@@ -8,3 +8,5 @@ const homeFolderNotFoundUrl =
     "https://gitlab.com/nkming2/nc-photos/-/wikis/help/home-folder-not-found";
 const enhanceZeroDceUrl =
     "https://gitlab.com/nkming2/nc-photos/-/wikis/help/enhance/zero-dce";
+const enhanceDeepLabPortraitBlurUrl =
+    "https://gitlab.com/nkming2/nc-photos/-/wikis/help/enhance/portrait-blur-(deeplab)";

app/lib/l10n/app_en.arb

@@ -1187,6 +1187,10 @@
 	"@deletePermanentlyLocalConfirmationDialogContent": {
 		"description": "Make sure the user wants to delete the items from the current device"
 	},
+	"enhancePortraitBlurTitle": "Portrait blur",
+	"@enhancePortraitBlurTitle": {
+		"description": "Blur the background of a photo"
+	},
 
 	"errorUnauthenticated": "Unauthenticated access. Please sign-in again if the problem continues",
 	"@errorUnauthenticated": {

app/lib/l10n/untranslated-messages.txt

@@ -88,6 +88,7 @@
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
     "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle",
     "errorAlbumDowngrade"
   ],
 
@@ -194,6 +195,7 @@
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
     "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle",
     "errorAlbumDowngrade"
   ],
 
@@ -355,6 +357,7 @@
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
     "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle",
     "errorAlbumDowngrade"
   ],
 
@@ -366,14 +369,16 @@
     "enhanceTooltip",
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
-    "deletePermanentlyLocalConfirmationDialogContent"
+    "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle"
   ],
 
   "fi": [
     "enhanceTooltip",
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
-    "deletePermanentlyLocalConfirmationDialogContent"
+    "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle"
   ],
 
   "fr": [
@@ -384,7 +389,8 @@
     "enhanceTooltip",
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
-    "deletePermanentlyLocalConfirmationDialogContent"
+    "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle"
   ],
 
   "pl": [
@@ -412,20 +418,23 @@
     "enhanceTooltip",
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
-    "deletePermanentlyLocalConfirmationDialogContent"
+    "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle"
   ],
 
   "pt": [
     "enhanceTooltip",
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
-    "deletePermanentlyLocalConfirmationDialogContent"
+    "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle"
   ],
 
   "ru": [
     "enhanceTooltip",
     "enhanceLowLightTitle",
     "collectionEnhancedPhotosLabel",
-    "deletePermanentlyLocalConfirmationDialogContent"
+    "deletePermanentlyLocalConfirmationDialogContent",
+    "enhancePortraitBlurTitle"
   ]
 }

app/lib/widget/handler/enhance_handler.dart

@@ -72,6 +72,16 @@ class EnhanceHandler {
           },
         );
         break;
+
+      case _Algorithm.deepLab3Portrait:
+        await ImageProcessor.deepLab3Portrait(
+          "${account.url}/${file.path}",
+          file.filename,
+          headers: {
+            "Authorization": Api.getAuthorizationHeaderValue(account),
+          },
+        );
+        break;
     }
   }
 
@@ -105,6 +115,13 @@ class EnhanceHandler {
             link: enhanceZeroDceUrl,
             algorithm: _Algorithm.zeroDce,
           ),
+        if (platform_k.isAndroid)
+          _Option(
+            title: L10n.global().enhancePortraitBlurTitle,
+            subtitle: "DeepLap v3",
+            link: enhanceDeepLabPortraitBlurUrl,
+            algorithm: _Algorithm.deepLab3Portrait,
+          ),
       ];
 
   final Account account;
@@ -115,6 +132,7 @@ class EnhanceHandler {
 
 enum _Algorithm {
   zeroDce,
+  deepLab3Portrait,
 }
 
 class _Option {

plugin/android/build.gradle

@@ -18,6 +18,7 @@ rootProject.allprojects {
     repositories {
         google()
         mavenCentral()
+        maven { url 'https://jitpack.io' }
     }
 }
 
@@ -55,6 +56,7 @@ dependencies {
     implementation "androidx.annotation:annotation:1.3.0"
     implementation "androidx.core:core-ktx:1.7.0"
     implementation "androidx.exifinterface:exifinterface:1.3.3"
+    implementation 'com.github.android:renderscript-intrinsics-replacement-toolkit:b6363490c3'
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
     implementation 'org.tensorflow:tensorflow-lite:2.8.0'
 }

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorChannelHandler.kt

@@ -30,6 +30,19 @@ class ImageProcessorChannelHandler(context: Context) :
 				}
 			}
 
+			"deepLab3Portrait" -> {
+				try {
+					deepLab3Portrait(
+						call.argument("fileUrl")!!,
+						call.argument("headers"),
+						call.argument("filename")!!,
+						result
+					)
+				} catch (e: Throwable) {
+					result.error("systemException", e.toString(), null)
+				}
+			}
+
 			else -> result.notImplemented()
 		}
 	}
@@ -45,12 +58,25 @@ class ImageProcessorChannelHandler(context: Context) :
 	private fun zeroDce(
 		fileUrl: String, headers: Map<String, String>?, filename: String,
 		result: MethodChannel.Result
+	) = method(
+		fileUrl, headers, filename, ImageProcessorService.METHOD_ZERO_DCE,
+		result
+	)
+
+	private fun deepLab3Portrait(
+		fileUrl: String, headers: Map<String, String>?, filename: String,
+		result: MethodChannel.Result
+	) = method(
+		fileUrl, headers, filename,
+		ImageProcessorService.METHOD_DEEL_LAP_PORTRAIT, result
+	)
+
+	private fun method(
+		fileUrl: String, headers: Map<String, String>?, filename: String,
+		method: String, result: MethodChannel.Result
 	) {
 		val intent = Intent(context, ImageProcessorService::class.java).apply {
-			putExtra(
-				ImageProcessorService.EXTRA_METHOD,
-				ImageProcessorService.METHOD_ZERO_DCE
-			)
+			putExtra(ImageProcessorService.EXTRA_METHOD, method)
 			putExtra(ImageProcessorService.EXTRA_FILE_URL, fileUrl)
 			putExtra(
 				ImageProcessorService.EXTRA_HEADERS,

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/ImageProcessorService.kt

@@ -18,6 +18,7 @@ import androidx.core.app.NotificationChannelCompat
 import androidx.core.app.NotificationCompat
 import androidx.core.app.NotificationManagerCompat
 import androidx.exifinterface.media.ExifInterface
+import com.nkming.nc_photos.plugin.image_processor.DeepLab3Portrait
 import com.nkming.nc_photos.plugin.image_processor.ZeroDce
 import java.io.File
 import java.net.HttpURLConnection
@@ -27,6 +28,7 @@ class ImageProcessorService : Service() {
 	companion object {
 		const val EXTRA_METHOD = "method"
 		const val METHOD_ZERO_DCE = "zero-dce"
+		const val METHOD_DEEL_LAP_PORTRAIT = "DeepLab3Portrait"
 		const val EXTRA_FILE_URL = "fileUrl"
 		const val EXTRA_HEADERS = "headers"
 		const val EXTRA_FILENAME = "filename"
@@ -91,6 +93,9 @@ class ImageProcessorService : Service() {
 		val method = intent.getStringExtra(EXTRA_METHOD)
 		when (method) {
 			METHOD_ZERO_DCE -> onZeroDce(startId, intent.extras!!)
+			METHOD_DEEL_LAP_PORTRAIT -> onDeepLapPortrait(
+				startId, intent.extras!!
+			)
 			else -> {
 				Log.e(TAG, "Unknown method: $method")
 				// we can't call stopSelf here as it'll stop the service even if
@@ -100,7 +105,20 @@ class ImageProcessorService : Service() {
 		}
 	}
 
-	private fun onZeroDce(startId: Int, extras: Bundle) {
+	private fun onZeroDce(startId: Int, extras: Bundle) =
+		onMethod(startId, extras, METHOD_ZERO_DCE)
+
+	private fun onDeepLapPortrait(startId: Int, extras: Bundle) =
+		onMethod(startId, extras, METHOD_DEEL_LAP_PORTRAIT)
+
+	/**
+	 * Handle methods without arguments
+	 *
+	 * @param startId
+	 * @param extras
+	 * @param method
+	 */
+	private fun onMethod(startId: Int, extras: Bundle, method: String) {
 		val fileUrl = extras.getString(EXTRA_FILE_URL)!!
 
 		@Suppress("Unchecked_cast")
@@ -109,7 +127,7 @@ class ImageProcessorService : Service() {
 		val filename = extras.getString(EXTRA_FILENAME)!!
 		addCommand(
 			ImageProcessorCommand(
-				startId, METHOD_ZERO_DCE, fileUrl, headers, filename
+				startId, method, fileUrl, headers, filename
 			)
 		)
 	}
@@ -396,6 +414,9 @@ private open class ImageProcessorCommandTask(context: Context) :
 				ImageProcessorService.METHOD_ZERO_DCE -> ZeroDce(context).infer(
 					fileUri
 				)
+				ImageProcessorService.METHOD_DEEL_LAP_PORTRAIT -> DeepLab3Portrait(
+					context
+				).infer(fileUri)
 				else -> throw IllegalArgumentException(
 					"Unknown method: ${cmd.method}"
 				)

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/Util.kt

@@ -22,3 +22,7 @@ inline fun <T> HttpURLConnection.use(block: (HttpURLConnection) -> T): T {
 		disconnect()
 	}
 }
+
+inline fun ByteArray.transform(transform: (Byte) -> Byte) {
+	forEachIndexed{ i, v -> this[i] = transform(v) }
+}

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/DeepLab3.kt

@@ -0,0 +1,150 @@
+package com.nkming.nc_photos.plugin.image_processor
+
+import android.content.Context
+import android.graphics.*
+import android.net.Uri
+import android.util.Log
+import com.google.android.renderscript.Toolkit
+import com.nkming.nc_photos.plugin.BitmapResizeMethod
+import com.nkming.nc_photos.plugin.BitmapUtil
+import com.nkming.nc_photos.plugin.transform
+import org.tensorflow.lite.Interpreter
+import java.io.File
+import java.nio.ByteBuffer
+import java.nio.FloatBuffer
+
+/**
+ * DeepLab is a state-of-art deep learning model for semantic image
+ * segmentation, where the goal is to assign semantic labels (e.g., person, dog,
+ * cat and so on) to every pixel in the input image
+ *
+ * See: https://github.com/tensorflow/models/tree/master/research/deeplab
+ */
+private class DeepLab3(context: Context) {
+	companion object {
+		private const val MODEL = "lite-model_mobilenetv2-dm05-coco_dr_1.tflite"
+		const val WIDTH = 513
+		const val HEIGHT = 513
+
+		private const val TAG = "DeepLab3"
+	}
+
+	enum class Label(val value: Int) {
+		BACKGROUND(0),
+		AEROPLANE(1),
+		BICYCLE(2),
+		BIRD(3),
+		BOAT(4),
+		BOTTLE(5),
+		BUS(6),
+		CAR(7),
+		CAT(8),
+		CHAIR(9),
+		COW(10),
+		DINING_TABLE(11),
+		DOG(12),
+		HORSE(13),
+		MOTORBIKE(14),
+		PERSON(15),
+		POTTED_PLANT(16),
+		SHEEP(17),
+		SOFA(18),
+		TRAIN(19),
+		TV(20),
+	}
+
+	fun infer(imageUri: Uri): ByteBuffer {
+		val interpreter =
+			Interpreter(TfLiteHelper.loadModelFromAsset(context, MODEL))
+		interpreter.allocateTensors()
+
+		Log.i(TAG, "Converting bitmap to input")
+		val inputBitmap =
+			BitmapUtil.loadImageFixed(context, imageUri, WIDTH, HEIGHT)
+		val input = TfLiteHelper.bitmapToRgbFloatArray(inputBitmap)
+		val output = FloatBuffer.allocate(WIDTH * HEIGHT * Label.values().size)
+		Log.i(TAG, "Inferring")
+		interpreter.run(input, output)
+		return TfLiteHelper.argmax(output, WIDTH, HEIGHT, Label.values().size)
+	}
+
+	private val context = context
+}
+
+class DeepLab3Portrait(context: Context) {
+	companion object {
+		private const val RADIUS = 16
+		private const val MAX_WIDTH = 2048
+		private const val MAX_HEIGHT = 1536
+
+		private const val TAG = "DeepLab3Portrait"
+	}
+
+	fun infer(imageUri: Uri): Bitmap {
+		val segmentMap = deepLab.infer(imageUri).also {
+			postProcessSegmentMap(it)
+		}
+		return enhance(imageUri, segmentMap, RADIUS)
+	}
+
+	/**
+	 * Post-process the segment map.
+	 *
+	 * The resulting segment map will:
+	 * 1. Contain only the most significant label (the one with the most pixel)
+	 * 2. The label value set to 255
+	 * 3. The background set to 0
+	 *
+	 * @param segmentMap
+	 */
+	private fun postProcessSegmentMap(segmentMap: ByteBuffer) {
+		// keep only the largest segment
+		val count = mutableMapOf<Byte, Int>()
+		segmentMap.array().forEach {
+			if (it != DeepLab3.Label.BACKGROUND.value.toByte()) {
+				count[it] = (count[it] ?: 0) + 1
+			}
+		}
+		val keep = count.maxByOrNull { it.value }?.key
+		segmentMap.array().transform { if (it == keep) 0xFF.toByte() else 0 }
+	}
+
+	private fun enhance(
+		imageUri: Uri, segmentMap: ByteBuffer, radius: Int
+	): Bitmap {
+		Log.i(TAG, "[enhance] Enhancing image")
+		// downscale original to prevent OOM
+		val orig = BitmapUtil.loadImage(
+			context, imageUri, MAX_WIDTH, MAX_HEIGHT, BitmapResizeMethod.FIT,
+			isAllowSwapSide = true, shouldUpscale = false
+		)
+		val bg = Toolkit.blur(orig, radius)
+
+		var alpha = Bitmap.createBitmap(
+			DeepLab3.WIDTH, DeepLab3.HEIGHT, Bitmap.Config.ALPHA_8
+		)
+		alpha.copyPixelsFromBuffer(segmentMap)
+		alpha = Bitmap.createScaledBitmap(alpha, orig.width, orig.height, true)
+		// blur the mask to smoothen the edge
+		alpha = Toolkit.blur(alpha, 16)
+		File(context.filesDir, "alpha.png").outputStream().use {
+			alpha.compress(Bitmap.CompressFormat.PNG, 50, it)
+		}
+
+		val shader = ComposeShader(
+			BitmapShader(orig, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP),
+			BitmapShader(alpha, Shader.TileMode.CLAMP, Shader.TileMode.CLAMP),
+			PorterDuff.Mode.DST_ATOP
+		)
+		val paint = Paint().apply {
+			setShader(shader)
+		}
+		Canvas(bg).apply {
+			drawRect(0f, 0f, orig.width.toFloat(), orig.height.toFloat(), paint)
+		}
+		return bg
+	}
+
+	private val context = context
+	private val deepLab = DeepLab3(context)
+}

plugin/android/src/main/kotlin/com/nkming/nc_photos/plugin/image_processor/TfLiteHelper.kt

@@ -88,5 +88,20 @@ interface TfLiteHelper {
 			outputBitmap.copyPixelsFromBuffer(buffer)
 			return outputBitmap
 		}
+
+		fun argmax(
+			output: FloatBuffer, width: Int, height: Int, channel: Int
+		): ByteBuffer {
+			val product = ByteBuffer.allocate(width * height)
+			val array = output.array()
+			var j = 0
+			for (i in 0 until width * height) {
+				val pixel = array.slice(j until j + channel)
+				val max = pixel.indices.maxByOrNull { pixel[it] }!!
+				product.put(i, max.toByte())
+				j += channel
+			}
+			return product
+		}
 	}
 }

plugin/lib/src/image_processor.dart

@@ -15,6 +15,17 @@ class ImageProcessor {
         "filename": filename,
       });
 
+  static Future<void> deepLab3Portrait(
+    String fileUrl,
+    String filename, {
+    Map<String, String>? headers,
+  }) =>
+      _methodChannel.invokeMethod("deepLab3Portrait", <String, dynamic>{
+        "fileUrl": fileUrl,
+        "headers": headers,
+        "filename": filename,
+      });
+
   static const _methodChannel =
       MethodChannel("${k.libId}/image_processor_method");
 }