[camera] Support scanning barcode from URL (#28445)

apps/native-component-list/src/navigation/ExpoComponentsStackNavigator.tsx

@@ -67,6 +67,12 @@ export const Screens = [
     },
     name: 'Camera (barcode)',
   },
+  {
+    getComponent() {
+      return optionalRequire(() => require('../screens/Camera/CameraScreenBarcodeFromURL'));
+    },
+    name: 'Camera (barcode from URL)',
+  },
   {
     getComponent() {
       return optionalRequire(() => require('../screens/TextScreen'));

apps/native-component-list/src/screens/Camera/CameraScreenBarcodeFromURL.tsx

@@ -0,0 +1,64 @@
+import { BarcodeScanningResult, Camera } from 'expo-camera';
+import { Image } from 'expo-image';
+import * as ImagePicker from 'expo-image-picker';
+import { useState } from 'react';
+import { View, Text, Button, ScrollView, StyleSheet } from 'react-native';
+
+export default function CameraScreenFromURL() {
+  const [image, setImage] = useState<string | null>(null);
+  const [results, setResults] = useState<BarcodeScanningResult | null>(null);
+
+  const pickImage = async () => {
+    const result = await ImagePicker.launchImageLibraryAsync({
+      mediaTypes: ImagePicker.MediaTypeOptions.Images,
+      allowsEditing: false,
+      quality: 1,
+    });
+
+    if (!result.canceled) {
+      setImage(result.assets[0].uri);
+    }
+  };
+
+  const scanImage = async () => {
+    if (!image) {
+      return;
+    }
+    const results = await Camera.scanFromURLAsync(image);
+    setResults(results);
+  };
+
+  return (
+    <View style={styles.container}>
+      {!image && <Text>Select an image from the photo library</Text>}
+      <Button title="Select image" onPress={pickImage} />
+      {image && <Image source={{ uri: image }} style={styles.image} />}
+      {image && <Button title="Scan Image" onPress={scanImage} />}
+      <ScrollView style={styles.scrollView} contentContainerStyle={styles.scollViewContent}>
+        {results && <Text>{JSON.stringify(results, null, 2)}</Text>}
+      </ScrollView>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    alignItems: 'center',
+    justifyContent: 'center',
+    gap: 20,
+    paddingTop: 20,
+  },
+  image: {
+    width: 200,
+    height: 200,
+  },
+  scrollView: {
+    flex: 1,
+    width: '100%',
+  },
+  scollViewContent: {
+    padding: 20,
+    alignItems: 'center',
+  },
+});

apps/native-component-list/src/screens/ExpoComponentsScreen.tsx

@@ -9,6 +9,7 @@ const screens = [
   'Camera',
   'Camera (legacy)',
   'Camera (barcode)',
+  'Camera (barcode from URL)',
   'Checkbox',
   'ClipboardPasteButton',
   'DateTimePicker',

packages/expo-camera/CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ### 🎉 New features
 
+- Support scanning barcodes from a provided image URL. ([#28445](https://github.com/expo/expo/pull/28445) by [@alanjhughes](https://github.com/alanjhughes))
+
 ### 🐛 Bug fixes
 
 ### 💡 Others

packages/expo-camera/android/src/main/java/expo/modules/camera/CameraExceptions.kt

@@ -6,4 +6,7 @@ class CameraExceptions {
   class ImageCaptureFailed : CodedException(message = "Failed to capture image")
 
   class VideoRecordingFailed(cause: String?) : CodedException("Video recording failed: $cause")
+
+  class ImageRetrievalException(url: String) :
+    CodedException("Could not get the image from given url: '$url'")
 }

packages/expo-camera/android/src/main/java/expo/modules/camera/CameraViewModule.kt

@@ -1,15 +1,20 @@
 package expo.modules.camera
 
 import android.Manifest
+import android.graphics.Bitmap
 import android.util.Log
+import expo.modules.camera.analyzers.BarCodeScannerResultSerializer
+import expo.modules.camera.analyzers.MLKitBarCodeScanner
 import expo.modules.camera.records.BarcodeSettings
+import expo.modules.camera.records.BarcodeType
 import expo.modules.camera.records.CameraMode
 import expo.modules.camera.records.CameraType
 import expo.modules.camera.records.FlashMode
 import expo.modules.camera.records.VideoQuality
 import expo.modules.camera.tasks.ResolveTakenPicture
 import expo.modules.core.errors.ModuleDestroyedException
 import expo.modules.core.utilities.EmulatorUtilities
+import expo.modules.interfaces.imageloader.ImageLoaderInterface
 import expo.modules.interfaces.permissions.Permissions
 import expo.modules.kotlin.Promise
 import expo.modules.kotlin.exception.Exceptions
@@ -33,6 +38,7 @@ val cameraEvents = arrayOf(
 
 class CameraViewModule : Module() {
   private val moduleScope = CoroutineScope(Dispatchers.Main)
+
   override fun definition() = ModuleDefinition {
     Name("ExpoCamera")
 
@@ -70,6 +76,30 @@ class CameraViewModule : Module() {
       )
     }
 
+    AsyncFunction("scanFromURLAsync") { url: String, barcodeTypes: List<BarcodeType>, promise: Promise ->
+      appContext.imageLoader?.loadImageForManipulationFromURL(
+        url,
+        object : ImageLoaderInterface.ResultListener {
+          override fun onSuccess(bitmap: Bitmap) {
+            val scanner = MLKitBarCodeScanner()
+            val formats = barcodeTypes.map { it.mapToBarcode() }
+            scanner.setSettings(formats)
+
+            moduleScope.launch {
+              val barcodes = scanner.scan(bitmap)
+                .filter { formats.contains(it.type) }
+                .map { BarCodeScannerResultSerializer.toBundle(it, 1.0f) }
+              promise.resolve(barcodes)
+            }
+          }
+
+          override fun onFailure(cause: Throwable?) {
+            promise.reject(CameraExceptions.ImageRetrievalException(url))
+          }
+        }
+      )
+    }
+
     OnDestroy {
       try {
         moduleScope.cancel(ModuleDestroyedException())

packages/expo-camera/android/src/main/java/expo/modules/camera/analyzers/BarcodeScannerResultSerializer.kt

@@ -0,0 +1,48 @@
+package expo.modules.camera.analyzers
+
+import android.os.Bundle
+import android.util.Pair
+import expo.modules.interfaces.barcodescanner.BarCodeScannerResult
+
+object BarCodeScannerResultSerializer {
+  fun toBundle(result: BarCodeScannerResult, density: Float) =
+    Bundle().apply {
+      putString("data", result.value)
+      putString("raw", result.raw)
+      putInt("type", result.type)
+      val cornerPointsAndBoundingBox = getCornerPointsAndBoundingBox(result.cornerPoints, result.boundingBox, density)
+      putParcelableArrayList("cornerPoints", cornerPointsAndBoundingBox.first)
+      putBundle("bounds", cornerPointsAndBoundingBox.second)
+    }
+
+  private fun getCornerPointsAndBoundingBox(
+    cornerPoints: List<Int>,
+    boundingBox: BarCodeScannerResult.BoundingBox,
+    density: Float
+  ): Pair<ArrayList<Bundle>, Bundle> {
+    val convertedCornerPoints = ArrayList<Bundle>()
+    for (i in cornerPoints.indices step 2) {
+      val x = cornerPoints[i].toFloat() / density
+      val y = cornerPoints[i + 1].toFloat() / density
+
+      convertedCornerPoints.add(getPoint(x, y))
+    }
+    val boundingBoxBundle = Bundle().apply {
+      putParcelable("origin", getPoint(boundingBox.x.toFloat() / density, boundingBox.y.toFloat() / density))
+      putParcelable("size", getSize(boundingBox.width.toFloat() / density, boundingBox.height.toFloat() / density))
+    }
+    return Pair(convertedCornerPoints, boundingBoxBundle)
+  }
+
+  private fun getSize(width: Float, height: Float) =
+    Bundle().apply {
+      putFloat("width", width)
+      putFloat("height", height)
+    }
+
+  private fun getPoint(x: Float, y: Float) =
+    Bundle().apply {
+      putFloat("x", x)
+      putFloat("y", y)
+    }
+}

packages/expo-camera/android/src/main/java/expo/modules/camera/analyzers/MLKitBarcodeAnalyzer.kt

@@ -0,0 +1,102 @@
+package expo.modules.camera.analyzers
+
+import android.graphics.Bitmap
+import android.util.Log
+import com.google.android.gms.tasks.Task
+import com.google.mlkit.vision.barcode.BarcodeScannerOptions
+import com.google.mlkit.vision.barcode.BarcodeScanning
+import com.google.mlkit.vision.barcode.common.Barcode
+import com.google.mlkit.vision.common.InputImage
+import expo.modules.interfaces.barcodescanner.BarCodeScannerResult
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlinx.coroutines.withContext
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+
+class MLKitBarCodeScanner {
+  private var barCodeTypes: List<Int>? = null
+  private var barcodeScannerOptions =
+    BarcodeScannerOptions.Builder()
+      .setBarcodeFormats(Barcode.FORMAT_ALL_FORMATS)
+      .build()
+  private var barcodeScanner = BarcodeScanning.getClient(barcodeScannerOptions)
+
+  suspend fun scan(bitmap: Bitmap): List<BarCodeScannerResult> = withContext(Dispatchers.IO) {
+    val inputImage = InputImage.fromBitmap(bitmap, 0)
+    try {
+      val result: List<Barcode> = barcodeScanner.process(inputImage).await()
+      val results = mutableListOf<BarCodeScannerResult>()
+      if (result.isEmpty()) {
+        return@withContext results
+      }
+      for (barcode in result) {
+        val raw = barcode.rawValue ?: barcode.rawBytes?.let { String(it) }
+        val value = if (barcode.valueType == Barcode.TYPE_CONTACT_INFO) {
+          raw
+        } else {
+          barcode.displayValue
+        }
+        val cornerPoints = mutableListOf<Int>()
+        barcode.cornerPoints?.let { points ->
+          for (point in points) {
+            cornerPoints.addAll(listOf(point.x, point.y))
+          }
+        }
+
+        results.add(BarCodeScannerResult(barcode.format, value, raw, cornerPoints, inputImage.height, inputImage.width))
+      }
+      return@withContext results
+    } catch (e: Exception) {
+      Log.e(TAG, "Failed to detect barcode: " + e.message)
+      return@withContext emptyList()
+    }
+  }
+
+  fun setSettings(formats: List<Int>) {
+    if (areNewAndOldBarCodeTypesEqual(formats)) {
+      return
+    }
+    val barcodeFormats = formats.reduce { acc, it ->
+      acc or it
+    }
+
+    barCodeTypes = formats
+    barcodeScannerOptions = BarcodeScannerOptions.Builder()
+      .setBarcodeFormats(barcodeFormats)
+      .build()
+    barcodeScanner = BarcodeScanning.getClient(barcodeScannerOptions)
+  }
+
+  private fun areNewAndOldBarCodeTypesEqual(newBarCodeTypes: List<Int>): Boolean {
+    barCodeTypes?.run {
+      // create distinct-values sets
+      val prevTypesSet = toHashSet()
+      val nextTypesSet = newBarCodeTypes.toHashSet()
+
+      // sets sizes are equal -> possible content equality
+      if (prevTypesSet.size == nextTypesSet.size) {
+        prevTypesSet.removeAll(nextTypesSet)
+        // every element from new set was in previous one -> sets are equal
+        return prevTypesSet.isEmpty()
+      }
+    }
+    return false
+  }
+
+  companion object {
+    private val TAG = MLKitBarCodeScanner::class.java.simpleName
+  }
+}
+
+suspend fun <T> Task<T>.await(): T = suspendCancellableCoroutine { continuation ->
+  addOnSuccessListener { result ->
+    continuation.resume(result)
+  }
+  addOnFailureListener { exception ->
+    continuation.resumeWithException(exception)
+  }
+  addOnCanceledListener {
+    continuation.cancel()
+  }
+}