feat: Create Frame.withBaseClass to ensure it's always a `FrameHost…

…Object` (#2849) * feat: Create `frame.withBaseClass` * fix: Add docs * Update CameraPage.tsx * feat: Implement `withBaseClass` for Android * fix: Fix Android impl * fix: Fix `getIsValid()` on Android * fix: Fix usage of Proxy
mrousavy · May 6, 2024 · 688d9f4 · 688d9f4
1 parent 9ce6469
commit 688d9f4
Show file tree

Hide file tree

Showing 12 changed files with 190 additions and 120 deletions.
diff --git a/package/android/src/main/cpp/frameprocessors/FrameHostObject.cpp b/package/android/src/main/cpp/frameprocessors/FrameHostObject.cpp
@@ -19,13 +19,13 @@ namespace vision {
 
 using namespace facebook;
 
-FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame) : frame(make_global(frame)) {}
+FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame) : _frame(make_global(frame)), _baseClass(nullptr) {}
 
 FrameHostObject::~FrameHostObject() {
   // Hermes GC might destroy HostObjects on an arbitrary Thread which might not be
   // connected to the JNI environment. To make sure fbjni can properly destroy
   // the Java method, we connect to a JNI environment first.
-  jni::ThreadScope::WithClassLoader([&] { frame.reset(); });
+  jni::ThreadScope::WithClassLoader([&] { _frame = nullptr; });
 }
 
 std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
@@ -35,7 +35,7 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("incrementRefCount")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("decrementRefCount")));
 
-  if (frame != nullptr && frame->getIsValid()) {
+  if (_frame->getIsValid()) {
     // Frame Properties
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
@@ -49,62 +49,82 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer")));
     result.push_back(jsi::PropNameID::forUtf8(rt, std::string("getNativeBuffer")));
+    result.push_back(jsi::PropNameID::forUtf8(rt, std::string("withBaseClass")));
   }
 
   return result;
 }
 
+jni::global_ref<JFrame> FrameHostObject::getFrame() {
+  if (!_frame->getIsValid()) [[unlikely]] {
+    throw std::runtime_error("Frame is already closed! "
+                             "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n"
+                             "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n"
+                             "- If you want to do async processing, use `runAsync(...)` instead.\n"
+                             "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
+  }
+  return _frame;
+}
+
 #define JSI_FUNC [=](jsi::Runtime & runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value
 
 jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
   auto name = propName.utf8(runtime);
 
   // Properties
   if (name == "isValid") {
-    return jsi::Value(this->frame && this->frame->getIsValid());
+    return jsi::Value(_frame->getIsValid());
   }
   if (name == "width") {
-    return jsi::Value(this->frame->getWidth());
+    const auto& frame = getFrame();
+    return jsi::Value(frame->getWidth());
   }
   if (name == "height") {
-    return jsi::Value(this->frame->getHeight());
+    const auto& frame = getFrame();
+    return jsi::Value(frame->getHeight());
   }
   if (name == "isMirrored") {
-    return jsi::Value(this->frame->getIsMirrored());
+    const auto& frame = getFrame();
+    return jsi::Value(frame->getIsMirrored());
   }
   if (name == "orientation") {
-    auto orientation = this->frame->getOrientation();
+    const auto& frame = getFrame();
+    auto orientation = frame->getOrientation();
     auto string = orientation->getUnionValue();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
   if (name == "pixelFormat") {
-    auto pixelFormat = this->frame->getPixelFormat();
+    const auto& frame = getFrame();
+    auto pixelFormat = frame->getPixelFormat();
     auto string = pixelFormat->getUnionValue();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
   if (name == "timestamp") {
-    return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
+    const auto& frame = getFrame();
+    return jsi::Value(static_cast<double>(frame->getTimestamp()));
   }
   if (name == "bytesPerRow") {
-    return jsi::Value(this->frame->getBytesPerRow());
+    const auto& frame = getFrame();
+    return jsi::Value(frame->getBytesPerRow());
   }
   if (name == "planesCount") {
-    return jsi::Value(this->frame->getPlanesCount());
+    const auto& frame = getFrame();
+    return jsi::Value(frame->getPlanesCount());
   }
 
   // Internal Methods
   if (name == "incrementRefCount") {
     jsi::HostFunctionType incrementRefCount = JSI_FUNC {
       // Increment retain count by one.
-      this->frame->incrementRefCount();
+      _frame->incrementRefCount();
       return jsi::Value::undefined();
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
   }
   if (name == "decrementRefCount") {
     auto decrementRefCount = JSI_FUNC {
       // Decrement retain count by one. If the retain count is zero, the Frame gets closed.
-      this->frame->decrementRefCount();
+      _frame->decrementRefCount();
       return jsi::Value::undefined();
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
@@ -113,11 +133,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   // Conversion methods
   if (name == "getNativeBuffer") {
     jsi::HostFunctionType getNativeBuffer = JSI_FUNC {
-      if (!this->frame) {
-        throw jsi::JSError(runtime, "Cannot get Platform Buffer - this Frame is already closed!");
-      }
 #if __ANDROID_API__ >= 26
-      AHardwareBuffer* hardwareBuffer = this->frame->getHardwareBuffer();
+      const auto& frame = getFrame();
+      AHardwareBuffer* hardwareBuffer = frame->getHardwareBuffer();
       AHardwareBuffer_acquire(hardwareBuffer);
       uintptr_t pointer = reinterpret_cast<uintptr_t>(hardwareBuffer);
       jsi::HostFunctionType deleteFunc = [=](jsi::Runtime& runtime, const jsi::Value& thisArg, const jsi::Value* args,
@@ -141,7 +159,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   if (name == "toArrayBuffer") {
     jsi::HostFunctionType toArrayBuffer = JSI_FUNC {
 #if __ANDROID_API__ >= 26
-      AHardwareBuffer* hardwareBuffer = this->frame->getHardwareBuffer();
+      const auto& frame = getFrame();
+      AHardwareBuffer* hardwareBuffer = frame->getHardwareBuffer();
       AHardwareBuffer_acquire(hardwareBuffer);
 
       AHardwareBuffer_Desc bufferDescription;
@@ -193,18 +212,34 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   }
   if (name == "toString") {
     jsi::HostFunctionType toString = JSI_FUNC {
-      if (!this->frame) {
+      if (!_frame->getIsValid()) {
         return jsi::String::createFromUtf8(runtime, "[closed frame]");
       }
-      auto width = this->frame->getWidth();
-      auto height = this->frame->getHeight();
-      auto format = this->frame->getPixelFormat();
+      auto width = _frame->getWidth();
+      auto height = _frame->getHeight();
+      auto format = _frame->getPixelFormat();
       auto formatString = format->getUnionValue();
       auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
       return jsi::String::createFromUtf8(runtime, str);
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
   }
+  if (name == "withBaseClass") {
+    auto withBaseClass = JSI_FUNC {
+      jsi::Object newBaseClass = arguments[0].asObject(runtime);
+      _baseClass = std::make_unique<jsi::Object>(std::move(newBaseClass));
+      return jsi::Object::createFromHostObject(runtime, shared_from_this());
+    };
+    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "withBaseClass"), 1, withBaseClass);
+  }
+
+  if (_baseClass != nullptr) {
+    // look up value in base class if we have a custom base class
+    jsi::Value value = _baseClass->getProperty(runtime, name.c_str());
+    if (!value.isUndefined()) {
+      return value;
+    }
+  }
 
   // fallback to base implementation
   return HostObject::get(runtime, propName);

diff --git a/package/android/src/main/cpp/frameprocessors/FrameHostObject.h b/package/android/src/main/cpp/frameprocessors/FrameHostObject.h
@@ -7,6 +7,7 @@
 #include <fbjni/fbjni.h>
 #include <jni.h>
 #include <jsi/jsi.h>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -16,7 +17,7 @@ namespace vision {
 
 using namespace facebook;
 
-class JSI_EXPORT FrameHostObject : public jsi::HostObject {
+class JSI_EXPORT FrameHostObject : public jsi::HostObject, public std::enable_shared_from_this<FrameHostObject> {
 public:
   explicit FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame);
   ~FrameHostObject();
@@ -26,7 +27,11 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
   std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
 
 public:
-  jni::global_ref<JFrame> frame;
+  jni::global_ref<JFrame> getFrame();
+
+private:
+  jni::global_ref<JFrame> _frame;
+  std::unique_ptr<jsi::Object> _baseClass;
 };
 
 } // namespace vision
diff --git a/package/android/src/main/cpp/frameprocessors/FrameProcessorPluginHostObject.cpp b/package/android/src/main/cpp/frameprocessors/FrameProcessorPluginHostObject.cpp
@@ -44,7 +44,7 @@ jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi:
             jsi::Object actualFrame = frameHolder.getPropertyAsObject(runtime, "__frame");
             frameHostObject = actualFrame.asHostObject<FrameHostObject>(runtime);
           }
-          auto frame = frameHostObject->frame;
+          auto frame = frameHostObject->getFrame();
 
           // Options are second argument (possibly undefined)
           local_ref<JMap<jstring, jobject>> options = nullptr;

diff --git a/package/android/src/main/cpp/frameprocessors/JSIJNIConversion.cpp b/package/android/src/main/cpp/frameprocessors/JSIJNIConversion.cpp
@@ -68,8 +68,8 @@ jni::local_ref<jobject> JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtim
       if (valueAsObject.isHostObject<FrameHostObject>(runtime)) {
         // Frame
 
-        auto frame = valueAsObject.getHostObject<FrameHostObject>(runtime);
-        return jni::make_local(frame->frame);
+        auto frameHostObject = valueAsObject.getHostObject<FrameHostObject>(runtime);
+        return jni::make_local(frameHostObject->getFrame());
 
       } else {
         throw std::runtime_error("The given HostObject is not supported by a Frame Processor Plugin.");

diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessors/Frame.java b/package/android/src/main/java/com/mrousavy/camera/frameprocessors/Frame.java
@@ -61,8 +61,7 @@ public int getHeight() throws FrameInvalidError {
 
     @SuppressWarnings("unused")
     @DoNotStrip
-    public boolean getIsValid() throws FrameInvalidError {
-        assertIsValid();
+    public boolean getIsValid() {
         return getIsImageValid(imageProxy);
     }
 

diff --git a/package/example/ios/Podfile.lock b/package/example/ios/Podfile.lock
@@ -467,16 +467,16 @@ PODS:
     - RCT-Folly (= 2021.07.22.00)
     - React-Core
   - SocketRocket (0.6.1)
-  - VisionCamera (4.0.1):
-    - VisionCamera/Core (= 4.0.1)
-    - VisionCamera/FrameProcessors (= 4.0.1)
-    - VisionCamera/React (= 4.0.1)
-  - VisionCamera/Core (4.0.1)
-  - VisionCamera/FrameProcessors (4.0.1):
+  - VisionCamera (4.0.2):
+    - VisionCamera/Core (= 4.0.2)
+    - VisionCamera/FrameProcessors (= 4.0.2)
+    - VisionCamera/React (= 4.0.2)
+  - VisionCamera/Core (4.0.2)
+  - VisionCamera/FrameProcessors (4.0.2):
     - React
     - React-callinvoker
     - react-native-worklets-core
-  - VisionCamera/React (4.0.1):
+  - VisionCamera/React (4.0.2):
     - React-Core
     - VisionCamera/FrameProcessors
   - Yoga (1.14.0)
@@ -708,7 +708,7 @@ SPEC CHECKSUMS:
   RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
   RNVectorIcons: 23b6e11af4aaf104d169b1b0afa7e5cf96c676ce
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
-  VisionCamera: 8e00df84a76cf26ca70ecd3b6ed05dc0d3b60beb
+  VisionCamera: 14a6a4a53c65b8b9bd441f80c2b29048f820ae01
   Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
 
 PODFILE CHECKSUM: 66976ac26c778d788a06e6c1bab624e6a1233cdd

diff --git a/package/ios/FrameProcessors/FrameHostObject.h b/package/ios/FrameProcessors/FrameHostObject.h
@@ -10,22 +10,24 @@
 
 #import <CoreMedia/CMSampleBuffer.h>
 #import <jsi/jsi.h>
+#import <memory.h>
 
 #import "Frame.h"
 
 using namespace facebook;
 
-class JSI_EXPORT FrameHostObject : public jsi::HostObject {
+class JSI_EXPORT FrameHostObject : public jsi::HostObject, public std::enable_shared_from_this<FrameHostObject> {
 public:
-  explicit FrameHostObject(Frame* frame) : frame(frame) {}
+  explicit FrameHostObject(Frame* frame) : _frame(frame), _baseClass(nullptr) {}
 
 public:
   jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
   std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
 
 public:
-  Frame* frame;
+  Frame* getFrame();
 
 private:
-  Frame* getFrame();
+  Frame* _frame;
+  std::unique_ptr<jsi::Object> _baseClass;
 };