diff --git a/packages/fiber/src/core/renderer.ts b/packages/fiber/src/core/renderer.ts
index 626ad147eb..c83e84e066 100644
--- a/packages/fiber/src/core/renderer.ts
+++ b/packages/fiber/src/core/renderer.ts
@@ -353,8 +353,10 @@ function createRenderer<TCanvas>(_roots: Map<TCanvas, Root>, _getEventPriority?:
       return Boolean(localState.handlers)
     },
     prepareUpdate(instance, _type, oldProps, newProps) {
+      const localState = instance?.__r3f ?? {}
+
       // Create diff-sets
-      if (instance.__r3f.primitive && newProps.object && newProps.object !== instance) {
+      if (localState.primitive && newProps.object && newProps.object !== instance) {
         return [true]
       } else {
         // This is a data object, let's extract critical information about it
diff --git a/packages/fiber/src/core/utils.ts b/packages/fiber/src/core/utils.ts
index 584ec5ba09..63d04b1aff 100644
--- a/packages/fiber/src/core/utils.ts
+++ b/packages/fiber/src/core/utils.ts
@@ -177,19 +177,18 @@ export function dispose<TObj extends { dispose?: () => void; type?: string; [key
 // Each object in the scene carries a small LocalState descriptor
 export function prepare<T = THREE.Object3D>(object: T, state?: Partial<LocalState>) {
   const instance = object as unknown as Instance
-  if (state?.primitive || !instance.__r3f) {
-    instance.__r3f = {
-      type: '',
-      root: null as unknown as UseBoundStore<RootState>,
-      previousAttach: null,
-      memoizedProps: {},
-      eventCount: 0,
-      handlers: {},
-      objects: [],
-      parent: null,
-      ...state,
-    }
+  instance.__r3f = {
+    type: '',
+    root: null as unknown as UseBoundStore<RootState>,
+    previousAttach: null,
+    memoizedProps: {},
+    eventCount: 0,
+    handlers: {},
+    objects: [],
+    parent: null,
+    ...state,
   }
+
   return object
 }