diff --git a/packages/reactivity/__tests__/readonly.spec.ts b/packages/reactivity/__tests__/readonly.spec.ts
index c37c8e190ed..d0c91f0fb9f 100644
--- a/packages/reactivity/__tests__/readonly.spec.ts
+++ b/packages/reactivity/__tests__/readonly.spec.ts
@@ -480,21 +480,27 @@ describe('reactivity/readonly', () => {
     const r = ref(false)
     const ror = readonly(r)
     const obj = reactive({ ror })
-    try {
+    expect(() => {
       obj.ror = true
-    } catch (e) {}
-
+    }).toThrow()
     expect(obj.ror).toBe(false)
   })
+
   test('replacing a readonly ref nested in a reactive object with a new ref', () => {
     const r = ref(false)
     const ror = readonly(r)
     const obj = reactive({ ror })
-    try {
-      obj.ror = ref(true) as unknown as boolean
-    } catch (e) {}
-
+    obj.ror = ref(true) as unknown as boolean
     expect(obj.ror).toBe(true)
     expect(toRaw(obj).ror).not.toBe(ror) // ref successfully replaced
   })
+
+  test('setting readonly object to writable nested ref', () => {
+    const r = ref<any>()
+    const obj = reactive({ r })
+    const ro = readonly({})
+    obj.r = ro
+    expect(obj.r).toBe(ro)
+    expect(r.value).toBe(ro)
+  })
 })
diff --git a/packages/reactivity/__tests__/ref.spec.ts b/packages/reactivity/__tests__/ref.spec.ts
index 2451c43ab45..76add567d89 100644
--- a/packages/reactivity/__tests__/ref.spec.ts
+++ b/packages/reactivity/__tests__/ref.spec.ts
@@ -10,7 +10,7 @@ import {
 } from '../src/index'
 import { computed } from '@vue/runtime-dom'
 import { shallowRef, unref, customRef, triggerRef } from '../src/ref'
-import { isShallow } from '../src/reactive'
+import { isShallow, readonly, shallowReactive } from '../src/reactive'
 
 describe('reactivity/ref', () => {
   it('should hold a value', () => {
@@ -400,4 +400,22 @@ describe('reactivity/ref', () => {
     b.value = obj
     expect(spy2).toBeCalledTimes(1)
   })
+
+  test('ref should preserve value shallow/readonly-ness', () => {
+    const original = {}
+    const r = reactive(original)
+    const s = shallowReactive(original)
+    const rr = readonly(original)
+    const a = ref(original)
+
+    expect(a.value).toBe(r)
+
+    a.value = s
+    expect(a.value).toBe(s)
+    expect(a.value).not.toBe(r)
+
+    a.value = rr
+    expect(a.value).toBe(rr)
+    expect(a.value).not.toBe(r)
+  })
 })
diff --git a/packages/reactivity/__tests__/shallowReactive.spec.ts b/packages/reactivity/__tests__/shallowReactive.spec.ts
index 200aa462496..4d39fa9b393 100644
--- a/packages/reactivity/__tests__/shallowReactive.spec.ts
+++ b/packages/reactivity/__tests__/shallowReactive.spec.ts
@@ -7,6 +7,7 @@ import {
 } from '../src/reactive'
 
 import { effect } from '../src/effect'
+import { Ref, isRef, ref } from '../src/ref'
 
 describe('shallowReactive', () => {
   test('should not make non-reactive properties reactive', () => {
@@ -46,6 +47,27 @@ describe('shallowReactive', () => {
     expect(isReactive(r.foo.bar)).toBe(false)
   })
 
+  // vuejs/vue#12597
+  test('should not unwrap refs', () => {
+    const foo = shallowReactive({
+      bar: ref(123)
+    })
+    expect(isRef(foo.bar)).toBe(true)
+    expect(foo.bar.value).toBe(123)
+  })
+
+  // vuejs/vue#12688
+  test('should not mutate refs', () => {
+    const original = ref(123)
+    const foo = shallowReactive<{ bar: Ref<number> | number }>({
+      bar: original
+    })
+    expect(foo.bar).toBe(original)
+    foo.bar = 234
+    expect(foo.bar).toBe(234)
+    expect(original.value).toBe(123)
+  })
+
   test('should respect shallow/deep versions of same target on access', () => {
     const original = {}
     const shallow = shallowReactive(original)
diff --git a/packages/reactivity/src/baseHandlers.ts b/packages/reactivity/src/baseHandlers.ts
index dac444ffdc1..dfff56f7fc5 100644
--- a/packages/reactivity/src/baseHandlers.ts
+++ b/packages/reactivity/src/baseHandlers.ts
@@ -158,10 +158,10 @@ function createSetter(shallow = false) {
     if (isReadonly(oldValue) && isRef(oldValue) && !isRef(value)) {
       return false
     }
-    if (!shallow && !isReadonly(value)) {
-      if (!isShallow(value)) {
-        value = toRaw(value)
+    if (!shallow) {
+      if (!isShallow(value) && !isReadonly(value)) {
         oldValue = toRaw(oldValue)
+        value = toRaw(value)
       }
       if (!isArray(target) && isRef(oldValue) && !isRef(value)) {
         oldValue.value = value
diff --git a/packages/reactivity/src/ref.ts b/packages/reactivity/src/ref.ts
index 2a3b3732e0e..5843b3f63ae 100644
--- a/packages/reactivity/src/ref.ts
+++ b/packages/reactivity/src/ref.ts
@@ -6,7 +6,14 @@ import {
 } from './effect'
 import { TrackOpTypes, TriggerOpTypes } from './operations'
 import { isArray, hasChanged, IfAny } from '@vue/shared'
-import { isProxy, toRaw, isReactive, toReactive } from './reactive'
+import {
+  isProxy,
+  toRaw,
+  isReactive,
+  toReactive,
+  isReadonly,
+  isShallow
+} from './reactive'
 import type { ShallowReactiveMarker } from './reactive'
 import { CollectionTypes } from './collectionHandlers'
 import { createDep, Dep } from './dep'
@@ -112,10 +119,12 @@ class RefImpl<T> {
   }
 
   set value(newVal) {
-    newVal = this.__v_isShallow ? newVal : toRaw(newVal)
+    const useDirectValue =
+      this.__v_isShallow || isShallow(newVal) || isReadonly(newVal)
+    newVal = useDirectValue ? newVal : toRaw(newVal)
     if (hasChanged(newVal, this._rawValue)) {
       this._rawValue = newVal
-      this._value = this.__v_isShallow ? newVal : toReactive(newVal)
+      this._value = useDirectValue ? newVal : toReactive(newVal)
       triggerRefValue(this, newVal)
     }
   }