diff --git a/src/node_worker.cc b/src/node_worker.cc
index d184d16ac8ee8b..32ee9b627a1a8b 100644
--- a/src/node_worker.cc
+++ b/src/node_worker.cc
@@ -680,7 +680,7 @@ void Worker::StopThread(const FunctionCallbackInfo<Value>& args) {
 void Worker::Ref(const FunctionCallbackInfo<Value>& args) {
   Worker* w;
   ASSIGN_OR_RETURN_UNWRAP(&w, args.This());
-  if (!w->has_ref_) {
+  if (!w->has_ref_ && !w->thread_joined_) {
     w->has_ref_ = true;
     w->env()->add_refs(1);
   }
@@ -689,7 +689,7 @@ void Worker::Ref(const FunctionCallbackInfo<Value>& args) {
 void Worker::Unref(const FunctionCallbackInfo<Value>& args) {
   Worker* w;
   ASSIGN_OR_RETURN_UNWRAP(&w, args.This());
-  if (w->has_ref_) {
+  if (w->has_ref_ && !w->thread_joined_) {
     w->has_ref_ = false;
     w->env()->add_refs(-1);
   }
diff --git a/test/parallel/test-worker-unref-from-message-during-exit.js b/test/parallel/test-worker-unref-from-message-during-exit.js
new file mode 100644
index 00000000000000..3e696f369e4179
--- /dev/null
+++ b/test/parallel/test-worker-unref-from-message-during-exit.js
@@ -0,0 +1,16 @@
+'use strict';
+const common = require('../common');
+const { Worker } = require('worker_threads');
+
+// This used to crash because the `.unref()` was unexpected while the Worker
+// was exiting.
+
+const w = new Worker(`
+require('worker_threads').parentPort.postMessage({});
+`, { eval: true });
+w.on('message', common.mustCall(() => {
+  w.unref();
+}));
+
+// Wait a bit so that the 'message' event is emitted while the Worker exits.
+Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);