diff --git a/packages/vite/src/node/plugins/worker.ts b/packages/vite/src/node/plugins/worker.ts
index 363b83edfa8d72..acb8c67eb1e88a 100644
--- a/packages/vite/src/node/plugins/worker.ts
+++ b/packages/vite/src/node/plugins/worker.ts
@@ -49,10 +49,33 @@ function saveEmitWorkerAsset(
   workerMap.assets.set(fileName, asset)
 }
 
+// Ensure that only one rollup build is called at the same time to avoid
+// leaking state in plugins between worker builds.
+// TODO: Review if we can parallelize the bundling of workers.
+const workerConfigSemaphore = new WeakMap<
+  ResolvedConfig,
+  Promise<OutputChunk>
+>()
 export async function bundleWorkerEntry(
   config: ResolvedConfig,
   id: string,
   query: Record<string, string> | null,
+): Promise<OutputChunk> {
+  const processing = workerConfigSemaphore.get(config)
+  if (processing) {
+    await processing
+    return bundleWorkerEntry(config, id, query)
+  }
+  const promise = serialBundleWorkerEntry(config, id, query)
+  workerConfigSemaphore.set(config, promise)
+  promise.then(() => workerConfigSemaphore.delete(config))
+  return promise
+}
+
+async function serialBundleWorkerEntry(
+  config: ResolvedConfig,
+  id: string,
+  query: Record<string, string> | null,
 ): Promise<OutputChunk> {
   // bundle the file as entry to support imports
   const { rollup } = await import('rollup')