diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts
index f005a521a97a52..b5e3f1d0b13565 100644
--- a/packages/vite/src/node/optimizer/index.ts
+++ b/packages/vite/src/node/optimizer/index.ts
@@ -615,7 +615,11 @@ export function getOptimizedDepPath(
 function getDepsCacheSuffix(config: ResolvedConfig): string {
   let suffix = ''
   if (config.command === 'build') {
-    suffix += '_build'
+    // Differentiate build caches depending on outDir to allow parallel builds
+    const { outDir } = config.build
+    const buildId =
+      outDir.length > 8 || outDir.includes('/') ? getHash(outDir) : outDir
+    suffix += `_build-${buildId}`
     if (config.build.ssr) {
       suffix += '_ssr'
     }
diff --git a/playground/assets/vite.config-relative-base.js b/playground/assets/vite.config-relative-base.js
index 12c3132f79a9b0..ae09766c0768ac 100644
--- a/playground/assets/vite.config-relative-base.js
+++ b/playground/assets/vite.config-relative-base.js
@@ -22,6 +22,5 @@ module.exports = {
   },
   testConfig: {
     baseRoute: '/relative-base/'
-  },
-  cacheDir: 'node_modules/.vite/relative-base'
+  }
 }
diff --git a/playground/assets/vite.config.js b/playground/assets/vite.config.js
index 23bd11908130cd..c9d821ae3d73ee 100644
--- a/playground/assets/vite.config.js
+++ b/playground/assets/vite.config.js
@@ -17,6 +17,5 @@ module.exports = {
     assetsInlineLimit: 8192, // 8kb
     manifest: true,
     watch: {}
-  },
-  cacheDir: 'node_modules/.vite/foo'
+  }
 }
diff --git a/playground/worker/vite.config-es.js b/playground/worker/vite.config-es.js
index 0d28d241ca8d42..6d6704de0bc213 100644
--- a/playground/worker/vite.config-es.js
+++ b/playground/worker/vite.config-es.js
@@ -38,6 +38,5 @@ module.exports = vite.defineConfig({
         }
       }
     }
-  ],
-  cacheDir: 'node_modules/.vite/es'
+  ]
 })
diff --git a/playground/worker/vite.config-relative-base.js b/playground/worker/vite.config-relative-base.js
index 64c2b7983fb527..8002883ca4abf1 100644
--- a/playground/worker/vite.config-relative-base.js
+++ b/playground/worker/vite.config-relative-base.js
@@ -40,6 +40,5 @@ module.exports = vite.defineConfig({
         }
       }
     }
-  ],
-  cacheDir: 'node_modules/.vite/relative-base'
+  ]
 })
diff --git a/playground/worker/vite.config-sourcemap.js b/playground/worker/vite.config-sourcemap.js
index a84d8e6699af14..c4cf61f6dea586 100644
--- a/playground/worker/vite.config-sourcemap.js
+++ b/playground/worker/vite.config-sourcemap.js
@@ -21,9 +21,6 @@ module.exports = vite.defineConfig((sourcemap) => {
         }
       }
     },
-    cacheDir: `node_modules/.vite/iife-${
-      typeof sourcemap === 'boolean' ? 'sourcemap' : 'sourcemap-' + sourcemap
-    }`,
     build: {
       outDir: `dist/iife-${
         typeof sourcemap === 'boolean' ? 'sourcemap' : 'sourcemap-' + sourcemap
diff --git a/playground/worker/vite.config.js b/playground/worker/vite.config.js
index f937372c6e0012..d62d6c4f6d6d36 100644
--- a/playground/worker/vite.config.js
+++ b/playground/worker/vite.config.js
@@ -23,6 +23,5 @@ module.exports = vite.defineConfig({
         entryFileNames: 'assets/[name].js'
       }
     }
-  },
-  cacheDir: 'node_modules/.vite/iife'
+  }
 })