diff --git a/packages/vite/src/node/build.ts b/packages/vite/src/node/build.ts
index 25b752d7512df6..5e2da01695a0a1 100644
--- a/packages/vite/src/node/build.ts
+++ b/packages/vite/src/node/build.ts
@@ -306,7 +306,7 @@ export function resolveBuildPlugins(config: ResolvedConfig): {
     pre: [
       ...(options.watch ? [ensureWatchPlugin()] : []),
       watchPackageDataPlugin(config),
-      ...(!isDepsOptimizerEnabled(config) || options.ssr
+      ...(!isDepsOptimizerEnabled(config)
         ? [commonjsPlugin(options.commonjsOptions)]
         : []),
       dataURIPlugin(),
@@ -402,7 +402,7 @@ async function doBuild(
     external = await cjsSsrResolveExternal(config, userExternal)
   }
 
-  if (isDepsOptimizerEnabled(config) && !ssr) {
+  if (isDepsOptimizerEnabled(config)) {
     await initDepsOptimizer(config)
   }
 
diff --git a/packages/vite/src/node/optimizer/index.ts b/packages/vite/src/node/optimizer/index.ts
index 4f4231354e0a6a..ea291675ccc8cf 100644
--- a/packages/vite/src/node/optimizer/index.ts
+++ b/packages/vite/src/node/optimizer/index.ts
@@ -610,13 +610,23 @@ export function getOptimizedDepPath(
   )
 }
 
+function getDepsCacheSuffix(config: ResolvedConfig): string {
+  let suffix = ''
+  if (config.command === 'build') {
+    suffix += '_build'
+    if (config.build.ssr) {
+      suffix += '_ssr'
+    }
+  }
+  return suffix
+}
 export function getDepsCacheDir(config: ResolvedConfig): string {
-  const dirName = config.command === 'build' ? 'depsBuild' : 'deps'
+  const dirName = 'deps' + getDepsCacheSuffix(config)
   return normalizePath(path.resolve(config.cacheDir, dirName))
 }
 
 function getProcessingDepsCacheDir(config: ResolvedConfig) {
-  const dirName = config.command === 'build' ? 'processingBuild' : 'processing'
+  const dirName = 'deps' + getDepsCacheSuffix(config) + '_temp'
   return normalizePath(path.resolve(config.cacheDir, dirName))
 }
 
diff --git a/packages/vite/src/node/plugins/resolve.ts b/packages/vite/src/node/plugins/resolve.ts
index f1f940c218716a..538389828ed2c2 100644
--- a/packages/vite/src/node/plugins/resolve.ts
+++ b/packages/vite/src/node/plugins/resolve.ts
@@ -89,6 +89,7 @@ export interface InternalResolveOptions extends ResolveOptions {
 export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin {
   const {
     root,
+    isBuild,
     isProduction,
     asSrc,
     ssrConfig,
@@ -266,7 +267,7 @@ export function resolvePlugin(baseOptions: InternalResolveOptions): Plugin {
           !external &&
           asSrc &&
           depsOptimizer &&
-          !ssr &&
+          (isBuild || !ssr) &&
           !options.scan &&
           (res = await tryOptimizedResolve(depsOptimizer, id, importer))
         ) {
@@ -668,7 +669,7 @@ export function tryNodeResolve(
     exclude?.includes(pkgId) ||
     exclude?.includes(nestedPath) ||
     SPECIAL_QUERY_RE.test(resolved) ||
-    ssr
+    (!isBuild && ssr)
   ) {
     // excluded from optimization
     // Inject a version query to npm deps so that the browser
@@ -682,7 +683,6 @@ export function tryNodeResolve(
       }
     }
   } else {
-    // TODO: depsBuild
     // this is a missing import, queue optimize-deps re-run and
     // get a resolved its optimized info
     const optimizedInfo = depsOptimizer.registerMissingImport(id, resolved)
diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts
index 983fa05efec6e0..095c7027c68558 100644
--- a/packages/vite/src/node/server/index.ts
+++ b/packages/vite/src/node/server/index.ts
@@ -320,18 +320,7 @@ export async function createServer(
     },
     transformIndexHtml: null!, // to be immediately set
     async ssrLoadModule(url, opts?: { fixStacktrace?: boolean }) {
-      if (!server._ssrExternals) {
-        let knownImports: string[] = []
-        const depsOptimizer = getDepsOptimizer(config)
-        if (depsOptimizer) {
-          await depsOptimizer.scanProcessing
-          knownImports = [
-            ...Object.keys(depsOptimizer.metadata.optimized),
-            ...Object.keys(depsOptimizer.metadata.discovered)
-          ]
-        }
-        server._ssrExternals = cjsSsrResolveExternals(config, knownImports)
-      }
+      await updateCjsSsrExternals(server)
       return ssrLoadModule(
         url,
         server,
@@ -755,3 +744,18 @@ async function restartServer(server: ViteDevServer) {
   // new server (the current server) can restart now
   newServer._restartPromise = null
 }
+
+async function updateCjsSsrExternals(server: ViteDevServer) {
+  if (!server._ssrExternals) {
+    let knownImports: string[] = []
+    const depsOptimizer = getDepsOptimizer(server.config)
+    if (depsOptimizer) {
+      await depsOptimizer.scanProcessing
+      knownImports = [
+        ...Object.keys(depsOptimizer.metadata.optimized),
+        ...Object.keys(depsOptimizer.metadata.discovered)
+      ]
+    }
+    server._ssrExternals = cjsSsrResolveExternals(server.config, knownImports)
+  }
+}