diff --git a/packages/vitest/src/node/create.ts b/packages/vitest/src/node/create.ts
index ce67eac9d977..1df9b7f62129 100644
--- a/packages/vitest/src/node/create.ts
+++ b/packages/vitest/src/node/create.ts
@@ -23,10 +23,11 @@ export async function createVitest(options: UserConfig, viteOverrides: ViteUserC
   }
 
   const server = await createServer(mergeConfig(config, viteOverrides))
-  await server.pluginContainer.buildStart({})
 
   if (ctx.config.api?.port)
     await server.listen()
+  else
+    await server.pluginContainer.buildStart({})
 
   return ctx
 }