diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts
index 463ee712cbbc0d..9fe551feb360e4 100644
--- a/packages/vite/src/node/server/index.ts
+++ b/packages/vite/src/node/server/index.ts
@@ -669,12 +669,28 @@ export async function createServer(
     depsOptimizer.server = server
   }
 
+  // httpServer.listen can be called multiple times
+  // when port when using next port number
+  // this code is to avoid calling buildStart multiple times
+  let initingServer: Promise<void> | undefined
+  let serverInited = false
+  const initServer = async () => {
+    if (serverInited) return
+    if (initingServer) return initingServer
+
+    initingServer = (async function () {
+      await container.buildStart({})
+      initingServer = undefined
+      serverInited = true
+    })()
+  }
+
   if (!middlewareMode && httpServer) {
     // overwrite listen to init optimizer before server start
     const listen = httpServer.listen.bind(httpServer)
     httpServer.listen = (async (port: number, ...args: any[]) => {
       try {
-        await container.buildStart({})
+        await initServer()
       } catch (e) {
         httpServer.emit('error', e)
         return
@@ -682,7 +698,7 @@ export async function createServer(
       return listen(port, ...args)
     }) as any
   } else {
-    await container.buildStart({})
+    await initServer()
   }
 
   return server