diff --git a/packages/histoire/src/node/vite.ts b/packages/histoire/src/node/vite.ts
index 5673b54e..be394b58 100644
--- a/packages/histoire/src/node/vite.ts
+++ b/packages/histoire/src/node/vite.ts
@@ -193,7 +193,7 @@ export async function getViteConfigWithPlugins (isServer: boolean, ctx: Context)
             file = setupFileConfig
           } else if (isServer && 'server' in setupFileConfig) {
             file = setupFileConfig.server
-          } else if ('browser' in setupFileConfig) {
+          } else if (!isServer && 'browser' in setupFileConfig) {
             file = setupFileConfig.browser
           }
           if (file) {