diff --git a/packages/vite/src/node/plugins/importMetaGlob.ts b/packages/vite/src/node/plugins/importMetaGlob.ts
index 7ff08e544a78c6..7d765f4dfee16d 100644
--- a/packages/vite/src/node/plugins/importMetaGlob.ts
+++ b/packages/vite/src/node/plugins/importMetaGlob.ts
@@ -51,7 +51,14 @@ export function getAffectedGlobModules(
 ): ModuleNode[] {
   const modules: ModuleNode[] = []
   for (const [id, allGlobs] of server._importGlobMap!) {
-    if (allGlobs.some((glob) => isMatch(file, glob)))
+    // (glob1 || glob2) && !glob3 && !glob4...
+    if (
+      allGlobs.some(
+        ({ affirmed, negated }) =>
+          (!affirmed.length || affirmed.some((glob) => isMatch(file, glob))) &&
+          (!negated.length || negated.every((glob) => isMatch(file, glob))),
+      )
+    )
       modules.push(...(server.moduleGraph.getModulesByFile(id) || []))
   }
   modules.forEach((i) => {
@@ -83,7 +90,18 @@ export function importGlobPlugin(config: ResolvedConfig): Plugin {
       if (result) {
         if (server) {
           const allGlobs = result.matches.map((i) => i.globsResolved)
-          server._importGlobMap.set(id, allGlobs)
+          server._importGlobMap.set(
+            id,
+            allGlobs.map((globs) => {
+              const affirmed: string[] = []
+              const negated: string[] = []
+
+              for (const glob of globs) {
+                ;(glob[0] === '!' ? negated : affirmed).push(glob)
+              }
+              return { affirmed, negated }
+            }),
+          )
         }
         return transformStableResult(result.s, id, config)
       }
diff --git a/packages/vite/src/node/server/index.ts b/packages/vite/src/node/server/index.ts
index ccec3b77400a65..67b455735b7be8 100644
--- a/packages/vite/src/node/server/index.ts
+++ b/packages/vite/src/node/server/index.ts
@@ -285,7 +285,7 @@ export interface ViteDevServer {
   /**
    * @internal
    */
-  _importGlobMap: Map<string, string[][]>
+  _importGlobMap: Map<string, { affirmed: string[]; negated: string[] }[]>
   /**
    * Deps that are externalized
    * @internal
diff --git a/playground/glob-import/__tests__/glob-import.spec.ts b/playground/glob-import/__tests__/glob-import.spec.ts
index cf1b6c199e73aa..931edfd99138de 100644
--- a/playground/glob-import/__tests__/glob-import.spec.ts
+++ b/playground/glob-import/__tests__/glob-import.spec.ts
@@ -178,6 +178,18 @@ if (!isBuild) {
       expect(JSON.parse(actualRemove)).toStrictEqual(allResult)
     })
   })
+
+  test('no hmr for adding/removing files', async () => {
+    let request = page.waitForResponse(/dir\/index\.js$/, { timeout: 200 })
+    addFile('nohmr.js', '')
+    let response = await request.catch(() => ({ status: () => -1 }))
+    expect(response.status()).toBe(-1)
+
+    request = page.waitForResponse(/dir\/index\.js$/, { timeout: 200 })
+    removeFile('nohmr.js')
+    response = await request.catch(() => ({ status: () => -1 }))
+    expect(response.status()).toBe(-1)
+  })
 }
 
 test('tree-shake eager css', async () => {
diff --git a/playground/glob-import/dir/index.js b/playground/glob-import/dir/index.js
index 63381e412ecc6f..94ca66f1017093 100644
--- a/playground/glob-import/dir/index.js
+++ b/playground/glob-import/dir/index.js
@@ -1,6 +1,10 @@
 const modules = import.meta.glob('./*.(js|ts)', { eager: true })
 const globWithAlias = import.meta.glob('@dir/al*.js', { eager: true })
 
+// test negative glob
+import.meta.glob(['@dir/*.js', '!@dir/x.js'])
+import.meta.glob(['!@dir/x.js', '@dir/*.js'])
+
 // test for sourcemap
 console.log('hello')