feat: more flexible ignores configuration (#2022)

Co-authored-by: Farnabaz <farnabaz@gmail.com>

docs/content/4.api/3.configuration.md

@@ -120,13 +120,42 @@ It is highly recommended to prevent modifying default source. If you want to loa
 - Type: `string[] | object[]`{lang=ts}
 - Default: `['\\.', '-']`{lang=ts}
 
-List of ignore pattern that will be used for excluding content from parsing and rendering.
+List of ignore patterns to exclude content from parsing, rendering and watching.
+
+Note that under the hood:
+
+- paths are managed by [unstorage](https://unstorage.unjs.io/usage#usage-1), so you will need to use `:` in place of `/`
+- patterns are converted to Regular Expressions, so you will need to escape appropriately
+- patterns are actually _prefixes_; use a match-all pattern to negate this (see final example) 
+
+```ts [nuxt.config.ts]
+export default defineNuxtConfig({
+  content: {
+    ignores: [
+      'hidden',        // any file or folder prefixed with the word "hidden"
+      'hidden:',       // any folder that exactly matches the word "hidden"
+      'path:to:file',  // any file path matching "/path/to/file"
+      '.+\\.bak$',     // any file with the extension ".bak"
+    ]
+  }
+})
+```
+
+From version 2.7 `ignores` will support a more flexible, idiomatic format.
+
+You can use this from version 2.6 using the `experimental.advancedIgnoresPattern` flag:
 
 ```ts [nuxt.config.ts]
 export default defineNuxtConfig({
   content: {
+    experimental: {
+      advancedIgnoresPattern: true
+    },
     ignores: [
-      'ghost'
+      'hidden',        // any file or folder that contains the word "hidden"
+      '/hidden/',      // any folder that exactly matches the word "hidden"
+      '/path/to/file', // any file path matching "/path/to/file"
+      '\\.bak$',       // any file with the extension ".bak"
     ]
   }
 })

playground/basic/content/2.features/7.ignored.md

@@ -0,0 +1,14 @@
+# Ignore
+
+The `/ignore` folder contains examples of ignored files.
+
+Navigating to these routes should result in a 404, and reloading any of these files should not trigger a content refresh:
+
+- [.dotted file](/features/ignored/dotted-file)
+- [-dashed file](/features/ignored/dashed-file)
+- [ignored file](/features/ignored/ignored-file)
+- [ignored folder](/features/ignored/folder)
+
+Navigating to these routes should show their content:
+
+- [included file](/features/ignored/included-file)

playground/basic/content/2.features/ignored/-dashed-file.txt

@@ -0,0 +1,4 @@
+This file is ignored:
+
+- reason: default `ignores` pattern `^[.-]|:[.-]`
+- because: it matches the leading -

playground/basic/content/2.features/ignored/.dotted-file.txt

@@ -0,0 +1,4 @@
+This file is ignored:
+
+- reason: default `ignores` pattern `^[.-]|:[.-]`
+- because: it matches the leading .

playground/basic/content/2.features/ignored/folder/index.md

@@ -0,0 +1,4 @@
+This folder is ignored:
+
+- reason: custom `ignores` pattern `ignored:hidden`
+- because: it matches the folder path

playground/basic/content/2.features/ignored/ignored-file.bak

@@ -0,0 +1,4 @@
+This file is ignored:
+
+- reason: custom `ignores` pattern `\\.bak$`
+- because: it matches the file extension

playground/basic/content/2.features/ignored/included-file.md

@@ -0,0 +1,5 @@
+---
+navigation: false
+---
+
+This file is included

playground/basic/nuxt.config.ts

@@ -9,7 +9,16 @@ export default defineNuxtConfig({
         driver: 'fs',
         base: resolve(__dirname, 'content-fa')
       }
-    }
+    },
+
+    experimental: {
+      advancedIgnoresPattern: true
+    },
+
+    ignores: [
+      '\\.bak$',
+      'ignored/folder'
+    ]
   },
   typescript: {
     includeWorkspace: true

playground/basic/pages/[...slug].vue

@@ -2,10 +2,15 @@
   <div class="content-page">
     <ContentDoc>
       <template #not-found>
-        Back
-        <NuxtLink to="/">
-          Home page
-        </NuxtLink>
+        <p>Not found!</p>
+        <p>
+          Go:
+          <NuxtLink to="#" @click="$router.go(-1)">
+            Back</NuxtLink>
+          or
+          <NuxtLink to="/">
+            Home</NuxtLink>
+        </p>
       </template>
     </ContentDoc>
   </div>

src/module.ts

@@ -22,6 +22,7 @@ import { createStorage } from 'unstorage'
 import { joinURL, withLeadingSlash, withTrailingSlash } from 'ufo'
 import type { Component } from '@nuxt/schema'
 import { name, version } from '../package.json'
+import { makeIgnored } from './runtime/utils/config'
 import {
   CACHE_VERSION,
   createWebSocket,
@@ -67,16 +68,18 @@ export interface ModuleOptions {
     ws: Partial<ListenOptions>
   }
   /**
-   * Contents can located in multiple places, in multiple directories or even in remote git repositories.
+   * Contents can be located in multiple places, in multiple directories or even in remote git repositories.
    * Using sources option you can tell Content module where to look for contents.
    *
    * @default ['content']
    */
   sources: Record<string, MountOptions> | Array<string | MountOptions>
   /**
-   * List of ignore pattern that will be used for excluding content from parsing and rendering.
+   * List of ignore patterns that will be used to exclude content from parsing, rendering and watching.
    *
-   * @default ['\\.', '-']
+   * Note that files with a leading . or - are ignored by default
+   *
+   * @default []
    */
   ignores: Array<string>
   /**
@@ -219,7 +222,8 @@ export interface ModuleOptions {
   },
   experimental: {
     clientDB: boolean
-    stripQueryParameters: boolean
+    stripQueryParameters: boolean,
+    advancedIgnoresPattern: boolean
   }
 }
 
@@ -258,7 +262,7 @@ export default defineNuxtModule<ModuleOptions>({
       }
     },
     sources: {},
-    ignores: ['\\.', '-'],
+    ignores: [],
     locales: [],
     defaultLocale: undefined,
     highlight: false,
@@ -280,7 +284,8 @@ export default defineNuxtModule<ModuleOptions>({
     documentDriven: false,
     experimental: {
       clientDB: false,
-      stripQueryParameters: false
+      stripQueryParameters: false,
+      advancedIgnoresPattern: false
     }
   },
   async setup (options, nuxt) {
@@ -600,7 +605,8 @@ export default defineNuxtModule<ModuleOptions>({
       integrity: buildIntegrity,
       experimental: {
         stripQueryParameters: options.experimental.stripQueryParameters,
-        clientDB: options.experimental.clientDB && nuxt.options.ssr === false
+        clientDB: options.experimental.clientDB && nuxt.options.ssr === false,
+        advancedIgnoresPattern: options.experimental.advancedIgnoresPattern
       },
       api: {
         baseURL: options.api.baseURL
@@ -632,6 +638,13 @@ export default defineNuxtModule<ModuleOptions>({
       tailwindConfig.content.push(resolve(nuxt.options.buildDir, 'content-cache', 'parsed/**/*.md'))
     })
 
+    // ignore files
+    const { advancedIgnoresPattern } = contentContext.experimental
+    const isIgnored = makeIgnored(contentContext.ignores, advancedIgnoresPattern)
+    if (contentContext.ignores.length && !advancedIgnoresPattern) {
+      logger.warn('The `ignores` config is being made more flexible in version 2.7. See the docs for more information: `https://content.nuxtjs.org/api/configuration#ignores`')
+    }
+
     // Setup content dev module
     if (!nuxt.options.dev) {
       nuxt.hook('build:before', async () => {
@@ -648,11 +661,8 @@ export default defineNuxtModule<ModuleOptions>({
 
         // Filter invalid characters & ignore patterns
         const invalidKeyCharacters = "'\"?#/".split('')
-        const contentIgnores: Array<RegExp> = contentContext.ignores.map((p: any) =>
-          typeof p === 'string' ? new RegExp(`^${p}|:${p}`) : p
-        )
         keys = keys.filter((key) => {
-          if (key.startsWith('preview:') || contentIgnores.some(prefix => prefix.test(key))) {
+          if (key.startsWith('preview:') || isIgnored(key)) {
             return false
           }
           if (invalidKeyCharacters.some(ik => key.includes(ik))) {
@@ -699,7 +709,7 @@ export default defineNuxtModule<ModuleOptions>({
       // Watch contents
       await nitro.storage.watch(async (event: WatchEvent, key: string) => {
         // Ignore events that are not related to content
-        if (!key.startsWith(MOUNT_PREFIX)) {
+        if (!key.startsWith(MOUNT_PREFIX) || isIgnored(key)) {
           return
         }
         key = key.substring(MOUNT_PREFIX.length)
@@ -718,6 +728,7 @@ interface ModulePublicRuntimeConfig {
   experimental: {
     stripQueryParameters: boolean
     clientDB: boolean
+    advancedIgnoresPattern: boolean
   }
 
   defaultLocale: ModuleOptions['defaultLocale']

src/runtime/server/storage.ts

@@ -8,6 +8,7 @@ import type { QueryBuilderParams, ParsedContent, QueryBuilder, ContentTransforme
 import { createQuery } from '../query/query'
 import { createPipelineFetcher } from '../query/match/pipeline'
 import { transformContent } from '../transformers'
+import { makeIgnored } from '../utils/config'
 import type { ModuleOptions } from '../../module'
 import { getPreview, isPreview } from './preview'
 import { getIndexedContentsList } from './content-index'
@@ -41,9 +42,7 @@ const contentConfig = useRuntimeConfig().content
 /**
  * Content ignore patterns
  */
-export const contentIgnores: Array<RegExp> = contentConfig.ignores.map((p: any) =>
-  typeof p === 'string' ? new RegExp(`^${p}|:${p}`) : p
-)
+const isIgnored = makeIgnored(contentConfig.ignores, contentConfig.experimental.advancedIgnoresPattern)
 
 /**
  * Invalid key characters
@@ -54,7 +53,7 @@ const invalidKeyCharacters = "'\"?#/".split('')
  * Filter predicate for ignore patterns
  */
 const contentIgnorePredicate = (key: string) => {
-  if (key.startsWith('preview:') || contentIgnores.some(prefix => prefix.test(key))) {
+  if (key.startsWith('preview:') || isIgnored(key)) {
     return false
   }
   if (invalidKeyCharacters.some(ik => key.includes(ik))) {

src/runtime/utils/config.ts

@@ -0,0 +1,22 @@
+/**
+ * Creates a predicate to test storage keys `foo:bar:baz` against configured `ignores` patterns
+ */
+export function makeIgnored (ignores: string[], experimental = false): (key: string) => boolean {
+  // filter empty
+  ignores = ignores.map(e => e)
+
+  // 2.7+ supports free regexp + slashes
+  if (experimental) {
+    const rxAll = ['/\\.', '/-', ...ignores].map(p => new RegExp(p))
+    return function isIgnored (key: string): boolean {
+      const path = '/' + key.replaceAll(':', '/')
+      return rxAll.some(rx => rx.test(path))
+    }
+  }
+
+  // 2.6 prefixed by unstorage delimiters
+  const rxAll = ['\\.', '-', ...ignores].map(p => new RegExp(`^${p}|:${p}`))
+  return function isIgnored (key: string): boolean {
+    return rxAll.some(rx => rx.test(key))
+  }
+}

test/basic.test.ts

@@ -18,6 +18,7 @@ import { testMarkdownRenderer } from './features/renderer-markdown'
 import { testParserOptions } from './features/parser-options'
 import { testComponents } from './features/components'
 import { testLocales } from './features/locales'
+import { testIgnores } from './features/ignores'
 
 // const spyConsoleWarn = vi.spyOn(global.console, 'warn')
 
@@ -86,4 +87,6 @@ describe('Basic usage', async () => {
   testHighlighter()
 
   testParserOptions()
+
+  testIgnores()
 })

test/features/ignores.ts

@@ -0,0 +1,64 @@
+import { describe, test, expect } from 'vitest'
+import { makeIgnored } from '../../src/runtime/utils/config'
+
+function run (key: string, pattern: string, result: boolean, experimental: boolean) {
+  const predicate = makeIgnored([pattern], experimental)
+  const label: string = experimental ? 'new format' : 'old format'
+  test(label, () => {
+    expect(predicate(key)).toBe(result)
+  })
+}
+
+function runOld (pattern: string, key: string, result: boolean) {
+  return run(key, pattern, result, false)
+}
+
+function runNew (pattern: string, key: string, result: boolean) {
+  return run(key, pattern, result, true)
+}
+
+export const testIgnores = () => {
+  describe('Ignores Options', () => {
+    describe('any file or folder that contains the word "hidden"', () => {
+      const key = 'source:content:my-hidden-folder:index.md'
+      runOld('.+hidden', key, true)
+      runNew('hidden', key, true)
+    })
+
+    describe('any file or folder prefixed with the word "hidden"', () => {
+      const key = 'source:content:hidden-folder:index.md'
+      runOld('hidden', key, true)
+      runNew('/hidden', key, true)
+    })
+
+    describe('any folder that exactly matches the word "hidden"', () => {
+      const key = 'source:content:hidden:index.md'
+      runOld('hidden:', key, true)
+      runNew('/hidden/', key, true)
+    })
+
+    describe('any file path matching "/path/to/file"', () => {
+      const key = 'source:content:path:to:file.md'
+      runOld('path:to:file', key, true)
+      runNew('/path/to/file', key, true)
+    })
+
+    describe('any file with the extension ".bak"', () => {
+      const key = 'source:content:path:to:ignored.bak'
+      runOld('.+\\.bak$', key, true)
+      runNew('\\.bak$', key, true)
+    })
+
+    describe('any file with a leading dash', () => {
+      const key = 'source:content:path:to:-dash.txt'
+      runOld('', key, true)
+      runNew('', key, true)
+    })
+
+    describe('any file with a leading dot', () => {
+      const key = 'source:content:path:to:.dot.txt'
+      runOld('', key, true)
+      runNew('', key, true)
+    })
+  })
+}