feat!: normalize extraction content sources (#2719)

Co-authored-by: sibbng <sibbngheid@gmail.com>

bench/run.mjs

@@ -1,6 +1,6 @@
 /* eslint-disable no-console */
-import { execSync } from 'child_process'
-import { join } from 'path'
+import { execSync } from 'node:child_process'
+import { join } from 'node:path'
 import fs from 'fs-extra'
 import { escapeSelector } from '@unocss/core'
 import { dir, getVersions, targets } from './meta.mjs'

docs/guide/extracting.md

@@ -1,17 +1,57 @@
+---
+outline: deep
+---
+
 # Extracting
 
-Since UnoCSS works **at build time**, it means that only statically presented utilities will be generated and shipped to your app. Utilities that are used dynamically or fetched from external resources at runtime might not be applied.
+UnoCSS works by searching for the utilities usages from your codebase and generate the corresponding CSS on-demand. And we call this process **extracting**.
+
+## Content Sources
+
+UnoCSS supports extracting utilities usages from multiple sources:
+
+- [Pipeline](#extracting-from-build-tools-pipeline) - Extract right from your build tools pipeline
+- [Filesystem](#extracting-from-filesystem) - Extract from your filesystem by reading and watching files
+- [Inline](#extracting-from-inline-text) - Extract from inline plain text
+
+Usages of utilities that comes from different sources will be merged together and generate the final CSS.
+
+
+### Extracting from Build Tools Pipeline
+
+This is supported in the [Vite](/integrations/vite) and [Webpack](/integrations/webpack) integrations.
 
-By default, UnoCSS will extract the utilities usage from files in your build pipeline with extension `.jsx`, `.tsx`, `.vue`, `.md`, `.html`, `.svelte`, `.astro` and then generate the appropriate CSS on demand.
+UnoCSS will read the content that goes through your build tools pipeline and extract the utilities usages from them. This is the most efficient and accurate way to extract as we only extract the usages that are actually used in your app smartly, and no additional file IO is made during the extraction.
 
-`.js` and `.ts` files are **NOT included by default**.
+By default, UnoCSS will extract the utilities usage from files in your build pipeline with extension `.jsx`, `.tsx`, `.vue`, `.md`, `.html`, `.svelte`, `.astro` and then generate the appropriate CSS on demand. `.js` and `.ts` files are **NOT included by default**.
 
-You can add `@unocss-include`, per-file basis, anywhere in the file that you want UnoCSS to scan, or add `*.js` or `*.ts` in the configuration to include all js/ts files as scan targets.
+To configure them, you can update your `uno.config.ts`:
+
+```ts
+// uno.config.ts
+export default defineConfig({
+  content: {
+    pipeline: {
+      include: [
+        // the default
+        /\.(vue|svelte|[jt]sx|mdx?|astro|elm|php|phtml|html)($|\?)/,
+        // include js/ts files
+        'src/**/*.{js,ts}',
+      ],
+      // exclude files
+      // exclude: []
+    }
+  }
+})
+```
+
+You can also add `@unocss-include` magic comment, per-file basis, anywhere in the file that you want UnoCSS to scan, or add `*.js` or `*.ts` in the configuration to include all js/ts files as scan targets.
 
 ```ts
 // ./some-utils.js
 
-// since `.js` files are not included by default, the following comment tells UnoCSS to force scan this file.
+// since `.js` files are not included by default,
+// the following comment tells UnoCSS to force scan this file.
 // @unocss-include
 export const classes = {
   active: 'bg-primary text-white',
@@ -21,7 +61,52 @@ export const classes = {
 
 Similarly, you can also add `@unocss-ignore` to bypass the scanning and transforming for a file.
 
-## Safelist
+### Extracting from Filesystem
+
+In cases that you are using integrations that does not have access to the build tools pipeline (for example, the [PostCSS](/integrations/postcss) plugin), or you are integrating with backend frameworks that the code does not go through the pipeline, you can manually specify the files to be extracted.
+
+```ts
+// uno.config.ts
+export default defineConfig({
+  content: {
+    filesystem: [
+      'src/**/*.php',
+      'public/*.html',
+    ]
+  }
+})
+```
+
+The files matched will be read directly from the filesystem and watched for changes at dev mode.
+
+### Extracting from Inline Text
+
+Additionally, you can also extract utilities usages from inline text, that you might retrieve from else where.
+
+You may also pass an async function to return the content. But note that the function will only be called once at the build time.
+
+```ts
+// uno.config.ts
+export default defineConfig({
+  content: {
+    inline: [
+      // plain text
+      '<div class="p-4 text-red">Some text</div>',
+      // async getter
+      async () => {
+        const response = await fetch('https://example.com')
+        return response.text()
+      }
+    ]
+  }
+})
+```
+
+## Limitations
+
+Since UnoCSS works **at build time**, it means that only statically presented utilities will be generated and shipped to your app. Utilities that are used dynamically or fetched from external resources at runtime might **NOT** be applied.
+
+### Safelist
 
 Sometimes you might want to use dynamic concatenations like:
 
@@ -56,7 +141,7 @@ safelist: [
 
 If you are seeking for a true dynamic generation at runtime, you may want to check out the [@unocss/runtime](https://github.com/unocss/unocss/tree/main/packages/runtime) package.
 
-## Blocklist
+### Blocklist
 
 Similar to `safelist`, you can also configure `blocklist` to exclude some utilities from being generated. Different from `safelist`, `blocklist` accept both string for exact match and regex for pattern match.
 

docs/integrations/astro.md

@@ -111,4 +111,4 @@ If you are building a meta framework on top of UnoCSS, see [this file](https://g
 
 ## Notes
 
-[`client:only`](https://docs.astro.build/en/reference/directives-reference/#clientonly) components must be placed in [`components`](https://docs.astro.build/en/core-concepts/project-structure/#srccomponents) folder or added to UnoCSS's `extraContent` config in order to be processed.
+[`client:only`](https://docs.astro.build/en/reference/directives-reference/#clientonly) components must be placed in [`components`](https://docs.astro.build/en/core-concepts/project-structure/#srccomponents) folder or added to UnoCSS's `content` config in order to be processed.

docs/integrations/postcss.md

@@ -23,10 +23,7 @@ npm i -D @unocss/postcss
 // postcss.config.cjs
 module.exports = {
   plugins: {
-    '@unocss/postcss': {
-      // Optional
-      content: ['**/*.{html,js,ts,jsx,tsx,vue,svelte,astro}'],
-    },
+    '@unocss/postcss': {},
   },
 }
 ```
@@ -36,6 +33,11 @@ module.exports = {
 import { defineConfig, presetUno } from 'unocss'
 
 export default defineConfig({
+  content: {
+    filesystem: [
+      '**/*.{html,js,ts,jsx,tsx,vue,svelte,astro}',
+    ]
+  },
   presets: [
     presetUno(),
   ],

packages/astro/src/index.ts

@@ -85,9 +85,10 @@ export default function UnoCSSAstroIntegration<Theme extends {}>(
     hooks: {
       'astro:config:setup': async ({ config, updateConfig, injectScript }) => {
         // Adding components to UnoCSS's extra content
-        options.extraContent ||= {}
-        options.extraContent.filesystem ||= []
-        options.extraContent.filesystem.push(resolve(fileURLToPath(config.srcDir), 'components/**/*').replace(/\\/g, '/'))
+        const source = resolve(fileURLToPath(config.srcDir), 'components/**/*').replace(/\\/g, '/')
+        options.content ||= {}
+        options.content.filesystem ||= []
+        options.content.filesystem.push(source)
 
         const injects: string[] = []
         if (injectReset) {

packages/core/src/types.ts

@@ -645,17 +645,56 @@ export interface SourceCodeTransformer {
   transform: (code: MagicString, id: string, ctx: UnocssPluginContext) => Awaitable<void>
 }
 
-export interface ExtraContentOptions {
+export interface ContentOptions {
   /**
-   * Glob patterns to match the files to be extracted
-   * In dev mode, the files will be watched and trigger HMR
+   * Glob patterns to extract from the file system, in addition to other content sources.
+   *
+   * In dev mode, the files will be watched and trigger HMR.
+   *
+   * @default []
    */
   filesystem?: string[]
 
   /**
-   * Plain text to be extracted
+   * Inline text to be extracted
+   */
+  inline?: (string | { code: string; id?: string } | (() => Awaitable<string | { code: string; id?: string }>)) []
+
+  /**
+   * Filters to determine whether to extract certain modules from the build tools' transformation pipeline.
+   *
+   * Currently only works for Vite and Webpack integration.
+   *
+   * Set `false` to disable.
+   */
+  pipeline?: false | {
+    /**
+     * Patterns that filter the files being extracted.
+     * Supports regular expressions and `picomatch` glob patterns.
+     *
+     * By default, `.ts` and `.js` files are NOT extracted.
+     *
+     * @see https://www.npmjs.com/package/picomatch
+     * @default [/\.(vue|svelte|[jt]sx|mdx?|astro|elm|php|phtml|html)($|\?)/]
+     */
+    include?: FilterPattern
+
+    /**
+     * Patterns that filter the files NOT being extracted.
+     * Supports regular expressions and `picomatch` glob patterns.
+     *
+     * By default, `node_modules` and `dist` are also extracted.
+     *
+     * @see https://www.npmjs.com/package/picomatch
+     * @default [/\.(css|postcss|sass|scss|less|stylus|styl)($|\?)/]
+     */
+    exclude?: FilterPattern
+  }
+
+  /**
+   * @deprecated Renamed to `inline`
    */
-  plain?: string[]
+  plain?: (string | { code: string; id?: string }) []
 }
 
 /**
@@ -675,24 +714,40 @@ export interface PluginOptions {
   configDeps?: string[]
 
   /**
-   * Patterns that filter the files being extracted.
+   * Custom transformers to the source code
    */
-  include?: FilterPattern
+  transformers?: SourceCodeTransformer[]
 
   /**
-   * Patterns that filter the files NOT being extracted.
+   * Options for sources to be extracted as utilities usages
+   *
+   * Supported sources:
+   * - `filesystem` - extract from file system
+   * - `plain` - extract from plain inline text
+   * - `pipeline` - extract from build tools' transformation pipeline, such as Vite and Webpack
+   *
+   * The usage extracted from each source will be **merged** together.
    */
-  exclude?: FilterPattern
+  content?: ContentOptions
+
+  /** ========== DEPRECATED OPTIONS ========== **/
 
   /**
-   * Custom transformers to the source code
+   * @deprecated Renamed to `content`
    */
-  transformers?: SourceCodeTransformer[]
+  extraContent?: ContentOptions
 
   /**
-   * Extra content outside of build pipeline (assets, backend, etc.) to be extracted
+   * Patterns that filter the files being extracted.
+   * @deprecated moved to `content.pipeline.include`
    */
-  extraContent?: ExtraContentOptions
+  include?: FilterPattern
+
+  /**
+    * Patterns that filter the files NOT being extracted.
+    * @deprecated moved to `content.pipeline.exclude`
+    */
+  exclude?: FilterPattern
 }
 
 export interface UserConfig<Theme extends {} = {}> extends ConfigBase<Theme>, UserOnlyOptions<Theme>, GeneratorOptions, PluginOptions, CliOptions {}

packages/nuxt/src/options.ts

@@ -5,7 +5,7 @@ import presetWebFonts from '@unocss/preset-web-fonts'
 import presetTypography from '@unocss/preset-typography'
 import presetTagify from '@unocss/preset-tagify'
 import presetWind from '@unocss/preset-wind'
-import { defaultExclude } from '../../shared-integration/src/defaults'
+import { defaultPipelineExclude } from '../../shared-integration/src/defaults'
 import type { UnocssNuxtOptions } from './types'
 
 export function resolveOptions(options: UnocssNuxtOptions) {
@@ -26,8 +26,13 @@ export function resolveOptions(options: UnocssNuxtOptions) {
         options.presets.push(preset(typeof option === 'boolean' ? {} : option))
     }
   }
-  options.exclude = options.exclude || defaultExclude
-  // ignore macro files created by Nuxt
-  if (Array.isArray(options.exclude))
-    options.exclude.push(/\?macro=true/)
+
+  options.content ??= {}
+  options.content.pipeline ??= {}
+  if (options.content.pipeline !== false) {
+    options.content.pipeline.exclude ??= defaultPipelineExclude
+    if (Array.isArray(options.content.pipeline.exclude))
+    // ignore macro files created by Nuxt
+      options.content.pipeline.exclude.push(/\?macro=true/)
+  }
 }

packages/postcss/src/index.ts

@@ -6,7 +6,7 @@ import type { Result, Root } from 'postcss'
 import postcss from 'postcss'
 import { createGenerator, warnOnce } from '@unocss/core'
 import { loadConfig } from '@unocss/config'
-import { defaultIncludeGlobs } from '../../shared-integration/src/defaults'
+import { defaultFilesystemGlobs } from '../../shared-integration/src/defaults'
 import { parseApply } from './apply'
 import { parseTheme, themeFnRE } from './theme'
 import { parseScreen } from './screen'
@@ -108,11 +108,8 @@ function unocss(options: UnoPostcssPluginOptions = {}) {
           throw new Error (`UnoCSS config not found: ${error.message}`)
         }
 
-        const globs = content?.filter(v => typeof v === 'string') as string[] ?? defaultIncludeGlobs
-        const rawContent = content?.filter(v => typeof v === 'object') as {
-          raw: string
-          extension: string
-        }[] ?? []
+        const globs = uno.config.content?.filesystem ?? defaultFilesystemGlobs
+        const plainContent = uno.config.content?.inline ?? []
 
         const entries = await fg(isScanTarget ? globs : from, {
           cwd,
@@ -126,10 +123,12 @@ function unocss(options: UnoPostcssPluginOptions = {}) {
         await parseScreen(root, uno, directiveMap.screen)
 
         promises.push(
-          ...rawContent.map(async (v) => {
-            const { matched } = await uno.generate(v.raw, {
-              id: `unocss.${v.extension}`,
-            })
+          ...plainContent.map(async (c, idx) => {
+            if (typeof c === 'function')
+              c = await c()
+            if (typeof c === 'string')
+              c = { code: c }
+            const { matched } = await uno.generate(c.code, { id: c.id ?? `__plain_content_${idx}__` })
 
             for (const candidate of matched)
               classes.add(candidate)

packages/shared-integration/src/extra-content.ts → packages/shared-integration/src/content.ts

@@ -4,22 +4,26 @@ import fg from 'fast-glob'
 import type { UnocssPluginContext } from '@unocss/core'
 import { applyTransformers } from './transformers'
 
-export async function setupExtraContent(ctx: UnocssPluginContext, shouldWatch = false) {
-  const { extraContent } = await ctx.getConfig()
+export async function setupContentExtractor(ctx: UnocssPluginContext, shouldWatch = false) {
+  const { content } = await ctx.getConfig()
   const { extract, tasks, root, filter } = ctx
 
-  // plain text
-  if (extraContent?.plain) {
+  // inline text
+  if (content?.inline) {
     await Promise.all(
-      extraContent.plain.map((code, idx) => {
-        return extract(code, `__extra_content_${idx}__`)
+      content.inline.map(async (c, idx) => {
+        if (typeof c === 'function')
+          c = await c()
+        if (typeof c === 'string')
+          c = { code: c }
+        return extract(c.code, c.id ?? `__plain_content_${idx}__`)
       }),
     )
   }
 
   // filesystem
-  if (extraContent?.filesystem) {
-    const files = await fg(extraContent.filesystem, { cwd: root })
+  if (content?.filesystem) {
+    const files = await fg(content.filesystem, { cwd: root })
 
     async function extractFile(file: string) {
       const code = await fs.readFile(file, 'utf-8')