refactor: extract nitro logic from transformers (#1352)

Co-authored-by: Yaël Guilloux <yael.guilloux@gmail.com>

src/module.ts

@@ -225,14 +225,7 @@ export default defineNuxtModule<ModuleOptions>({
     const { resolve } = createResolver(import.meta.url)
     const resolveRuntimeModule = (path: string) => resolveModule(path, { paths: resolve('./runtime') })
     const contentContext: ContentContext = {
-      transformers: [
-        // Register internal content plugins
-        resolveRuntimeModule('./server/transformers/markdown'),
-        resolveRuntimeModule('./server/transformers/yaml'),
-        resolveRuntimeModule('./server/transformers/json'),
-        resolveRuntimeModule('./server/transformers/csv'),
-        resolveRuntimeModule('./server/transformers/path-meta')
-      ],
+      transformers: [],
       ...options
     }
 
@@ -313,7 +306,7 @@ export default defineNuxtModule<ModuleOptions>({
       nitroConfig.virtual['#content/virtual/transformers'] = [
         // TODO: remove kit usage
         templateUtils.importSources(contentContext.transformers),
-        `const transformers = [${contentContext.transformers.map(templateUtils.importName).join(', ')}]`,
+        `export const transformers = [${contentContext.transformers.map(templateUtils.importName).join(', ')}]`,
         'export const getParser = (ext) => transformers.find(p => ext.match(new RegExp(p.extensions.join("|"),  "i")) && p.parse)',
         'export const getTransformers = (ext) => transformers.filter(p => ext.match(new RegExp(p.extensions.join("|"),  "i")) && p.transform)',
         'export default () => {}'
@@ -398,7 +391,9 @@ export default defineNuxtModule<ModuleOptions>({
 
     // Register highlighter
     if (options.highlight) {
-      contentContext.transformers.push(resolveRuntimeModule('./server/transformers/shiki'))
+      contentContext.transformers.push(resolveRuntimeModule('./transformers/shiki'))
+      // @ts-ignore
+      contentContext.highlight.apiURL = `/api/${options.base}/highlight`
 
       nuxt.hook('nitro:config', (nitroConfig) => {
         nitroConfig.handlers = nitroConfig.handlers || []

src/runtime/server/index.ts

@@ -1,2 +1 @@
-export { serverQueryContent } from './storage'
-export { parseContent } from './transformers'
+export { serverQueryContent, parseContent } from './storage'

src/runtime/server/navigation.ts

@@ -1,5 +1,5 @@
 import { NavItem, ParsedContentMeta } from '../types'
-import { generateTitle } from './transformers/path-meta'
+import { generateTitle } from '../transformers/path-meta'
 import { useRuntimeConfig } from '#imports'
 
 type PrivateNavItem = NavItem & { path?: string }

src/runtime/server/storage.ts

@@ -2,13 +2,30 @@ import { prefixStorage } from 'unstorage'
 import { joinURL, withLeadingSlash, withoutTrailingSlash } from 'ufo'
 import { hash as ohash } from 'ohash'
 import type { CompatibilityEvent } from 'h3'
-import type { QueryBuilderParams, ParsedContent, QueryBuilder } from '../types'
+import defu from 'defu'
+import type { QueryBuilderParams, ParsedContent, QueryBuilder, ContentTransformer } from '../types'
 import { createQuery } from '../query/query'
 import { createPipelineFetcher } from '../query/match/pipeline'
-import { parseContent } from './transformers'
+import { transformContent } from '../transformers'
+import type { ModuleOptions } from '../../module'
 import { getPreview, isPreview } from './preview'
 // eslint-disable-next-line import/named
-import { useRuntimeConfig, useStorage } from '#imports'
+import { useNitroApp, useRuntimeConfig, useStorage } from '#imports'
+import { transformers as customTransformers } from '#content/virtual/transformers'
+
+interface ParseContentOptions {
+  csv?: ModuleOptions['csv']
+  yaml?: ModuleOptions['yaml']
+  highlight?: ModuleOptions['highlight']
+  markdown?: ModuleOptions['markdown']
+  transformers?: ContentTransformer[]
+  pathMeta?: {
+    locales?: ModuleOptions['locales']
+    defaultLocale?: ModuleOptions['defaultLocale']
+  }
+  // Allow passing options for custom transformers
+  [key: string]: any
+}
 
 export const sourceStorage = prefixStorage(useStorage(), 'content:source')
 export const cacheStorage = prefixStorage(useStorage(), 'cache:content')
@@ -127,13 +144,45 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise
     return { _id: contentId, body: null }
   }
 
-  const parsed = await parseContent(contentId, body as string)
+  const parsed = await parseContent(contentId, body as string) as ParsedContent
 
   await cacheParsedStorage.setItem(id, { parsed, hash }).catch(() => {})
 
   return parsed
 }
 
+/**
+ * Parse content file using registered plugins
+ */
+export async function parseContent (id: string, content: string, opts: ParseContentOptions = {}) {
+  const nitroApp = useNitroApp()
+  const options = defu(
+    opts,
+    {
+      markdown: contentConfig.markdown,
+      csv: contentConfig.csv,
+      yaml: contentConfig.yaml,
+      highlight: contentConfig.highlight,
+      transformers: customTransformers,
+      pathMeta: {
+        defaultLocale: contentConfig.defaultLocale,
+        locales: contentConfig.locales
+      }
+    }
+  )
+
+  // Call hook before parsing the file
+  const file = { _id: id, body: content }
+  await nitroApp.hooks.callHook('content:file:beforeParse', file)
+
+  const result = await transformContent(id, file.body, options)
+
+  // Call hook after parsing the file
+  await nitroApp.hooks.callHook('content:file:afterParse', result)
+
+  return result
+}
+
 export const createServerQueryFetch = <T = ParsedContent>(event: CompatibilityEvent, path?: string) => (query: QueryBuilder<T>) => {
   if (path) {
     if (query.params().first) {

src/runtime/transformers/csv.ts

@@ -0,0 +1,19 @@
+import { ParsedContent } from '../types'
+import { defineTransformer } from './utils'
+
+export default defineTransformer({
+  name: 'csv',
+  extensions: ['.csv'],
+  parse: async (_id, content, options = {}) => {
+    const csvToJson: any = await import('csvtojson').then(m => m.default || m)
+
+    const parsed = await csvToJson({ output: 'json', ...options })
+      .fromString(content)
+
+    return <ParsedContent> {
+      _id,
+      _type: 'csv',
+      body: parsed
+    }
+  }
+})

src/runtime/transformers/index.ts

@@ -0,0 +1,64 @@
+import { extname } from 'pathe'
+import { camelCase } from 'scule'
+import type { ContentTransformer, TransformContentOptions } from '../types'
+import csv from './csv'
+import markdown from './markdown'
+import yaml from './yaml'
+import pathMeta from './path-meta'
+import json from './json'
+import highlight from './shiki'
+
+const TRANSFORMERS = [
+  csv,
+  markdown,
+  json,
+  yaml,
+  pathMeta,
+  highlight
+]
+
+function getParser (ext, additionalTransformers: ContentTransformer[] = []): ContentTransformer {
+  let parser = additionalTransformers.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse)
+  if (!parser) {
+    parser = TRANSFORMERS.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse)
+  }
+
+  return parser
+}
+
+function getTransformers (ext, additionalTransformers: ContentTransformer[] = []) {
+  return [
+    ...additionalTransformers.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform),
+    ...TRANSFORMERS.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform)
+  ]
+}
+
+/**
+ * Parse content file using registered plugins
+ */
+export async function transformContent (id, content, options: TransformContentOptions = {}) {
+  const { transformers = [] } = options
+  // Call hook before parsing the file
+  const file = { _id: id, body: content }
+
+  const ext = extname(id)
+  const parser: ContentTransformer = getParser(ext, transformers)
+  if (!parser) {
+    // eslint-disable-next-line no-console
+    console.warn(`${ext} files are not supported, "${id}" falling back to raw content`)
+    return file
+  }
+
+  const parserOptions = options[camelCase(parser.name)] || {}
+  const parsed = await parser.parse!(file._id, file.body, parserOptions)
+
+  const matchedTransformers = getTransformers(ext, transformers)
+  const result = await matchedTransformers.reduce(async (prev, cur) => {
+    const next = (await prev) || parsed
+
+    const transformOptions = options[camelCase(cur.name)] || {}
+    return cur.transform!(next, transformOptions)
+  }, Promise.resolve(parsed))
+
+  return result
+}

src/runtime/server/transformers/json.ts → src/runtime/transformers/json.ts

@@ -1,10 +1,12 @@
 import destr from 'destr'
+import { ParsedContent } from '../types'
+import { defineTransformer } from './utils'
 
-export default {
+export default defineTransformer({
   name: 'Json',
   extensions: ['.json', '.json5'],
   parse: async (_id, content) => {
-    let parsed = content
+    let parsed
 
     if (typeof content === 'string') {
       if (_id.endsWith('json5')) {
@@ -24,10 +26,10 @@ export default {
       }
     }
 
-    return {
+    return <ParsedContent> {
       ...parsed,
       _id,
       _type: 'json'
     }
   }
-}
+})

src/runtime/server/transformers/markdown.ts → src/runtime/transformers/markdown.ts

@@ -1,13 +1,13 @@
-import { parse } from '../../markdown-parser'
-import type { MarkdownOptions, MarkdownPlugin } from '../../types'
-import { MarkdownParsedContent } from '../../types'
-import { useRuntimeConfig } from '#imports'
+import { parse } from '../markdown-parser'
+import type { MarkdownOptions, MarkdownPlugin } from '../types'
+import { MarkdownParsedContent } from '../types'
+import { defineTransformer } from './utils'
 
-export default {
+export default defineTransformer({
   name: 'markdown',
   extensions: ['.md'],
-  parse: async (_id, content) => {
-    const config: MarkdownOptions = { ...useRuntimeConfig().content?.markdown || {} }
+  parse: async (_id, content, options = {}) => {
+    const config = { ...options } as MarkdownOptions
     config.rehypePlugins = await importPlugins(config.rehypePlugins)
     config.remarkPlugins = await importPlugins(config.remarkPlugins)
 
@@ -20,7 +20,7 @@ export default {
       _id
     }
   }
-}
+})
 
 async function importPlugins (plugins: Record<string, false | MarkdownPlugin> = {}) {
   const resolvedPlugins = {}

src/runtime/server/transformers/path-meta.ts → src/runtime/transformers/path-meta.ts

@@ -1,8 +1,8 @@
 import { pascalCase } from 'scule'
 import slugify from 'slugify'
 import { withoutTrailingSlash, withLeadingSlash } from 'ufo'
-import { ParsedContentMeta } from '../../types'
-import { useRuntimeConfig } from '#imports'
+import { ParsedContent } from '../types'
+import { defineTransformer } from './utils'
 
 const SEMVER_REGEX = /^(\d+)(\.\d+)*(\.x)?$/
 
@@ -13,19 +13,19 @@ const describeId = (_id: string) => {
   parts[parts.length - 1] = filename
   const _path = parts.join('/')
 
-  return <Pick<ParsedContentMeta, '_source' | '_path' | '_extension' | '_file'>> {
+  return {
     _source,
     _path,
     _extension,
     _file: _extension ? `${_path}.${_extension}` : _path
   }
 }
 
-export default {
+export default defineTransformer({
   name: 'path-meta',
   extensions: ['.*'],
-  transform (content) {
-    const { locales, defaultLocale } = useRuntimeConfig().content || {}
+  transform (content, options: any = {}) {
+    const { locales = [], defaultLocale = 'en' } = options
     const { _source, _file, _path, _extension } = describeId(content._id)
     const parts = _path.split('/')
 
@@ -34,7 +34,7 @@ export default {
 
     const filePath = parts.join('/')
 
-    return <ParsedContentMeta> {
+    return <ParsedContent> {
       _path: generatePath(filePath),
       _draft: isDraft(filePath),
       _partial: isPartial(filePath),
@@ -47,7 +47,7 @@ export default {
       _extension
     }
   }
-}
+})
 
 /**
  * When file name ends with `.draft` then it will mark as draft.