Core, Server, Addon-docs: Replace story indexers with new API

code/addons/docs/src/preset.ts

@@ -3,7 +3,7 @@ import remarkSlug from 'remark-slug';
 import remarkExternalLinks from 'remark-external-links';
 import { dedent } from 'ts-dedent';
 
-import type { DocsOptions, Options, StorybookConfig, StoryIndexer } from '@storybook/types';
+import type { DocsOptions, Indexer, Options, StorybookConfig } from '@storybook/types';
 import type { CsfPluginOptions } from '@storybook/csf-plugin';
 import type { JSXOptions, CompileOptions } from '@storybook/mdx2-csf';
 import { global } from '@storybook/global';
@@ -129,23 +129,42 @@ async function webpack(
   return result;
 }
 
-const storyIndexers: StorybookConfig['storyIndexers'] = (existingIndexers) => {
-  const mdxIndexer: StoryIndexer['indexer'] = async (fileName, opts) => {
+export const createStoriesMdxIndexer = (legacyMdx1?: boolean): Indexer => ({
+  test: /(stories|story)\.mdx$/,
+  index: async (fileName, opts) => {
     let code = (await fs.readFile(fileName, 'utf-8')).toString();
-    const { compile } = global.FEATURES?.legacyMdx1
+    const { compile } = legacyMdx1
       ? await import('@storybook/mdx1-csf')
       : await import('@storybook/mdx2-csf');
     code = await compile(code, {});
-    return loadCsf(code, { ...opts, fileName }).parse();
-  };
-  return [
-    {
-      test: /(stories|story)\.mdx$/,
-      indexer: mdxIndexer,
-    },
-    ...(existingIndexers || []),
-  ];
-};
+    const csf = loadCsf(code, { ...opts, fileName }).parse();
+
+    // eslint-disable-next-line no-underscore-dangle
+    return Object.entries(csf._stories).map(([exportName, story]) => {
+      const docsOnly = story.parameters?.docsOnly;
+      const tags = (story.tags ?? csf.meta.tags ?? []).concat(
+        docsOnly ? 'stories-mdx-docsOnly' : []
+      );
+      // the mdx-csf compiler automatically adds the 'stories-mdx' tag to meta, here' we're just making sure it is always there
+      if (!tags.includes('stories-mdx')) {
+        tags.push('stories-mdx');
+      }
+      return {
+        type: 'story',
+        importPath: fileName,
+        exportName,
+        name: story.name,
+        title: csf.meta.title,
+        metaId: csf.meta.id,
+        tags,
+        __id: story.id,
+      };
+    });
+  },
+});
+
+const indexers: StorybookConfig['indexers'] = (existingIndexers) =>
+  [createStoriesMdxIndexer(global.FEATURES?.legacyMdx1)].concat(existingIndexers || []);
 
 const docs = (docsOptions: DocsOptions) => {
   return {
@@ -164,9 +183,9 @@ export const addons: StorybookConfig['addons'] = [
  * something down the dependency chain is using typescript namespaces, which are not supported by rollup-plugin-dts
  */
 const webpackX = webpack as any;
-const storyIndexersX = storyIndexers as any;
+const indexersX = indexers as any;
 const docsX = docs as any;
 
 ensureReactPeerDeps();
 
-export { webpackX as webpack, storyIndexersX as storyIndexers, docsX as docs };
+export { webpackX as webpack, indexersX as indexers, docsX as docs };

code/lib/cli/src/generators/SERVER/index.ts

@@ -3,7 +3,7 @@ import type { Generator } from '../types';
 
 const generator: Generator = async (packageManager, npmOptions, options) => {
   await baseGenerator(packageManager, npmOptions, options, 'server', {
-    extensions: ['json'],
+    extensions: ['json', 'yaml', 'yml'],
   });
 };
 

code/lib/core-server/package.json

@@ -98,6 +98,7 @@
     "ws": "^8.2.3"
   },
   "devDependencies": {
+    "@storybook/addon-docs": "workspace:*",
     "@types/compression": "^1.7.0",
     "@types/ip": "^1.1.0",
     "@types/node-fetch": "^2.5.7",

code/lib/core-server/src/presets/common-preset.ts

@@ -11,12 +11,12 @@ import {
 import type {
   CLIOptions,
   CoreConfig,
+  Indexer,
   Options,
   PresetPropertyFn,
   StorybookConfig,
-  StoryIndexer,
 } from '@storybook/types';
-import { loadCsf, printConfig, readConfig } from '@storybook/csf-tools';
+import { printConfig, readConfig, readCsf } from '@storybook/csf-tools';
 import { join } from 'path';
 import { dedent } from 'ts-dedent';
 import fetch from 'node-fetch';
@@ -194,20 +194,28 @@ export const features = async (
   legacyDecoratorFileOrder: false,
 });
 
-export const storyIndexers: StorybookConfig['storyIndexers'] = async (existingIndexers) => {
-  const csfIndexer: StoryIndexer['indexer'] = async (fileName, opts) => {
-    const code = (await readFile(fileName, 'utf-8')).toString();
-    return loadCsf(code, { ...opts, fileName }).parse();
-  };
-  return [
-    {
-      test: /(stories|story)\.(m?js|ts)x?$/,
-      indexer: csfIndexer,
-    },
-    ...(existingIndexers || []),
-  ];
+export const csfIndexer: Indexer = {
+  test: /\.stories\.(m?js|ts)x?$/,
+  index: async (fileName, options) => {
+    const csf = (await readCsf(fileName, options)).parse();
+
+    // eslint-disable-next-line no-underscore-dangle
+    return Object.entries(csf._stories).map(([exportName, story]) => ({
+      type: 'story',
+      importPath: fileName,
+      exportName,
+      name: story.name,
+      title: csf.meta.title,
+      metaId: csf.meta.id,
+      tags: story.tags ?? csf.meta.tags,
+      __id: story.id,
+    }));
+  },
 };
 
+export const indexers: StorybookConfig['indexers'] = (existingIndexers) =>
+  [csfIndexer].concat(existingIndexers || []);
+
 export const frameworkOptions = async (
   _: never,
   options: Options

code/lib/core-server/src/utils/StoryIndexGenerator.test.ts

@@ -6,17 +6,19 @@
  */
 
 import path from 'path';
-import fs from 'fs-extra';
+// @ts-expect-error -- cannot find declaration file
+import { createStoriesMdxIndexer } from '@storybook/addon-docs/preset';
 import { normalizeStoriesEntry } from '@storybook/core-common';
-import type { Indexer, NormalizedStoriesSpecifier, StoryIndexEntry } from '@storybook/types';
-import { loadCsf, getStorySortParameter } from '@storybook/csf-tools';
+import type { NormalizedStoriesSpecifier, StoryIndexEntry } from '@storybook/types';
+// import * as csfTools from '@storybook/csf-tools';
+import { readCsf, getStorySortParameter } from '@storybook/csf-tools';
 import { toId } from '@storybook/csf';
 import { logger, once } from '@storybook/node-logger';
 
 import type { StoryIndexGeneratorOptions } from './StoryIndexGenerator';
 import { StoryIndexGenerator } from './StoryIndexGenerator';
+import { csfIndexer } from '../presets/common-preset';
 
-jest.mock('@storybook/csf-tools');
 jest.mock('@storybook/csf', () => {
   const csf = jest.requireActual('@storybook/csf');
   return {
@@ -28,67 +30,32 @@ jest.mock('@storybook/csf', () => {
 jest.mock('@storybook/node-logger');
 
 const toIdMock = toId as jest.Mock<ReturnType<typeof toId>>;
-const loadCsfMock = loadCsf as jest.Mock<ReturnType<typeof loadCsf>>;
+jest.mock('@storybook/csf-tools', () => {
+  const csfTools = jest.requireActual('@storybook/csf-tools');
+  return {
+    ...csfTools,
+    readCsf: jest.fn(csfTools.readCsf),
+    getStorySortParameter: jest.fn(csfTools.getStorySortParameter),
+  };
+});
+
+const readCsfMock = readCsf as jest.Mock<ReturnType<typeof readCsf>>;
 const getStorySortParameterMock = getStorySortParameter as jest.Mock<
   ReturnType<typeof getStorySortParameter>
 >;
 
-const storiesMdxIndexer: Indexer = {
-  test: /\.stories\.mdx$/,
-  index: async (fileName, opts) => {
-    let code = (await fs.readFile(fileName, 'utf-8')).toString();
-    const { compile } = await import('@storybook/mdx2-csf');
-    code = await compile(code, {});
-    const csf = loadCsf(code, { ...opts, fileName }).parse();
-
-    // eslint-disable-next-line no-underscore-dangle
-    return Object.entries(csf._stories).map(([exportName, story]) => ({
-      type: 'story',
-      importPath: fileName,
-      exportName,
-      name: story.name,
-      title: csf.meta.title,
-      metaId: csf.meta.id,
-      tags: story.tags ?? csf.meta.tags,
-      __id: story.id,
-    }));
-  },
-};
-
-const csfIndexer: Indexer = {
-  test: /\.stories\.(m?js|ts)x?$/,
-  index: async (fileName, options) => {
-    const code = (await fs.readFile(fileName, 'utf-8')).toString();
-    const csf = loadCsf(code, { ...options, fileName }).parse();
-
-    // eslint-disable-next-line no-underscore-dangle
-    return Object.entries(csf._stories).map(([exportName, story]) => ({
-      type: 'story',
-      importPath: fileName,
-      exportName,
-      name: story.name,
-      title: csf.meta.title,
-      metaId: csf.meta.id,
-      tags: story.tags ?? csf.meta.tags,
-      __id: story.id,
-    }));
-  },
-};
-
 const options: StoryIndexGeneratorOptions = {
   configDir: path.join(__dirname, '__mockdata__'),
   workingDir: path.join(__dirname, '__mockdata__'),
   storyIndexers: [],
-  indexers: [csfIndexer, storiesMdxIndexer],
+  indexers: [csfIndexer, createStoriesMdxIndexer(false)],
   storiesV2Compatibility: false,
   storyStoreV7: true,
   docs: { defaultName: 'docs', autodocs: false },
 };
 
 describe('StoryIndexGenerator', () => {
   beforeEach(() => {
-    const actual = jest.requireActual('@storybook/csf-tools');
-    loadCsfMock.mockImplementation(actual.loadCsf);
     jest.mocked(logger.warn).mockClear();
     jest.mocked(once.warn).mockClear();
   });
@@ -1196,7 +1163,7 @@ describe('StoryIndexGenerator', () => {
       const generator = new StoryIndexGenerator([docsSpecifier, storiesSpecifier], options);
       await generator.initialize();
 
-      (getStorySortParameter as jest.Mock).mockReturnValueOnce({
+      getStorySortParameterMock.mockReturnValueOnce({
         order: ['docs2', 'D', 'B', 'nested', 'A', 'second-nested', 'first-nested/deeply'],
       });
 
@@ -1227,15 +1194,15 @@ describe('StoryIndexGenerator', () => {
           options
         );
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         const generator = new StoryIndexGenerator([specifier], options);
         await generator.initialize();
         await generator.getIndex();
-        expect(loadCsfMock).toHaveBeenCalledTimes(7);
+        expect(readCsfMock).toHaveBeenCalledTimes(7);
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         await generator.getIndex();
-        expect(loadCsfMock).not.toHaveBeenCalled();
+        expect(readCsfMock).not.toHaveBeenCalled();
       });
 
       it('does not extract docs files a second time', async () => {
@@ -1284,17 +1251,17 @@ describe('StoryIndexGenerator', () => {
           options
         );
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         const generator = new StoryIndexGenerator([specifier], options);
         await generator.initialize();
         await generator.getIndex();
-        expect(loadCsfMock).toHaveBeenCalledTimes(7);
+        expect(readCsfMock).toHaveBeenCalledTimes(7);
 
         generator.invalidate(specifier, './src/B.stories.ts', false);
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         await generator.getIndex();
-        expect(loadCsfMock).toHaveBeenCalledTimes(1);
+        expect(readCsfMock).toHaveBeenCalledTimes(1);
       });
 
       it('calls extract docs file for just the one file', async () => {
@@ -1369,17 +1336,17 @@ describe('StoryIndexGenerator', () => {
           options
         );
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         const generator = new StoryIndexGenerator([specifier], options);
         await generator.initialize();
         await generator.getIndex();
-        expect(loadCsfMock).toHaveBeenCalledTimes(7);
+        expect(readCsfMock).toHaveBeenCalledTimes(7);
 
         generator.invalidate(specifier, './src/B.stories.ts', true);
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         await generator.getIndex();
-        expect(loadCsfMock).not.toHaveBeenCalled();
+        expect(readCsfMock).not.toHaveBeenCalled();
       });
 
       it('does call the sort function a second time', async () => {
@@ -1408,11 +1375,11 @@ describe('StoryIndexGenerator', () => {
           options
         );
 
-        loadCsfMock.mockClear();
+        readCsfMock.mockClear();
         const generator = new StoryIndexGenerator([specifier], options);
         await generator.initialize();
         await generator.getIndex();
-        expect(loadCsfMock).toHaveBeenCalledTimes(7);
+        expect(readCsfMock).toHaveBeenCalledTimes(7);
 
         generator.invalidate(specifier, './src/B.stories.ts', true);
 

code/lib/core-server/src/utils/StoryIndexGenerator.ts

@@ -346,8 +346,12 @@ export class StoryIndexGenerator {
       });
     }
 
+    const entriesWithoutDocsOnlyStories = entries.filter(
+      (entry) => !(entry.type === 'story' && entry.tags.includes('stories-mdx-docsOnly'))
+    );
+
     return {
-      entries,
+      entries: entriesWithoutDocsOnlyStories,
       dependents: [],
       type: 'stories',
     };