chore: add a docs report actions for PRs (#4889)

* chore(docs): add a way to extract doc information from definition file

* chore(docs): build up a list of mutations for docs information, keep version info

* chore: add a docs report actions for PRs

* chore: add detailed summary to comment

* chore: code cleanup and organization

* refactor(doc-report): use a local copy of envVars for docs report

* chore(docs): only show affected documentation version

- Rename table header to more explicit

---------

Co-authored-by: Bjørge Næss <bjoerge@gmail.com>

.github/workflows/docReport.yml

@@ -0,0 +1,69 @@
+name: Create Documentation Report
+
+on:
+  # Build on pushes branches that have a PR (including drafts)
+  pull_request:
+  # Build on commits pushed to branches without a PR if it's in the allowlist
+  push:
+    branches: [current]
+
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+      TURBO_TEAM: sanity-io
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-modules-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-modules-${{ env.cache-name }}-
+            ${{ runner.os }}-modules-
+            ${{ runner.os }}-
+
+      - name: Install project dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Build packages
+        run: yarn build
+        env:
+          NODE_OPTIONS: --max_old_space_size=8192
+
+      - name: Create Docs Report on current
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/current' }}
+        env:
+          DOCS_REPORT_TOKEN: ${{ secrets.DOCS_REPORT_DATASET_TOKEN }}
+          DOCS_REPORT_DATASET: production
+        run: yarn docs:report:create
+
+      - name: Create Docs Report on PR
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          DOCS_REPORT_TOKEN: ${{ secrets.DOCS_REPORT_DATASET_TOKEN }}
+          DOCS_REPORT_DATASET: ${{ github.ref_name }}
+        run: yarn docs:report:create
+
+      - name: Compare Docs Coverage on PR
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          DOCS_REPORT_TOKEN: ${{ secrets.DOCS_REPORT_DATASET_TOKEN }}
+          DOCS_REPORT_DATASET: ${{ github.ref_name }}
+        run: yarn docs:report
+
+      - name: PR comment with report
+        uses: thollander/actions-comment-pull-request@d61db783da9abefc3437960d0cce08552c7c004f # v2
+        if: ${{ github.event_name == 'pull_request' }}
+        with:
+          comment_tag: 'docs-report'
+          filePath: ${{ github.workspace }}/scripts/docs-report.md

.github/workflows/docReportTeardown.yml

@@ -0,0 +1,40 @@
+name: 'Cleanup: Documentation Report'
+
+on:
+  # Build on closed or merged PRs
+  pull_request:
+    types: [closed]
+
+jobs:
+  reportTeardown:
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+      TURBO_TEAM: sanity-io
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: '**/node_modules'
+          key: ${{ runner.os }}-modules-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-modules-${{ env.cache-name }}-
+            ${{ runner.os }}-modules-
+            ${{ runner.os }}-
+
+      - name: Install project dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: yarn install --frozen-lockfile
+
+      - name: Remove datasets for closed PRs
+        env:
+          PERF_TEST_METRICS_TOKEN: ${{ secrets.PERF_TEST_SANITY_TOKEN }}
+          DOCS_REPORT_DATASET: $GITHUB_REF_NAME
+        run: yarn docs:report:cleanup

.gitignore

@@ -76,3 +76,6 @@ etc
 .yarn
 .yarnrc
 .turbo
+
+## Documentation Report
+scripts/docs-report.md

package.json

@@ -31,6 +31,9 @@
     "dev:starter-studio": "yarn --cwd dev/starter-studio dev",
     "dev:strict-studio": "yarn --cwd dev/strict-studio dev",
     "dev:test-studio": "yarn --cwd dev/test-studio dev",
+    "docs:report": "node -r dotenv-flow/config -r esbuild-register scripts/doc-report/docReport",
+    "docs:report:create": "node -r dotenv-flow/config -r esbuild-register scripts/doc-report/docReportCreate",
+    "docs:report:cleanup": "node -r dotenv-flow/config -r esbuild-register scripts/doc-report/docReportCleanup",
     "eslint": "eslint --ext=.cjs,.js,.jsx,.mjs,.ts,.tsx --quiet",
     "example:blog-studio": "yarn --cwd examples/blog-studio start",
     "example:clean-studio": "yarn --cwd examples/clean-studio start",
@@ -83,6 +86,7 @@
     "@babel/preset-env": "^7.19.1",
     "@babel/preset-react": "^7.18.6",
     "@babel/preset-typescript": "^7.18.6",
+    "@bjoerge/mutiny": "^0.0.2",
     "@optimize-lodash/rollup-plugin": "^4.0.1",
     "@playwright/test": "^1.37.0",
     "@sanity/client": "^6.4.9",
@@ -131,6 +135,7 @@
     "prettier-plugin-packagejson": "^2.4.5",
     "pretty-quick": "^3.1.3",
     "rimraf": "^3.0.2",
+    "rxjs": "^7.8.1",
     "semver": "^7.3.5",
     "turbo": "^1.10.12",
     "typescript": "^5.1.6",

scripts/doc-report/docClient.ts

@@ -0,0 +1,13 @@
+import {type SanityClient, createClient} from '@sanity/client'
+import {sanityIdify} from '../utils/sanityIdify'
+import {readEnv} from './envVars'
+
+export function createDocClient(dataset: string): SanityClient {
+  return createClient({
+    projectId: 'c1zuxvqn',
+    dataset: sanityIdify(dataset),
+    token: readEnv('DOCS_REPORT_TOKEN'),
+    apiVersion: '2023-02-03',
+    useCdn: false,
+  })
+}

scripts/doc-report/docReport.ts

@@ -0,0 +1,143 @@
+import fs from 'fs/promises'
+import path from 'path'
+import {groupBy} from 'lodash'
+import {combineLatest, map} from 'rxjs'
+import {startTimer} from '../utils/startTimer'
+import {createDocClient} from './docClient'
+import {readEnv} from './envVars'
+
+const QUERY = `*[_type=='exportSymbol'] {
+  _id,
+  "package": exportedBy->normalized,
+  "isDocumented": defined(versions[0].comment)
+}
+`
+
+interface ExportSymbol {
+  _id: string
+  package: string
+  isDocumented: boolean
+}
+
+interface Report {
+  package: string
+  documented: number
+  notDocumented: number
+}
+
+interface TransformResult {
+  package: string
+  documentedChange: number
+  prodDocumented: number
+  prodNotDocumented: number
+  branchDocumented: number
+  branchNotDocumented: number
+}
+
+const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET'))
+const studioMetricsClientProduction = createDocClient('production')
+
+function getDocumentationReport(symbols: ExportSymbol[]): Report[] {
+  const obj = groupBy(symbols, 'package')
+
+  return Object.entries(obj).map(([key, val]) => {
+    // return total number of documented and not documented symbols
+    return {
+      package: key,
+      documented: val.filter((s) => s.isDocumented).length,
+      notDocumented: val.filter((s) => !s.isDocumented).length,
+    }
+  })
+}
+
+const timer = startTimer(`Fetching docs report`)
+
+combineLatest([
+  studioMetricsClient.observable.fetch(QUERY),
+  studioMetricsClientProduction.observable.fetch(QUERY),
+])
+  .pipe(
+    map(([branch, production]: [ExportSymbol[], ExportSymbol[]]): TransformResult[] => {
+      const branchGroup = getDocumentationReport(branch)
+      const productionGroup = getDocumentationReport(production)
+
+      // Compare the two groups and return percent difference
+      return branchGroup.map((br) => {
+        const prod = productionGroup.find((p) => p.package === br.package)
+
+        if (!prod) {
+          return {
+            package: br.package,
+            documentedChange: 0,
+            prodDocumented: 0,
+            prodNotDocumented: 0,
+            branchDocumented: br.documented,
+            branchNotDocumented: br.notDocumented,
+          }
+        }
+
+        const documentedPercentDiff = (br.documented - prod.documented) / prod.documented
+
+        return {
+          package: br.package,
+          documentedChange: Number.isNaN(documentedPercentDiff)
+            ? 0
+            : Math.floor(documentedPercentDiff * 100),
+          prodDocumented: prod.documented,
+          prodNotDocumented: prod.notDocumented,
+          branchDocumented: br.documented,
+          branchNotDocumented: br.notDocumented,
+        }
+      })
+    }),
+  )
+  .subscribe(async (res) => {
+    // convert the result to a markdown table with heading
+    const table = `
+| Package | Documentation Change |
+| ------- | ----------------- |
+${res
+  .sort((a, b) => b.documentedChange - a.documentedChange)
+  .filter((r) => r.documentedChange !== 0)
+  .map(
+    (r) =>
+      `| ${r.package} | ${
+        r.documentedChange > 0 ? `+${r.documentedChange}` : r.documentedChange
+      }% |`,
+  )
+  .join('\n')}
+
+<details>
+  <summary>Full Report</summary>
+  ${res
+    .sort((a, b) => b.documentedChange - a.documentedChange)
+    .map(
+      (r) =>
+        `<details>
+  <summary>${r.package}</summary>
+  <table>
+    <tr>
+      <th>This branch</th>
+      <th>Current release</th>
+    </tr>
+    <tr>
+      <td>${r.branchDocumented} documented</td>
+      <td>${r.prodDocumented} documented</td>
+    </tr>
+    <tr>
+      <td>${r.branchNotDocumented} not documented</td>
+      <td>${r.prodNotDocumented} not documented</td>
+    </tr>
+  </table>
+</details>
+`,
+    )
+    .join('\n')}
+</details>
+`
+
+    // save it to a file
+    await fs.writeFile(path.resolve(path.join(__dirname, '..', 'docs-report.md')), table, 'utf8')
+
+    timer.end()
+  })

scripts/doc-report/docReportCleanup.ts

@@ -0,0 +1,19 @@
+import {sanityIdify} from '../utils/sanityIdify'
+import {createDocClient} from './docClient'
+import {readEnv} from './envVars'
+
+const DATASET = readEnv('DOCS_REPORT_DATASET')
+const studioMetricsClient = createDocClient(DATASET)
+
+studioMetricsClient.datasets
+  .delete(sanityIdify(DATASET))
+  .then((res) => {
+    if (res.deleted) {
+      console.log('Deleted dataset')
+    } else {
+      console.log('Dataset was not deleted')
+    }
+  })
+  .catch((err) => {
+    console.error(`Something went wrong! ${err?.response?.body?.message}`)
+  })