feat(gatsby): Slices API (#36489)

Co-authored-by: Lennart <lekoarts@gmail.com>
Co-authored-by: Ty Hopp <tyhopp@users.noreply.github.com>
Co-authored-by: pieh <misiek.piechowiak@gmail.com>
Co-authored-by: Jude Agboola <marvinjudehk@gmail.com>

integration-tests/artifacts/__tests__/index.js

@@ -14,6 +14,23 @@ const gatsbyBin = path.join(`node_modules`, `gatsby`, `cli.js`)
 const manifest = {}
 const filesToRevert = {}
 
+let _CFLAGS_ = {
+  GATSBY_MAJOR: `4`,
+}
+if (process.env.COMPILER_OPTIONS) {
+  // COMPILER_OPTIONS syntax is key=value,key2=value2
+  _CFLAGS_ = process.env.COMPILER_OPTIONS.split(`,`).reduce((acc, curr) => {
+    const [key, value] = curr.split(`=`)
+
+    if (key) {
+      acc[key] = value
+    }
+
+    return acc
+  }, _CFLAGS_)
+}
+
+let SLICES_ENABLED = _CFLAGS_.GATSBY_MAJOR === `5` && process.env.GATSBY_SLICES
 let exitCode
 
 function runGatsbyWithRunTestSetup(runNumber = 1) {
@@ -694,101 +711,146 @@ describe(`Second run (different pages created, data changed)`, () => {
   assertNodeCorrectness(runNumber)
 })
 
-describe(`Third run (js change, all pages are recreated)`, () => {
-  const runNumber = 3
-
-  const expectedPages = [
-    `/stale-pages/only-not-in-first`,
-    `/page-query-dynamic-3/`,
-    `/page-that-will-have-trailing-slash-removed`,
-    `/stale-pages/sometimes-i-have-trailing-slash-sometimes-i-dont`,
-  ]
-
-  const unexpectedPages = [
-    `/stale-pages/only-in-first/`,
-    `/page-query-dynamic-1/`,
-    `/page-query-dynamic-2/`,
-    `/stateful-page-not-recreated-in-third-run/`,
-  ]
-
-  let changedFileOriginalContent
-  const changedFileAbspath = path.join(
-    process.cwd(),
-    `src`,
-    `pages`,
-    `gatsby-browser.js`
-  )
+describe(
+  SLICES_ENABLED
+    ? `Third run (js template change, just pages of that template are recreated, all pages are stitched)`
+    : `Third run (js change, all pages are recreated)`,
+
+  () => {
+    const runNumber = 3
+
+    const expectedPagesToRemainFromPreviousBuild = [
+      `/stale-pages/stable/`,
+      `/page-query-stable/`,
+      `/page-query-changing-but-not-invalidating-html/`,
+      `/static-query-result-tracking/stable/`,
+      `/static-query-result-tracking/rerun-query-but-dont-recreate-html/`,
+      `/page-that-will-have-trailing-slash-removed`,
+    ]
+
+    const expectedPagesToBeGenerated = [
+      // this is page that gets template change
+      `/gatsby-browser/`,
+      // those change happen on every build
+      `/page-query-dynamic-3/`,
+      `/stale-pages/sometimes-i-have-trailing-slash-sometimes-i-dont`,
+      `/changing-context/`,
+    ]
+
+    const expectedPages = [
+      // this page should remain from first build
+      ...expectedPagesToRemainFromPreviousBuild,
+      // those pages should have been (re)created
+      ...expectedPagesToBeGenerated,
+    ]
+
+    const unexpectedPages = [
+      `/stale-pages/only-in-first/`,
+      `/page-query-dynamic-1/`,
+      `/page-query-dynamic-2/`,
+      `/stateful-page-not-recreated-in-third-run/`,
+    ]
+
+    let changedFileOriginalContent
+    const changedFileAbspath = path.join(
+      process.cwd(),
+      `src`,
+      `pages`,
+      `gatsby-browser.js`
+    )
 
-  beforeAll(async () => {
-    // make change to some .js
-    changedFileOriginalContent = fs.readFileSync(changedFileAbspath, `utf-8`)
-    filesToRevert[changedFileAbspath] = changedFileOriginalContent
+    beforeAll(async () => {
+      // make change to some .js
+      changedFileOriginalContent = fs.readFileSync(changedFileAbspath, `utf-8`)
+      filesToRevert[changedFileAbspath] = changedFileOriginalContent
 
-    const newContent = changedFileOriginalContent.replace(/sad/g, `not happy`)
+      const newContent = changedFileOriginalContent.replace(/sad/g, `not happy`)
 
-    if (newContent === changedFileOriginalContent) {
-      throw new Error(`Test setup failed`)
-    }
+      if (newContent === changedFileOriginalContent) {
+        throw new Error(`Test setup failed`)
+      }
 
-    fs.writeFileSync(changedFileAbspath, newContent)
-    await runGatsbyWithRunTestSetup(runNumber)()
-  })
+      fs.writeFileSync(changedFileAbspath, newContent)
+      await runGatsbyWithRunTestSetup(runNumber)()
+    })
 
-  assertExitCode(runNumber)
+    assertExitCode(runNumber)
 
-  describe(`html files`, () => {
-    const type = `html`
+    describe(`html files`, () => {
+      const type = `html`
 
-    describe(`should have expected html files`, () => {
-      assertFileExistenceForPagePaths({
-        pagePaths: expectedPages,
-        type,
-        shouldExist: true,
+      describe(`should have expected html files`, () => {
+        assertFileExistenceForPagePaths({
+          pagePaths: expectedPages,
+          type,
+          shouldExist: true,
+        })
       })
-    })
 
-    describe(`shouldn't have unexpected html files`, () => {
-      assertFileExistenceForPagePaths({
-        pagePaths: unexpectedPages,
-        type,
-        shouldExist: false,
+      describe(`shouldn't have unexpected html files`, () => {
+        assertFileExistenceForPagePaths({
+          pagePaths: unexpectedPages,
+          type,
+          shouldExist: false,
+        })
       })
-    })
 
-    it(`should recreate all html files`, () => {
-      expect(manifest[runNumber].generated.sort()).toEqual(
-        manifest[runNumber].allPages.sort()
-      )
+      if (SLICES_ENABLED) {
+        it(`should recreate only some html files`, () => {
+          expect(manifest[runNumber].generated.sort()).toEqual(
+            expectedPagesToBeGenerated.sort()
+          )
+        })
+
+        it(`should stitch fragments back in all html files (browser bundle changed)`, () => {
+          expect(manifest[runNumber].stitched.sort()).toEqual(
+            manifest[runNumber].allPages.sort()
+          )
+        })
+      } else {
+        it(`should recreate all html files`, () => {
+          expect(manifest[runNumber].generated.sort()).toEqual(
+            manifest[runNumber].allPages.sort()
+          )
+        })
+      }
     })
-  })
 
-  describe(`page-data files`, () => {
-    const type = `page-data`
+    describe(`page-data files`, () => {
+      const type = `page-data`
 
-    describe(`should have expected page-data files`, () => {
-      assertFileExistenceForPagePaths({
-        pagePaths: expectedPages,
-        type,
-        shouldExist: true,
+      describe(`should have expected page-data files`, () => {
+        assertFileExistenceForPagePaths({
+          pagePaths: expectedPages,
+          type,
+          shouldExist: true,
+        })
       })
-    })
 
-    describe(`shouldn't have unexpected page-data files`, () => {
-      assertFileExistenceForPagePaths({
-        pagePaths: unexpectedPages,
-        type,
-        shouldExist: false,
+      describe(`shouldn't have unexpected page-data files`, () => {
+        assertFileExistenceForPagePaths({
+          pagePaths: unexpectedPages,
+          type,
+          shouldExist: false,
+        })
       })
     })
-  })
 
-  // third run - we modify module used by both ssr and browser bundle - both bundles should change
-  assertWebpackBundleChanges({ browser: true, ssr: true, runNumber })
+    if (SLICES_ENABLED) {
+      // third run - we modify template used by both ssr and browser bundle - global, shared SSR won't change
+      // as the change is localized in just one of templates, which in Gatsby 5 doesn't invalidate all html
+      // files anymore
+      assertWebpackBundleChanges({ browser: true, ssr: false, runNumber })
+    } else {
+      // third run - we modify module used by both ssr and browser bundle - both bundles should change
+      assertWebpackBundleChanges({ browser: true, ssr: true, runNumber })
+    }
 
-  assertHTMLCorrectness(runNumber)
+    assertHTMLCorrectness(runNumber)
 
-  assertNodeCorrectness(runNumber)
-})
+    assertNodeCorrectness(runNumber)
+  }
+)
 
 describe(`Fourth run (gatsby-browser change - cache get invalidated)`, () => {
   const runNumber = 4

integration-tests/artifacts/gatsby-node.js

@@ -9,8 +9,9 @@ let changedBrowserCompilationHash
 let changedSsrCompilationHash
 let regeneratedPages = []
 let deletedPages = []
+let stitchedPages = []
 
-exports.onPreInit = ({ emitter }) => {
+exports.onPreInit = ({ emitter, store }) => {
   emitter.on(`SET_WEBPACK_COMPILATION_HASH`, action => {
     changedBrowserCompilationHash = action.payload
   })
@@ -28,6 +29,17 @@ exports.onPreInit = ({ emitter }) => {
   emitter.on(`HTML_REMOVED`, action => {
     deletedPages.push(action.payload)
   })
+
+  // this is last step before stitching slice html into page html
+  // we don't have action specific for stitching, so we just use this one
+  // to read state that determine which page htmls will be stitched
+  emitter.on(`SLICES_PROPS_REMOVE_STALE`, () => {
+    stitchedPages = []
+
+    for (const path of store.getState().html.pagesThatNeedToStitchSlices) {
+      stitchedPages.push(path)
+    }
+  })
 }
 
 let previouslyCreatedNodes = new Map()
@@ -215,6 +227,7 @@ exports.onPreBuild = () => {
   changedSsrCompilationHash = `not-changed`
   regeneratedPages = []
   deletedPages = []
+  stitchedPages = []
 }
 
 let counter = 1
@@ -250,6 +263,7 @@ exports.onPostBuild = async ({ graphql, getNodes }) => {
       changedSsrCompilationHash,
       generated: regeneratedPages,
       removed: deletedPages,
+      stitched: stitchedPages,
     }
   )
 }

integration-tests/ssr/test-output.js

@@ -33,6 +33,15 @@ async function run() {
       // Only in dev
       $(`meta[name="note"]`).remove()
 
+      // remove any comments
+      $.root()
+        .find("*")
+        .contents()
+        .filter(function () {
+          return this.type === "comment"
+        })
+        .remove()
+
       return $.html()
     }
 

packages/gatsby-cli/src/reporter/loggers/ink/components/pageTree.tsx

@@ -13,6 +13,7 @@ import {
 interface IPageTreeProps {
   components: Map<string, IComponentWithPageModes>
   root: string
+  slices: Set<string>
 }
 
 const Description: React.FC<BoxProps> = function Description(props) {
@@ -84,6 +85,7 @@ const ComponentTree: React.FC<{
 const PageTree: React.FC<IPageTreeProps> = function PageTree({
   components,
   root,
+  slices,
 }) {
   const componentList: Array<ReactElement> = []
   let i = 0
@@ -107,6 +109,20 @@ const PageTree: React.FC<IPageTreeProps> = function PageTree({
         <Text underline>Pages</Text>
       </Box>
       {componentList}
+      {slices.size > 0 && (
+        <>
+          <Box paddingTop={1} paddingBottom={1}>
+            <Text underline>Slices</Text>
+          </Box>
+          {Array.from(slices).map(slice => (
+            <Box key={slice}>
+              <Text>
+                <Text bold>·</Text> {path.posix.relative(root, slice)}
+              </Text>
+            </Box>
+          ))}
+        </>
+      )}
       <Description marginTop={1} marginBottom={1} />
     </Box>
   )
@@ -116,21 +132,31 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() {
   const state = useContext(StoreStateContext)
 
   const componentWithPages = new Map<string, IComponentWithPageModes>()
+  const slices = new Set<string>()
 
-  for (const { componentPath, pages } of state.pageTree!.components.values()) {
+  for (const {
+    componentPath,
+    pages,
+    isSlice,
+  } of state.pageTree!.components.values()) {
     const layoutComponent = getPathToLayoutComponent(componentPath)
     const pagesByMode = componentWithPages.get(layoutComponent) || {
       SSG: new Set<string>(),
       DSG: new Set<string>(),
       SSR: new Set<string>(),
       FN: new Set<string>(),
     }
-    pages.forEach(pagePath => {
-      const gatsbyPage = state.pageTree!.pages.get(pagePath)
 
-      pagesByMode[gatsbyPage!.mode].add(pagePath)
-    })
-    componentWithPages.set(layoutComponent, pagesByMode)
+    if (isSlice) {
+      slices.add(componentPath)
+    } else {
+      pages.forEach(pagePath => {
+        const gatsbyPage = state.pageTree!.pages.get(pagePath)
+
+        pagesByMode[gatsbyPage!.mode].add(pagePath)
+      })
+      componentWithPages.set(layoutComponent, pagesByMode)
+    }
   }
 
   for (const {
@@ -146,7 +172,11 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() {
   }
 
   return (
-    <PageTree components={componentWithPages} root={state.pageTree!.root} />
+    <PageTree
+      components={componentWithPages}
+      slices={slices}
+      root={state.pageTree!.root}
+    />
   )
 }