.circleci/config.yml

@@ -36,7 +36,7 @@ aliases:
   validate_renovate: &validate_renovate
     run:
       name: Validate renovate-config
-      command: (node scripts/renovate-config-generator.js && (git status --porcelain renovate.json5 | grep "M renovate.json5")) && exit 1 || npx -p renovate renovate-config-validator .
+      command: (node scripts/renovate-config-generator.js && (git status --porcelain renovate.json5 | grep "M renovate.json5")) && (echo "Please run \"node scripts/renovate-config-generator.js\" to update renovate.json5" && exit 1) || npx -p renovate renovate-config-validator .
 
   persist_cache: &persist_cache
     save_cache:
@@ -253,6 +253,7 @@ jobs:
       image: "14.17.0"
     environment:
       GATSBY_EXPERIMENTAL_LMDB_STORE: 1
+      GATSBY_EXPERIMENTAL_LMDB_INDEXES: 1
       GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING: 1
     <<: *test_template
 

.eslintignore

@@ -35,3 +35,4 @@ packages/gatsby-plugin-preact/fast-refresh
 packages/gatsby-source-wordpress/test-site/**
 !packages/gatsby-source-wordpress/test-site/__tests__
 !packages/gatsby-source-wordpress/test-site/test-utils
+!packages/gatsby-worker/src/__tests__/fixtures/**

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -11,6 +11,7 @@ body:
         Please fill out each section below. This info allows Gatsby maintainers to diagnose (and fix!) your issue as quickly as possible. Otherwise we might need to close the issue without e.g. clear reproduction steps.
 
         Also please verify that your issue is present in `gatsby@next`, too. The `next` version is the latest beta version of Gatsby and has any pending changes that are in the next stable release.
+        Bug reports also shoulnd't be used for support requests about a specific problem in your site, please use GitHub Discussions for that.
 
         Useful Links:
         - Documentation: https://www.gatsbyjs.com/docs/
@@ -22,7 +23,7 @@ body:
       options:
         - label: "This issue is not a duplicate. Before opening a new issue, please search existing issues: https://github.com/gatsbyjs/gatsby/issues"
           required: true
-        - label: "This issue is not a question, feature request, RFC, or anything other than a bug report. Please post those things in GitHub Discussions: https://github.com/gatsbyjs/gatsby/discussions"
+        - label: "This issue is not a question, feature request, RFC, or anything other than a bug report directly related to Gatsby. Please post those things in GitHub Discussions: https://github.com/gatsbyjs/gatsby/discussions"
           required: true
   - type: textarea
     attributes:

.prettierignore

@@ -29,5 +29,7 @@ packages/gatsby-source-wordpress/test-site/**
 **/__testfixtures__/**
 **/__tests__/fixtures/**
 
+!packages/gatsby-worker/src/__tests__/fixtures/**
+
 # coverage
 coverage

SECURITY.md

@@ -6,10 +6,15 @@ The following versions are currently being supported with security updates.
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 2.0.x   | :white_check_mark: |
-| 1.0.x   | :white_check_mark: |
+| 3.x     | :heavy_check_mark: |
+| 2.x     | :x:                |
+| 1.x     | :x:                |
 | < 1.0   | :x:                |
 
 ## Reporting a Vulnerability
 
-Please email security@gatsbyjs.com
+If you believe you have found a security issue with any of Gatsby's open source or commercial offerings, we would love to receive your report! Security findings can be emailed to security@gatsbyjs.com.
+
+When reporting a security issue, describe the issue in detail and include steps to reproduce. The more detail provided, the more likely we will be able to reproduce the issue and determine a course of action.
+
+Please do not report findings from `npm audit`. We are aware of package dependency issues that are reported by this tool and do review these reports. In many cases the issues reported by `npm audit` are misleading and do not present a tangible/exploitable security risk for Gatsby users.

benchmarks/query-filters-sort/README.md

@@ -1,23 +1,52 @@
 # Filters and sort benchmark
 
-Stress tests various query filters (with and without sorting by random value).
+Stress tests various query filters (with optional sorting and counting).
 
 # Usage
 
 ```shell
-NUM_NODES=1000 NUM_PAGES=1000 FILTER=eq SORT=1 TEXT=1 yarn bench
+NUM_NODES=1000 NUM_PAGES=1000 FILTER=eq SORT=1 TEXT=1 COUNT=1 yarn bench
 ```
 
-Explanation:
+## Description
 
-- `FILTER`: one of `eq`, `gt`, `gt-lt`, `in`, `lt`, `ne`, `nin`, `regex`, `elemMatch-eq` (default: `eq`).
-  See `src/templates` for specific queries.
-- `SORT`: when set, the benchmark will also add sorting to the query (by random integer value)
-- `TEXT`: all nodes are lightweight by default (just 5 fields with numeric values or short strings).\
-  When settings this env variable - each node will get additional field with 4k of text
-  (useful for memory usage measurements)
-- `NUM_NODES`: the number of nodes created (1000 by default)
-- `NUM_PAGES`: the number of pages created (1000 by default, must be >= `NUM_NODES`)
+All queries have `limit=100` (although some of them may return just several items or 0).
+
+### Env vars:
+
+- `NUM_NODES`: The number of nodes created (1000 by default)
+- `NUM_PAGES`: The number of pages created (1000 by default, must be <= `NUM_NODES`)
+
+- `FILTER`. Available values:
+
+  - `eq`: captures 1/4 of all nodes (default)
+  - `eq-id`: captures a single node by id
+  - `eq-uniq`: captures a single node by unique value (e.g. `slug`)
+  - `eq-two-fields`: applies `eq` filter to two fields and captures 1/4 of all nodes
+  - `elemMatch-eq`: captures 1/2 of all nodes
+  - `in`: captures 1/2 of all nodes
+  - `gt`: the first query captures all nodes, the last one - 0 nodes
+  - `lt`: the first query captures 0 nodes, the last - all nodes
+  - `gt-lt`: any query captures 1000 items; last 1000 pages will capture from 999 to 0 (`gt: currentPage, lt: currentPage + 1000`)
+  - `nin`: captures 1/2 of all nodes
+  - `ne`: captures 3/4 of all nodes
+  - `regex`: captures from 1/4 to 1/3 of all nodes (simple and fast regexp)
+
+- `SORT`. Available values:
+
+  - `0`: no sort (default)
+  - `1`: sorts by random number
+  - comma-separate list of fields (e.g. `SORT=fooBar,random` sorts by fields `[foo, bar]`)
+
+- `TEXT`. Available values:
+
+  - `0`: small nodes without big text content (default)
+  - `1`: adds 4kb of random text to each node.
+    Note: text is returned by graphql queries, so it affects `page-data.json` file size.
+
+- `COUNT`. Available values:
+  - `0`: query doesn't request total count of items (default)
+  - `1`: adds `totalCount` to query request
 
 # Example
 

benchmarks/query-filters-sort/gatsby-config.js

@@ -4,5 +4,8 @@ module.exports = {
     description: `The filters and sort benchmark`,
     author: `@gatsbyjs`,
   },
+  flags: {
+    PARALLEL_QUERY_RUNNING: true,
+  }
   // plugins: [`gatsby-plugin-benchmark-reporting`],
 }

benchmarks/query-filters-sort/gatsby-node.js

@@ -2,7 +2,8 @@ const NUM_PAGES = parseInt(process.env.NUM_PAGES || 1000, 10)
 const NUM_NODES = parseInt(process.env.NUM_NODES || NUM_PAGES, 10)
 const SORT = process.env.SORT
 const FILTER = process.env.FILTER || `eq`
-const TEXT = Boolean(process.env.TEXT)
+const COUNT = Boolean(process.env.COUNT) && process.env.COUNT !== `0`
+const TEXT = Boolean(process.env.TEXT) && process.env.TEXT !== `0`
 
 if (NUM_NODES < NUM_PAGES) {
   throw new Error("Expecting NUM_NODES >= NUM_PAGES")
@@ -16,16 +17,22 @@ exports.createSchemaCustomization = ({ actions }) => {
     type Test implements Node @dontInfer {
       id: ID!
       nodeNum: Int!
+      nodeNumStr: String!
       pageNum: Int!
-      unique: String!
+      pageNumStr: String!
       fooBar: String!
+      fooBar2: String!
       fooBarArray: [TestFooBarArray!]
       text: String!
-      randon: Float
+      random: Int!
+      randomPage: Int!
     }
     type TestFooBarArray {
       fooBar: String!
     }
+    type SitePage implements Node @dontInfer {
+      id: ID!
+    }
   `)
 }
 
@@ -36,15 +43,18 @@ exports.sourceNodes = async ({ actions: { createNode } }) => {
     createNode({
       id: String(nodeNum),
       nodeNum,
+      nodeNumStr: String(nodeNum),
       pageNum,
-      unique: String(nodeNum),
+      pageNumStr: String(pageNum),
       fooBar: [`foo`, `bar`, `baz`, `foobar`][nodeNum % 4],
+      fooBar2: [`foo`, `bar`, `baz`, `foobar`][pageNum % 4],
       fooBarArray: [
         { fooBar: [`foo`, `bar`, `baz`, `foobar`][nodeNum % 4] },
         { fooBar: [`bar`, `baz`, `foobar`, `foo`][nodeNum % 4] },
       ],
       text: TEXT ? randomStr(4128) : `${nodeNum}`,
-      random: Math.random() * NUM_NODES,
+      random: Math.round(Math.random() * NUM_NODES),
+      randomPage: Math.round(Math.random() * NUM_PAGES),
       internal: {
         type: `Test`,
         contentDigest: String(nodeNum),
@@ -62,7 +72,11 @@ exports.sourceNodes = async ({ actions: { createNode } }) => {
 
 const pageTemplate = require.resolve(`./src/templates/${FILTER}.js`)
 exports.createPages = async ({ actions: { createPage } }) => {
-  console.log(`Creating ${NUM_PAGES} pages for filter: ${FILTER}`)
+  console.log(
+    `Creating ${NUM_PAGES} pages for filter: ${FILTER} (SORT: ${
+      SORT || `0`
+    }, COUNT: ${COUNT || `0`})`
+  )
   for (let pageNum = 0; pageNum < NUM_PAGES; pageNum++) {
     createPage({
       path: `/path/${pageNum}/`,
@@ -74,17 +88,23 @@ exports.createPages = async ({ actions: { createPage } }) => {
         ],
         fooBar: [`foo`, `bar`, `baz`, `foobar`][pageNum % 4],
         pageNum: pageNum,
-        pagesLeft: NUM_PAGES - pageNum,
+        pageNumPlus1000: pageNum + 1000,
+        pageNumStr: String(pageNum),
         limit: nodesPerPage,
         skip: nodesPerPage * pageNum,
         nodesTotal: NUM_NODES,
         pagesTotal: NUM_PAGES,
-        sort: SORT
-          ? {
-              fields: SORT === `fooBar` ? ["fooBar", "random"] : ["random"],
-            }
-          : undefined,
-        regex: `/^${String(pageNum).slice(0, 1)}/`, // node id starts with the same number as page id
+        sort:
+          !SORT || SORT === `0`
+            ? undefined
+            : {
+                fields:
+                  SORT === `1`
+                    ? ["random"]
+                    : SORT.split(`,`).map(f => f.trim()),
+              },
+        count: COUNT,
+        fooBarRegex: `/${[`foo`, `bar`, `baz`, `foobar`][pageNum % 4]}/`,
       },
     })
     if (pageNum % 50 === 0) {

benchmarks/query-filters-sort/package.json

@@ -9,7 +9,8 @@
     "serve": "gatsby serve"
   },
   "dependencies": {
-    "gatsby": "^3.4.1",
+    "gatsby": "next",
+    "lmdb-store": "^1.6.1",
     "process-top": "^1.2.0",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",

benchmarks/query-filters-sort/src/templates/elemMatch-eq.js

@@ -9,7 +9,7 @@ export default ({ data }) => {
 }
 
 export const query = graphql`
-  query($fooBar: String, $sort: TestSortInput) {
+  query($fooBar: String, $sort: TestSortInput, $count: Boolean!) {
     allTest(
       filter: { fooBarArray: { elemMatch: { fooBar: { eq: $fooBar } } } }
       sort: $sort
@@ -19,6 +19,7 @@ export const query = graphql`
         nodeNum
         text
       }
+      totalCount @include(if: $count)
     }
   }
 `

benchmarks/query-filters-sort/src/templates/eq-id.js

@@ -0,0 +1,21 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Wrong data: " + JSON.stringify(data))
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($pageNumStr: String!, $sort: TestSortInput, $count: Boolean!) {
+    allTest(filter: { id: { eq: $pageNumStr } }, sort: $sort, limit: 100) {
+      nodes {
+        nodeNum
+        text
+      }
+      totalCount @include(if: $count)
+    }
+  }
+`

benchmarks/query-filters-sort/src/templates/eq-two-fields.js

@@ -0,0 +1,25 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Wrong data: " + JSON.stringify(data))
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($fooBar: String!, $sort: TestSortInput, $count: Boolean!) {
+    allTest(
+      filter: { fooBar: { eq: $fooBar }, fooBar2: { eq: $fooBar } }
+      sort: $sort
+      limit: 100
+    ) {
+      nodes {
+        nodeNum
+        text
+      }
+      totalCount @include(if: $count)
+    }
+  }
+`

benchmarks/query-filters-sort/src/templates/eq-uniq.js

@@ -0,0 +1,21 @@
+import React from "react"
+import { graphql } from "gatsby"
+
+export default ({ data }) => {
+  if (!data?.allTest?.nodes) {
+    throw new Error("Wrong data: " + JSON.stringify(data))
+  }
+  return <div>{JSON.stringify(data)}</div>
+}
+
+export const query = graphql`
+  query($pageNum: Int!, $sort: TestSortInput, $count: Boolean!) {
+    allTest(filter: { nodeNum: { eq: $pageNum } }, sort: $sort, limit: 1) {
+      nodes {
+        nodeNum
+        text
+      }
+      totalCount @include(if: $count)
+    }
+  }
+`

benchmarks/query-filters-sort/src/templates/eq.js

@@ -9,12 +9,13 @@ export default ({ data }) => {
 }
 
 export const query = graphql`
-  query($fooBar: String!, $sort: TestSortInput) {
+  query($fooBar: String!, $sort: TestSortInput, $count: Boolean!) {
     allTest(filter: { fooBar: { eq: $fooBar } }, sort: $sort, limit: 100) {
       nodes {
         nodeNum
         text
       }
+      totalCount @include(if: $count)
     }
   }
 `

benchmarks/query-filters-sort/src/templates/gt-lt.js

@@ -9,16 +9,22 @@ export default ({ data }) => {
 }
 
 export const query = graphql`
-  query($pageNum: Int, $pagesTotal: Int, $sort: TestSortInput) {
+  query(
+    $pageNum: Int
+    $pageNumPlus1000: Int
+    $sort: TestSortInput
+    $count: Boolean!
+  ) {
     allTest(
-      filter: { nodeNum: { gt: $pageNum, lt: $pagesTotal } }
+      filter: { randomPage: { gt: $pageNum, lt: $pageNumPlus1000 } }
       sort: $sort
       limit: 100
     ) {
       nodes {
         nodeNum
         text
       }
+      totalCount @include(if: $count)
     }
   }
 `

benchmarks/query-filters-sort/src/templates/gt.js

@@ -9,12 +9,13 @@ export default ({ data }) => {
 }
 
 export const query = graphql`
-  query($pagesLeft: Int, $sort: TestSortInput) {
-    allTest(filter: { nodeNum: { gt: $pagesLeft } }, sort: $sort, limit: 100) {
+  query($pageNum: Int, $sort: TestSortInput, $count: Boolean!) {
+    allTest(filter: { randomPage: { gt: $pageNum } }, sort: $sort, limit: 100) {
       nodes {
         nodeNum
         text
       }
+      totalCount @include(if: $count)
     }
   }
 `