diff --git a/packages/gatsby-plugin-utils/src/types.ts b/packages/gatsby-plugin-utils/src/types.ts index 551641f0111e5..0c98ccc5b3e6d 100644 --- a/packages/gatsby-plugin-utils/src/types.ts +++ b/packages/gatsby-plugin-utils/src/types.ts @@ -52,6 +52,10 @@ export interface IPluginRefObject { resolve: string options?: IPluginRefOptions parentDir?: string + + subPluginPaths?: Array + module?: any + modulePath?: string } export type PluginRef = string | IPluginRefObject diff --git a/packages/gatsby-plugin-utils/src/utils/plugin-options-schema-joi-type.ts b/packages/gatsby-plugin-utils/src/utils/plugin-options-schema-joi-type.ts index a4dca6be61833..c9811a58dc7fe 100644 --- a/packages/gatsby-plugin-utils/src/utils/plugin-options-schema-joi-type.ts +++ b/packages/gatsby-plugin-utils/src/utils/plugin-options-schema-joi-type.ts @@ -1198,7 +1198,7 @@ interface Context { interface State { key?: string - path?: string + path: Array parent?: any reference?: any ancestors?: any diff --git a/packages/gatsby/cache-dir/query-engine/index.d.ts b/packages/gatsby/cache-dir/query-engine/index.d.ts new file mode 100644 index 0000000000000..5926b3cb98023 --- /dev/null +++ b/packages/gatsby/cache-dir/query-engine/index.d.ts @@ -0,0 +1,31 @@ +// IGatsbyPage, SystemPath and copied/inlined from redux/types so this file is self contained +type SystemPath = string +type Identifier = string + +export interface IGatsbyPage { + internalComponentName: string + path: string + matchPath: undefined | string + component: SystemPath + componentChunkName: string + isCreatedByStatefulCreatePages: boolean + context: Record + updatedAt: number + // eslint-disable-next-line @typescript-eslint/naming-convention + pluginCreator___NODE: Identifier + pluginCreatorId: Identifier + componentPath: SystemPath + ownerNodeId: Identifier +} + +// also inlined +export interface IQueryResult { + errors?: Array + data?: any +} + +export class GraphQLEngine { + constructor({ dbPath }: { dbPath: string }) + runQuery(query: string, context: Record): Promise + findPageByPath(pathName: string): IGatsbyPage | undefined +} diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index 71f22ca40fb99..c956c9966c93f 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -24,6 +24,7 @@ "@types/http-proxy": "^1.17.4", "@typescript-eslint/eslint-plugin": "^4.29.3", "@typescript-eslint/parser": "^4.29.3", + "@vercel/webpack-asset-relocator-loader": "^1.6.0", "address": "1.1.2", "anser": "^2.0.1", "autoprefixer": "^10.2.4", diff --git a/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.ts.snap b/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.ts.snap index 60e67b2ca73f9..e1ca310e06d84 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.ts.snap +++ b/packages/gatsby/src/bootstrap/load-plugins/__tests__/__snapshots__/load-plugins.ts.snap @@ -104,6 +104,8 @@ Array [ Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -117,11 +119,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -135,11 +140,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -153,11 +161,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -171,11 +182,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -189,11 +203,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -207,11 +224,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -225,11 +245,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-typescript", "nodeAPIs": Array [ "pluginOptionsSchema", @@ -245,6 +268,7 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { @@ -262,6 +286,8 @@ Array [ Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -275,6 +301,7 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, ] @@ -396,6 +423,8 @@ Array [ Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -409,11 +438,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -427,11 +459,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -445,11 +480,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -463,11 +501,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -481,11 +522,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -499,11 +543,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -517,11 +564,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -535,11 +585,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-typescript", "nodeAPIs": Array [ "pluginOptionsSchema", @@ -555,6 +608,7 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { @@ -572,6 +626,8 @@ Array [ Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -585,6 +641,7 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, ] @@ -694,6 +751,8 @@ Array [ Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -709,11 +768,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -727,11 +789,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -745,11 +810,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -763,11 +831,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -781,11 +852,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -799,11 +873,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -817,11 +894,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -835,11 +915,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -853,11 +936,14 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-typescript", "nodeAPIs": Array [ "pluginOptionsSchema", @@ -873,6 +959,7 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, Object { @@ -890,6 +977,8 @@ Array [ Object { "browserAPIs": Array [], "id": "", + "module": undefined, + "modulePath": undefined, "name": "gatsby-plugin-page-creator", "nodeAPIs": Array [ "createPagesStatefully", @@ -903,6 +992,7 @@ Array [ }, "resolve": "", "ssrAPIs": Array [], + "subPluginPaths": undefined, "version": "1.0.0", }, ] diff --git a/packages/gatsby/src/bootstrap/load-plugins/index.ts b/packages/gatsby/src/bootstrap/load-plugins/index.ts index 200fdecd25eb4..b3f9e1758a37c 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/index.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/index.ts @@ -36,11 +36,27 @@ const getAPI = ( const flattenPlugins = (plugins: Array): Array => { const flattened: Array = [] const extractPlugins = (plugin: IPluginInfo): void => { - if (plugin.pluginOptions && plugin.pluginOptions.plugins) { - plugin.pluginOptions.plugins.forEach(subPlugin => { - flattened.push(subPlugin) - extractPlugins(subPlugin) - }) + if (plugin.subPluginPaths) { + for (const subPluginPath of plugin.subPluginPaths) { + // @pieh: + // subPluginPath can look like someOption.randomFieldThatIsMarkedAsSubplugins + // Reason for doing stringified path with . separator was that it was just easier to prevent duplicates + // in subPluginPaths array (as each subplugin in the gatsby-config would add subplugin path). + const segments = subPluginPath.split(`.`) + let roots: Array = [plugin.pluginOptions] + for (const segment of segments) { + if (segment === `[]`) { + roots = roots.flat() + } else { + roots = roots.map(root => root[segment]) + } + } + + roots.forEach(subPlugin => { + flattened.push(subPlugin) + extractPlugins(subPlugin) + }) + } } } diff --git a/packages/gatsby/src/bootstrap/load-plugins/load.ts b/packages/gatsby/src/bootstrap/load-plugins/load.ts index 39dc41af54225..aa7997adbfc79 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/load.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/load.ts @@ -40,6 +40,7 @@ function createFileContentHash(root: string, globPattern: string): string { * (docs, blogs). * * @param name Name of the plugin + * @param pluginObject Object of the plugin */ const createPluginId = ( name: string, @@ -210,13 +211,25 @@ export function loadPlugins( } // Plugins can have plugins. - const subplugins: Array = [] - if (plugin.options.plugins) { - plugin.options.plugins.forEach(p => { - subplugins.push(processPlugin(p)) - }) - - plugin.options.plugins = subplugins + if (plugin.subPluginPaths) { + for (const subPluginPath of plugin.subPluginPaths) { + const segments = subPluginPath.split(`.`) + let roots: Array = [plugin.options] + + let pathToSwap = segments + + for (const segment of segments) { + if (segment === `[]`) { + pathToSwap = pathToSwap.slice(0, pathToSwap.length - 1) + roots = roots.flat() + } else { + roots = roots.map(root => root[segment]) + } + } + + const processed = roots.map(processPlugin) + _.set(plugin.options, pathToSwap, processed) + } } // Add some default values for tests as we don't actually @@ -239,6 +252,11 @@ export function loadPlugins( return { ...info, + modulePath: plugin.modulePath, + module: plugin.module, + subPluginPaths: plugin.subPluginPaths + ? Array.from(plugin.subPluginPaths) + : undefined, id: createPluginId(info.name, plugin), pluginOptions: _.merge({ plugins: [] }, plugin.options), } diff --git a/packages/gatsby/src/bootstrap/load-plugins/types.ts b/packages/gatsby/src/bootstrap/load-plugins/types.ts index d525afb42859e..4793303976af0 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/types.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/types.ts @@ -25,6 +25,10 @@ export interface IPluginInfo { /** Options passed to the plugin */ pluginOptions?: IPluginInfoOptions + + subPluginPaths?: Array + module?: any + modulePath?: string } export interface IPluginInfoOptions { @@ -44,6 +48,9 @@ export interface IPluginRefObject { resolve: string options?: IPluginRefOptions parentDir?: string + subPluginPaths?: Array + module?: any + modulePath?: string } export type PluginRef = string | IPluginRefObject diff --git a/packages/gatsby/src/bootstrap/load-plugins/validate.ts b/packages/gatsby/src/bootstrap/load-plugins/validate.ts index 608fe922251b2..30e84cf2493b4 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/validate.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/validate.ts @@ -197,17 +197,84 @@ async function validatePluginsOptions( if (!gatsbyNode.pluginOptionsSchema) return plugin + const subPluginPaths = new Set() + let optionsSchema = ( gatsbyNode.pluginOptionsSchema as Exclude< GatsbyNode["pluginOptionsSchema"], undefined > )({ - Joi, + Joi: Joi.extend(joi => { + return { + base: joi.any(), + type: `subPlugins`, + args: (_, args: any): any => { + const entry = args?.entry ?? `index` + + return joi + .array() + .items( + joi + .alternatives( + joi.string(), + joi.object({ + resolve: Joi.string(), + options: Joi.object({}).unknown(true), + }) + ) + .custom((value, helpers) => { + if (typeof value === `string`) { + value = { resolve: value } + } + + try { + const resolvedPlugin = resolvePlugin(value, rootDir) + const modulePath = require.resolve( + `${resolvedPlugin.resolve}/${entry}` + ) + value.modulePath = modulePath + value.module = require(modulePath) + + const normalizedPath = helpers.state.path + .map((key, index) => { + // if subplugin is part of an array - swap concrete index key with `[]` + if ( + typeof key === `number` && + Array.isArray( + helpers.state.ancestors[ + helpers.state.path.length - index - 1 + ] + ) + ) { + if (index !== helpers.state.path.length - 1) { + throw new Error( + `No support for arrays not at the end of path` + ) + } + return `[]` + } + + return key + }) + .join(`.`) + + subPluginPaths.add(normalizedPath) + } catch (err) { + console.log(err) + } + + return value + }, `Gatsby specific subplugin validation`) + ) + .default([]) + }, + } + }), }) - // Validate correct usage of pluginOptionsSchema if (!Joi.isSchema(optionsSchema) || optionsSchema.type !== `object`) { + // Validate correct usage of pluginOptionsSchema reporter.warn( `Plugin "${plugin.resolve}" has an invalid options schema so we cannot verify your configuration for it.` ) @@ -236,8 +303,14 @@ async function validatePluginsOptions( rootDir ) plugin.options.plugins = subPlugins + if (subPlugins.length > 0) { + subPluginPaths.add(`plugins.[]`) + } errors += subErrors } + if (subPluginPaths.size > 0) { + plugin.subPluginPaths = Array.from(subPluginPaths) + } } catch (error) { if (error instanceof Joi.ValidationError) { // Show a small warning on unknown options rather than erroring diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index a041d03f71bb0..faccfde61410f 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -47,6 +47,8 @@ import { mergeWorkerState, runQueriesInWorkersQueue, } from "../utils/worker/pool" +import { createGraphqlEngineBundle } from "../schema/graphql-engine/bundle-webpack" +import { shouldGenerateEngines } from "../utils/engines-helpers" module.exports = async function build(program: IBuildArgs): Promise { if (isTruthy(process.env.VERBOSE)) { @@ -89,6 +91,13 @@ module.exports = async function build(program: IBuildArgs): Promise { parentSpan: buildSpan, }) + const engineBundlingPromises: Array> = [] + + if (_CFLAGS_.GATSBY_MAJOR === `4` && shouldGenerateEngines()) { + // bundle graphql-engine + engineBundlingPromises.push(createGraphqlEngineBundle()) + } + const graphqlRunner = new GraphQLRunner(store, { collectStats: true, graphqlTracing: program.graphqlTracing, @@ -96,6 +105,13 @@ module.exports = async function build(program: IBuildArgs): Promise { const { queryIds } = await calculateDirtyQueries({ store }) + // Only run queries with mode SSG + if (_CFLAGS_.GATSBY_MAJOR === `4`) { + queryIds.pageQueryIds = queryIds.pageQueryIds.filter( + query => query.mode === `SSG` + ) + } + let waitForWorkerPoolRestart = Promise.resolve() if (process.env.GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING) { await runQueriesInWorkersQueue(workerPool, queryIds) @@ -311,6 +327,8 @@ module.exports = async function build(program: IBuildArgs): Promise { report.info(`.cache/deletedPages.txt created`) } + await Promise.all(engineBundlingPromises) + showExperimentNotices() if (await userGetsSevenDayFeedback()) { diff --git a/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts b/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts index 179e92c3c9e6c..3a6f5e6f3d9ca 100644 --- a/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts +++ b/packages/gatsby/src/datastore/lmdb/lmdb-datastore.ts @@ -27,24 +27,32 @@ const lmdbDatastore = { getNodesByType, } -const rootDbFile = - process.env.NODE_ENV === `test` - ? `test-datastore-${ - // FORCE_TEST_DATABASE_ID will be set if this gets executed in worker context - // when running jest tests. JEST_WORKER_ID will be set when this gets executed directly - // in test context (jest will use jest-worker internally). - process.env.FORCE_TEST_DATABASE_ID ?? process.env.JEST_WORKER_ID - }` - : `datastore` +function getDefaultDbPath(): string { + const dbFileName = + process.env.NODE_ENV === `test` + ? `test-datastore-${ + // FORCE_TEST_DATABASE_ID will be set if this gets executed in worker context + // when running jest tests. JEST_WORKER_ID will be set when this gets executed directly + // in test context (jest will use jest-worker internally). + process.env.FORCE_TEST_DATABASE_ID ?? process.env.JEST_WORKER_ID + }` + : `datastore` + + return process.cwd() + `/.cache/data/` + dbFileName +} +let fullDbPath let rootDb let databases function getRootDb(): RootDatabase { if (!rootDb) { + if (!fullDbPath) { + throw new Error(`LMDB path is not set!`) + } rootDb = open({ name: `root`, - path: process.cwd() + `/.cache/data/` + rootDbFile, + path: fullDbPath, compression: true, }) } @@ -211,7 +219,11 @@ async function ready(): Promise { await lastOperationPromise } -export function setupLmdbStore(): IDataStore { +export function setupLmdbStore({ + dbPath = getDefaultDbPath(), +}: { dbPath?: string } = {}): IDataStore { + fullDbPath = dbPath + replaceReducer({ nodes: (state = new Map(), action) => action.type === `DELETE_CACHE` ? new Map() : state, diff --git a/packages/gatsby/src/query/index.ts b/packages/gatsby/src/query/index.ts index 568cbf0f4c6f8..23de7f3914a7b 100644 --- a/packages/gatsby/src/query/index.ts +++ b/packages/gatsby/src/query/index.ts @@ -251,12 +251,6 @@ function createPageQueryJob( } const { path, componentPath, context } = page - if (_CFLAGS_.GATSBY_MAJOR === `4`) { - const { mode } = page - if (mode !== `SSG`) { - return undefined - } - } const { query } = component return { diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts index d91fde2a589a7..33f1a971b4777 100644 --- a/packages/gatsby/src/redux/types.ts +++ b/packages/gatsby/src/redux/types.ts @@ -245,6 +245,8 @@ export interface IGatsbyState { > ssrAPIs: Array<"onRenderBody" | "onPreRenderHTML"> pluginFilepath: SystemPath + subPluginPaths?: Array + modulePath?: string }> config: IGatsbyConfig functions: Array diff --git a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts new file mode 100644 index 0000000000000..46e9c6075d1e7 --- /dev/null +++ b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts @@ -0,0 +1,108 @@ +/* eslint-disable @typescript-eslint/naming-convention */ + +import * as path from "path" +import * as fs from "fs-extra" +import webpack from "webpack" +import { printQueryEnginePlugins } from "./print-plugins" +import mod from "module" + +const extensions = [`.mjs`, `.js`, `.json`, `.node`, `.ts`, `.tsx`] + +const outputDir = path.join(process.cwd(), `.cache`, `query-engine`) + +export async function createGraphqlEngineBundle(): Promise { + const schemaSnapshotString = await fs.readFile( + process.cwd() + `/.cache/schema.gql`, + `utf-8` + ) + await printQueryEnginePlugins() + + const compiler = webpack({ + // mode: `production`, + mode: `none`, + entry: path.join(__dirname, `entry.js`), + output: { + path: outputDir, + filename: `index.js`, + libraryTarget: `commonjs`, + }, + target: `node`, + externalsPresets: { + node: false, + }, + // those are required in some runtime paths, but we don't need them + externals: [ + `cbor-x`, // optional dep of lmdb-store, but we are using `msgpack` (default) encoding, so we don't need it + `babel-runtime/helpers/asyncToGenerator`, // undeclared dep of yurnalist (but used in code path we don't use) + `electron`, // :shrug: `got` seems to have electron specific code path + mod.builtinModules.reduce((acc, builtinModule) => { + if (builtinModule === `fs`) { + acc[builtinModule] = `global _actualFsWrapper` + } else { + acc[builtinModule] = `commonjs ${builtinModule}` + } + + return acc + }, {}), + ], + module: { + rules: [ + { + test: /\.m?js$/, + type: `javascript/auto`, + resolve: { + byDependency: { + esm: { + fullySpecified: false, + }, + }, + }, + }, + { + // For node binary relocations, include ".node" files as well here + test: /\.(m?js|node)$/, + // it is recommended for Node builds to turn off AMD support + parser: { amd: false }, + use: { + loader: require.resolve(`@vercel/webpack-asset-relocator-loader`), + options: { + outputAssetBase: `assets`, + }, + }, + }, + { + test: /\.txt/, + type: `asset/resource`, + }, + ], + }, + resolve: { + extensions, + alias: { + ".cache": process.cwd() + `/.cache/`, + }, + }, + plugins: [ + new webpack.DefinePlugin({ + // "process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`), + "process.env.GATSBY_EXPERIMENTAL_LMDB_STORE": `true`, + "process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE": `true`, + SCHEMA_SNAPSHOT: JSON.stringify(schemaSnapshotString), + }), + ], + }) + + return new Promise((resolve, reject) => { + compiler.run((err, stats) => { + compiler.close(closeErr => { + if (err) { + return reject(err) + } + if (closeErr) { + return reject(closeErr) + } + return resolve(stats?.compilation) + }) + }) + }) +} diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts new file mode 100644 index 0000000000000..89078517bfe1f --- /dev/null +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -0,0 +1,109 @@ +import "../../utils/engines-fs-provider" +import { build } from "../index" +import { setupLmdbStore } from "../../datastore/lmdb/lmdb-datastore" +import { store } from "../../redux" +import { actions } from "../../redux/actions" +import reporter from "gatsby-cli/lib/reporter" +import { + createGraphQLRunner, + Runner, +} from "../../bootstrap/create-graphql-runner" +import { waitUntilAllJobsComplete } from "../../utils/wait-until-jobs-complete" + +import { setGatsbyPluginCache } from "../../utils/require-gatsby-plugin" +import apiRunnerNode from "../../utils/api-runner-node" +import type { IGatsbyPage, IGatsbyState } from "../../redux/types" +import { findPageByPath } from "../../utils/find-page-by-path" +import { getDataStore } from "../../datastore" +import { + gatsbyNodes, + gatsbyWorkers, + flattenedPlugins, + // @ts-ignore +} from ".cache/query-engine-plugins" + +export class GraphQLEngine { + // private schema: GraphQLSchema + private runnerPromise?: Promise + + constructor({ dbPath }: { dbPath: string }) { + setupLmdbStore({ dbPath }) + // start initializing runner ASAP + this.getRunner() + } + + private async _doGetRunner(): Promise { + // @ts-ignore SCHEMA_SNAPSHOT is being "inlined" by bundler + store.dispatch(actions.createTypes(SCHEMA_SNAPSHOT)) + + // TODO: FLATTENED_PLUGINS needs to be merged with plugin options from gatsby-config + // (as there might be non-serializable options, i.e. functions) + store.dispatch({ + type: `SET_SITE_FLATTENED_PLUGINS`, + payload: flattenedPlugins, + }) + + for (const pluginName of Object.keys(gatsbyNodes)) { + setGatsbyPluginCache( + { name: pluginName, resolve: `` }, + `gatsby-node`, + gatsbyNodes[pluginName] + ) + } + for (const pluginName of Object.keys(gatsbyWorkers)) { + setGatsbyPluginCache( + { name: pluginName, resolve: `` }, + `gatsby-worker`, + gatsbyWorkers[pluginName] + ) + } + await apiRunnerNode(`unstable_onPluginInit`) + await apiRunnerNode(`createSchemaCustomization`) + + // Build runs + // Note: skipping inference metadata because we rely on schema snapshot + await build({ fullMetadataBuild: false, freeze: true }) + + return createGraphQLRunner(store, reporter) + } + + private async getRunner(): Promise { + if (!this.runnerPromise) { + this.runnerPromise = this._doGetRunner() + } + return this.runnerPromise + } + + public async runQuery(...args: Parameters): ReturnType { + const graphqlRunner = await this.getRunner() + const result = await graphqlRunner(...args) + // Def not ideal - this is just waiting for all jobs and not jobs for current + // query, but we don't track jobs per query right now + // TODO: start tracking jobs per query to be able to await just those + await waitUntilAllJobsComplete() + return result + } + + public findPageByPath(pathName: string): IGatsbyPage | undefined { + // adapter so `findPageByPath` use SitePage nodes in datastore + // instead of `pages` redux slice + const state = { + pages: { + get(pathName: string): IGatsbyPage | undefined { + return getDataStore().getNode(`SitePage ${pathName}`) as + | IGatsbyPage + | undefined + }, + values(): Iterable { + return getDataStore().iterateNodesByType( + `SitePage` + ) as Iterable + }, + }, + } as unknown as IGatsbyState + + return findPageByPath(state, pathName, false) + } +} + +export default { GraphQLEngine } diff --git a/packages/gatsby/src/schema/graphql-engine/print-plugins.ts b/packages/gatsby/src/schema/graphql-engine/print-plugins.ts new file mode 100644 index 0000000000000..dbd8d7b021be9 --- /dev/null +++ b/packages/gatsby/src/schema/graphql-engine/print-plugins.ts @@ -0,0 +1,214 @@ +/* eslint @typescript-eslint/no-unused-vars: ["error", { "ignoreRestSiblings": true }] */ +import * as fs from "fs-extra" +import * as path from "path" +import * as _ from "lodash" +import { slash } from "gatsby-core-utils" +import { store } from "../../redux" +import { IGatsbyState } from "../../redux/types" +import { requireGatsbyPlugin } from "../../utils/require-gatsby-plugin" + +const schemaCustomizationAPIs = new Set([ + `setFieldsOnGraphQLNodeType`, + `createSchemaCustomization`, + `createResolvers`, +]) + +const excludePlugins = new Set([`internal-data-bridge`, `default-site-plugin`]) +const includePlugins = new Set([`gatsby-plugin-sharp`]) + +// Emit file that imports required node APIs +const schemaCustomizationPluginsPath = + process.cwd() + `/.cache/query-engine-plugins.js` + +export async function printQueryEnginePlugins(): Promise { + try { + await fs.remove(schemaCustomizationPluginsPath) + } catch (e) { + // no-op + } + return await fs.writeFile( + schemaCustomizationPluginsPath, + renderQueryEnginePlugins() + ) +} + +function renderQueryEnginePlugins(): string { + const { flattenedPlugins } = store.getState() + const usedPlugins = flattenedPlugins.filter( + p => + includePlugins.has(p.name) || + (!excludePlugins.has(p.name) && + p.nodeAPIs.some(api => schemaCustomizationAPIs.has(api))) + ) + const usedSubPlugins = findSubPlugins(usedPlugins, flattenedPlugins) + return render(usedPlugins, usedSubPlugins) +} + +function relativePluginPath(resolve: string): string { + return slash( + path.relative(path.dirname(schemaCustomizationPluginsPath), resolve) + ) +} + +function render( + usedPlugins: IGatsbyState["flattenedPlugins"], + usedSubPlugins: IGatsbyState["flattenedPlugins"] +): string { + const uniqGatsbyNode = uniq(usedPlugins) + const uniqSubPlugins = uniq(usedSubPlugins) + + const sanitizedUsedPlugins = usedPlugins.map(plugin => { + // TODO: We don't support functions in pluginOptions here + return { + ...plugin, + resolve: ``, + pluginFilepath: ``, + subPluginPaths: undefined, + } + }) + + const pluginsWithWorkers = filterPluginsWithWorkers(uniqGatsbyNode) + + const subPluginModuleToImportNameMapping = new Map() + const imports: Array = [ + ...uniqGatsbyNode.map( + (plugin, i) => + `import * as pluginGatsbyNode${i} from "${relativePluginPath( + plugin.resolve + )}/gatsby-node"` + ), + ...pluginsWithWorkers.map( + (plugin, i) => + `import * as pluginGatsbyWorker${i} from "${relativePluginPath( + plugin.resolve + )}/gatsby-worker"` + ), + ...uniqSubPlugins.map((plugin, i) => { + const importName = `subPlugin${i}` + subPluginModuleToImportNameMapping.set(plugin.modulePath!, importName) + return `import * as ${importName} from "${relativePluginPath( + plugin.modulePath! + )}"` + }), + ] + const gatsbyNodeExports = uniqGatsbyNode.map( + (plugin, i) => `"${plugin.name}": pluginGatsbyNode${i},` + ) + const gatsbyWorkerExports = pluginsWithWorkers.map( + (plugin, i) => `"${plugin.name}": pluginGatsbyWorker${i},` + ) + const indexExports = uniqSubPlugins.map( + (plugin, i) => ` "${plugin.name}": subPlugin${i},` + ) + const output = ` +${imports.join(`\n`)} + +export const gatsbyNodes = { +${gatsbyNodeExports.join(`\n`)} +} + +export const gatsbyWorkers = { +${gatsbyWorkerExports.join(`\n`)} +} + +export const indexes = { +${indexExports.join(`\n`)} +} + +export const flattenedPlugins = + ${JSON.stringify( + sanitizedUsedPlugins.map(plugin => { + return { + ...plugin, + pluginOptions: _.cloneDeepWith( + plugin.pluginOptions, + (value: any): any => { + if (typeof value === `object` && value.module && value.modulePath) { + const { module, modulePath, ...subPlugin } = value + return { + ...subPlugin, + module: `_SKIP_START_${subPluginModuleToImportNameMapping.get( + modulePath + )}_SKIP_END_`, + resolve: ``, + pluginFilepath: ``, + } + } + return undefined + } + ), + } + }), + null, + 2 + ).replace(/"_SKIP_START_|_SKIP_END_"/g, ``)} +` + return output +} + +function filterPluginsWithWorkers( + plugins: IGatsbyState["flattenedPlugins"] +): IGatsbyState["flattenedPlugins"] { + return plugins.filter(plugin => { + try { + return Boolean(requireGatsbyPlugin(plugin, `gatsby-worker`)) + } catch (err) { + return false + } + }) +} + +type ArrayElement> = ArrayType extends Array< + infer ElementType +> + ? ElementType + : never + +function getSubpluginsByPluginPath( + parentPlugin: ArrayElement, + path: string +): IGatsbyState["flattenedPlugins"] { + const segments = path.split(`.`) + let roots: Array = [parentPlugin.pluginOptions] + + for (const segment of segments) { + if (segment === `[]`) { + roots = roots.flat() + } else { + roots = roots.map(root => root[segment]) + } + } + + return roots +} + +function findSubPlugins( + plugins: IGatsbyState["flattenedPlugins"], + allFlattenedPlugins: IGatsbyState["flattenedPlugins"] +): IGatsbyState["flattenedPlugins"] { + const usedSubPluginNames = new Set( + plugins + .flatMap(plugin => { + if (plugin.subPluginPaths) { + const subPlugins: IGatsbyState["flattenedPlugins"] = [] + for (const subPluginPath of plugin.subPluginPaths) { + subPlugins.push(...getSubpluginsByPluginPath(plugin, subPluginPath)) + } + return subPlugins + } + + return [] + }) + .map(plugin => plugin[`name`]) + .filter((p: unknown): p is string => typeof p === `string`) + ) + return allFlattenedPlugins.filter( + p => usedSubPluginNames.has(p.name) && !!p.modulePath + ) +} + +function uniq( + plugins: IGatsbyState["flattenedPlugins"] +): IGatsbyState["flattenedPlugins"] { + return Array.from(new Map(plugins.map(p => [p.name, p])).values()) +} diff --git a/packages/gatsby/src/schema/index.js b/packages/gatsby/src/schema/index.js index 9a721d0cf2312..f4227bbc697af 100644 --- a/packages/gatsby/src/schema/index.js +++ b/packages/gatsby/src/schema/index.js @@ -8,6 +8,7 @@ const { buildSchema, rebuildSchemaWithSitePage } = require(`./schema`) const { builtInFieldExtensions } = require(`./extensions`) const { builtInTypeDefinitions } = require(`./types/built-in-types`) const { TypeConflictReporter } = require(`./infer/type-conflict-reporter`) +const { shouldGenerateEngines } = require(`../utils/engines-helpers`) const getAllTypeDefinitions = () => { const { @@ -101,10 +102,19 @@ const build = async ({ schemaCustomization: { thirdPartySchemas, printConfig }, inferenceMetadata, config: { mapping: typeMapping }, + program: { directory }, } = store.getState() const typeConflictReporter = new TypeConflictReporter() + const enginePrintConfig = + _CFLAGS_.GATSBY_MAJOR === `4` && shouldGenerateEngines() + ? { + path: `${directory}/.cache/schema.gql`, + rewrite: true, + } + : undefined + const fieldExtensions = getAllFieldExtensions() const schemaComposer = createSchemaComposer({ fieldExtensions }) const schema = await buildSchema({ @@ -114,6 +124,7 @@ const build = async ({ thirdPartySchemas, typeMapping, printConfig, + enginePrintConfig, typeConflictReporter, inferenceMetadata, freeze, diff --git a/packages/gatsby/src/schema/print.js b/packages/gatsby/src/schema/print.js index 0bd0b98cdebb9..0e8d7c5a2b041 100644 --- a/packages/gatsby/src/schema/print.js +++ b/packages/gatsby/src/schema/print.js @@ -14,7 +14,13 @@ const { internalExtensionNames } = require(`./extensions`) const printTypeDefinitions = ({ config, schemaComposer }) => { if (!config) return Promise.resolve() - const { path, include = {}, exclude = {}, withFieldTypes } = config || {} + const { + path, + include = {}, + exclude = {}, + withFieldTypes, + rewrite = false, + } = config || {} if (!path) { report.error( @@ -23,7 +29,7 @@ const printTypeDefinitions = ({ config, schemaComposer }) => { return Promise.resolve() } - if (fs.existsSync(path)) { + if (!rewrite && fs.existsSync(path)) { report.error( `Printing type definitions aborted. The file \`${path}\` already exists.` ) diff --git a/packages/gatsby/src/schema/schema.js b/packages/gatsby/src/schema/schema.js index 1d63e9eb36e4d..dcdef53137742 100644 --- a/packages/gatsby/src/schema/schema.js +++ b/packages/gatsby/src/schema/schema.js @@ -58,6 +58,7 @@ const buildSchema = async ({ fieldExtensions, thirdPartySchemas, printConfig, + enginePrintConfig, typeConflictReporter, inferenceMetadata, freeze = false, @@ -72,6 +73,7 @@ const buildSchema = async ({ fieldExtensions, thirdPartySchemas, printConfig, + enginePrintConfig, typeConflictReporter, inferenceMetadata, parentSpan, @@ -159,6 +161,7 @@ const updateSchemaComposer = async ({ fieldExtensions, thirdPartySchemas, printConfig, + enginePrintConfig, typeConflictReporter, inferenceMetadata, parentSpan, @@ -190,11 +193,21 @@ const updateSchemaComposer = async ({ parentSpan: parentSpan, }) activity.start() - await printTypeDefinitions({ - config: printConfig, - schemaComposer, - parentSpan: activity.span, - }) + if (!process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE) { + await printTypeDefinitions({ + config: printConfig, + schemaComposer, + parentSpan: activity.span, + }) + if (enginePrintConfig) { + // make sure to print schema that will be used when bundling graphql-engine + await printTypeDefinitions({ + config: enginePrintConfig, + schemaComposer, + parentSpan: activity.span, + }) + } + } await addSetFieldsOnGraphQLNodeTypeFields({ schemaComposer, parentSpan: activity.span, diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index b680d3a2556dd..d573f83d3b0c1 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -26,6 +26,7 @@ const { emitter, store } = require(`../redux`) const { getNodes, getNode, getNodesByType } = require(`../datastore`) const { getNodeAndSavePathDependency, loadNodeContent } = require(`./nodes`) const { getPublicPath } = require(`./get-public-path`) +const { requireGatsbyPlugin } = require(`./require-gatsby-plugin`) const { getNonGatsbyCodeFrameFormatted } = require(`./stack-trace-utils`) const { trackBuildError, decorateEvent } = require(`gatsby-telemetry`) import errorParser from "./api-runner-error-parser" @@ -248,16 +249,10 @@ const getUninitializedCache = plugin => { } } -const pluginNodeCache = new Map() - const availableActionsCache = new Map() let publicPath const runAPI = async (plugin, api, args, activity) => { - let gatsbyNode = pluginNodeCache.get(plugin.name) - if (!gatsbyNode) { - gatsbyNode = require(`${plugin.resolve}/gatsby-node`) - pluginNodeCache.set(plugin.name, gatsbyNode) - } + const gatsbyNode = requireGatsbyPlugin(plugin, `gatsby-node`) if (gatsbyNode[api]) { const parentSpan = args && args.parentSpan @@ -447,7 +442,7 @@ const apisRunningById = new Map() const apisRunningByTraceId = new Map() let waitingForCasacadeToFinish = [] -module.exports = (api, args = {}, { pluginSource, activity } = {}) => { +function apiRunnerNode(api, args = {}, { pluginSource, activity } = {}) { const plugins = store.getState().flattenedPlugins // Get the list of plugins that implement this API. @@ -563,12 +558,7 @@ module.exports = (api, args = {}, { pluginSource, activity } = {}) => { return null } - let gatsbyNode = pluginNodeCache.get(plugin.name) - if (!gatsbyNode) { - gatsbyNode = require(`${plugin.resolve}/gatsby-node`) - pluginNodeCache.set(plugin.name, gatsbyNode) - } - + const gatsbyNode = requireGatsbyPlugin(plugin, `gatsby-node`) const pluginName = plugin.name === `default-site-plugin` ? `gatsby-node.js` : plugin.name @@ -686,3 +676,5 @@ module.exports = (api, args = {}, { pluginSource, activity } = {}) => { }) }) } + +module.exports = apiRunnerNode diff --git a/packages/gatsby/src/utils/engines-fs-provider.ts b/packages/gatsby/src/utils/engines-fs-provider.ts new file mode 100644 index 0000000000000..f4d8003646701 --- /dev/null +++ b/packages/gatsby/src/utils/engines-fs-provider.ts @@ -0,0 +1,32 @@ +// This module should be imported as first or one first modules in engines. +// It allows to provide alternative `fs` module by setting `global._fsWrapper` +// variable before importing engine. It will automatically fallback to regular +// `fs` if alternative `fs` is not provided. + +/* eslint-disable @typescript-eslint/no-namespace */ + +declare global { + namespace NodeJS { + // eslint-disable-next-line @typescript-eslint/naming-convention + interface Global { + _fsWrapper: typeof import("fs") + _actualFsWrapper: typeof import("fs") + } + } +} + +// eslint-disable-next-line @typescript-eslint/naming-convention +declare const __non_webpack_require__: typeof require + +if (global._fsWrapper) { + global._actualFsWrapper = global._fsWrapper +} else { + // fs alternative not provided - falling back to regular fs + global._actualFsWrapper = __non_webpack_require__(`fs`) +} + +// hydrate webpack module cache (consume global, so it's not lazy) +require(`fs`) + +// https://stackoverflow.com/a/59499895 +export {} diff --git a/packages/gatsby/src/utils/engines-helpers.ts b/packages/gatsby/src/utils/engines-helpers.ts new file mode 100644 index 0000000000000..56f8b18ffb3c8 --- /dev/null +++ b/packages/gatsby/src/utils/engines-helpers.ts @@ -0,0 +1,4 @@ +export function shouldGenerateEngines(): boolean { + // TODO: make it smart and actually check if there are any non-ssg pages + return true +} diff --git a/packages/gatsby/src/utils/jobs/__tests__/manager.js b/packages/gatsby/src/utils/jobs/__tests__/manager.js index e5c0b166b1125..ef8a3f7a17e06 100644 --- a/packages/gatsby/src/utils/jobs/__tests__/manager.js +++ b/packages/gatsby/src/utils/jobs/__tests__/manager.js @@ -17,7 +17,7 @@ jest.mock(`gatsby-cli/lib/reporter`, () => { }) jest.mock( - `/node_modules/gatsby-plugin-test/gatsby-worker.js`, + `/node_modules/gatsby-plugin-test/gatsby-worker`, () => { return { TEST_JOB: jest.fn(), @@ -27,7 +27,7 @@ jest.mock( ) jest.mock( - `/gatsby-plugin-local/gatsby-worker.js`, + `/gatsby-plugin-local/gatsby-worker`, () => { return { TEST_JOB: jest.fn(), @@ -40,7 +40,7 @@ jest.mock(`uuid/v4`, () => jest.fn().mockImplementation(jest.requireActual(`uuid/v4`)) ) -const worker = require(`/node_modules/gatsby-plugin-test/gatsby-worker.js`) +const worker = require(`/node_modules/gatsby-plugin-test/gatsby-worker`) const reporter = require(`gatsby-cli/lib/reporter`) const hasha = require(`hasha`) const fs = require(`fs-extra`) @@ -489,7 +489,7 @@ describe(`Jobs manager`, () => { it(`should run the worker locally when it's a local plugin`, async () => { jest.useRealTimers() - const worker = require(`/gatsby-plugin-local/gatsby-worker.js`) + const worker = require(`/gatsby-plugin-local/gatsby-worker`) const { enqueueJob, createInternalJob } = jobManager const jobArgs = createInternalJob(createMockJob(), { name: `gatsby-plugin-local`, diff --git a/packages/gatsby/src/utils/jobs/manager.ts b/packages/gatsby/src/utils/jobs/manager.ts index 144fa1d498d35..8907a219f1a14 100644 --- a/packages/gatsby/src/utils/jobs/manager.ts +++ b/packages/gatsby/src/utils/jobs/manager.ts @@ -17,6 +17,7 @@ import { IJobNotWhitelisted, WorkerError, } from "./types" +import { requireGatsbyPlugin } from "../require-gatsby-plugin" type IncomingMessages = IJobCompletedMessage | IJobFailed | IJobNotWhitelisted @@ -157,7 +158,7 @@ function runJob( ): Promise> { const { plugin } = job try { - const worker = require(path.posix.join(plugin.resolve, `gatsby-worker.js`)) + const worker = requireGatsbyPlugin(plugin, `gatsby-worker`) if (!worker[job.name]) { throw new Error(`No worker function found for ${job.name}`) } diff --git a/packages/gatsby/src/utils/require-gatsby-plugin.ts b/packages/gatsby/src/utils/require-gatsby-plugin.ts new file mode 100644 index 0000000000000..1fa3e702cdef3 --- /dev/null +++ b/packages/gatsby/src/utils/require-gatsby-plugin.ts @@ -0,0 +1,24 @@ +const pluginModuleCache = new Map() + +export function setGatsbyPluginCache( + plugin: { name: string; resolve: string }, + module: string, + moduleObject: any +): void { + const key = `${plugin.name}/${module}` + pluginModuleCache.set(key, moduleObject) +} + +export function requireGatsbyPlugin( + plugin: { name: string; resolve: string }, + module: string +): any { + const key = `${plugin.name}/${module}` + + let pluginModule = pluginModuleCache.get(key) + if (!pluginModule) { + pluginModule = require(`${plugin.resolve}/${module}`) + pluginModuleCache.set(key, pluginModule) + } + return pluginModule +} diff --git a/packages/gatsby/src/utils/stack-trace-utils.ts b/packages/gatsby/src/utils/stack-trace-utils.ts index e2ff01d4dba84..87786b8a7844e 100644 --- a/packages/gatsby/src/utils/stack-trace-utils.ts +++ b/packages/gatsby/src/utils/stack-trace-utils.ts @@ -12,11 +12,21 @@ const path = require(`path`) const chalk = require(`chalk`) const { isNodeInternalModulePath } = require(`gatsby-core-utils`) -const gatsbyLocation = path.dirname(require.resolve(`gatsby/package.json`)) -const reduxThunkLocation = path.dirname( +const getDirName = (arg: unknown): string => { + // Caveat related to executing in engines: + // After webpack bundling we would get number here (webpack module id) and that would crash when doing + // path.dirname(number). + if (typeof arg === `string`) { + return path.dirname(arg) + } + return `-cant-resolve-` +} + +const gatsbyLocation = getDirName(require.resolve(`gatsby/package.json`)) +const reduxThunkLocation = getDirName( require.resolve(`redux-thunk/package.json`) ) -const reduxLocation = path.dirname(require.resolve(`redux/package.json`)) +const reduxLocation = getDirName(require.resolve(`redux/package.json`)) const getNonGatsbyCallSite = (): StackFrame | undefined => stackTrace diff --git a/packages/gatsby/src/utils/worker/pool.ts b/packages/gatsby/src/utils/worker/pool.ts index f8306959619d4..4a3ecd00a659f 100644 --- a/packages/gatsby/src/utils/worker/pool.ts +++ b/packages/gatsby/src/utils/worker/pool.ts @@ -21,6 +21,7 @@ export const create = (): GatsbyWorkerPool => { numWorkers, env: { GATSBY_WORKER_POOL_WORKER: `true`, + GATSBY_SKIP_WRITING_SCHEMA_TO_FILE: `true`, }, }) diff --git a/yarn.lock b/yarn.lock index 8dc88b61b2d04..e1003f47c545d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5217,6 +5217,11 @@ async-retry "^1.3.1" debug "^3.1.0" +"@vercel/webpack-asset-relocator-loader@^1.6.0": + version "1.7.0" + resolved "https://registry.yarnpkg.com/@vercel/webpack-asset-relocator-loader/-/webpack-asset-relocator-loader-1.7.0.tgz#d3b707e0aba3111719f941dacb2408eff3c27319" + integrity sha512-1Dy3BdOliDwxA7VZSIg55E1d/us2KvsCQOZV25fgufG//CsnZBGiSAL7qewTQf7YVHH0A9PHgzwMmKIZ8aFYVw== + "@verdaccio/commons-api@9.7.1", "@verdaccio/commons-api@^9.7.1": version "9.7.1" resolved "https://registry.yarnpkg.com/@verdaccio/commons-api/-/commons-api-9.7.1.tgz#816f08eb6cb0dbe345f2546428c837be6804796d"