diff --git a/packages/gatsby-plugin-gatsby-cloud/src/__tests__/build-headers-program.js b/packages/gatsby-plugin-gatsby-cloud/src/__tests__/build-headers-program.js index fcafe533ef132..ab467262bdd60 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/__tests__/build-headers-program.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/__tests__/build-headers-program.js @@ -319,15 +319,18 @@ describe(`build-headers-program`, () => { process.send = jest.fn() await buildHeadersProgram(pluginData, pluginOptions) - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_HEADER_ENTRY`, - payload: { - url: `/hello`, - headers: [`X-Frame-Options: SAMEORIGIN`], + expect(process.send).toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_HEADER_ENTRY`, + payload: { + url: `/hello`, + headers: [`X-Frame-Options: SAMEORIGIN`], + }, }, }, - }) + expect.any(Function) + ) }) }) diff --git a/packages/gatsby-plugin-gatsby-cloud/src/__tests__/create-redirects.js b/packages/gatsby-plugin-gatsby-cloud/src/__tests__/create-redirects.js index fa1383c27e083..bfec4ca5d8e4d 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/__tests__/create-redirects.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/__tests__/create-redirects.js @@ -269,27 +269,33 @@ describe(`create-redirects`, () => { ] ) - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_REDIRECT_ENTRY`, - payload: { - fromPath: `/old-url`, - toPath: `/new-url`, - isPermanent: true, + expect(process.send).toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_REDIRECT_ENTRY`, + payload: { + fromPath: `/old-url`, + toPath: `/new-url`, + isPermanent: true, + }, }, }, - }) + expect.any(Function) + ) - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_REWRITE_ENTRY`, - payload: { - fromPath: `/url_that_is/ugly`, - toPath: `/not_ugly/url`, + expect(process.send).toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_REWRITE_ENTRY`, + payload: { + fromPath: `/url_that_is/ugly`, + toPath: `/not_ugly/url`, + }, }, }, - }) + expect.any(Function) + ) }) }) diff --git a/packages/gatsby-plugin-gatsby-cloud/src/__tests__/routes.js b/packages/gatsby-plugin-gatsby-cloud/src/__tests__/routes.js index f7ce1eaeb4408..e2a2e6264c470 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/__tests__/routes.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/__tests__/routes.js @@ -64,111 +64,79 @@ describe(`Routes IPC`, () => { ) if (os.platform() !== `win32`) { - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "index.html": `DSR`, - "page-data/index/page-data.json": `DSR`, - }, - }, - }, - }) - - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "path/1/index.html": `DSR`, - "page-data/path/1/page-data.json": `DSR`, - }, - }, - }, - }) - - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "path/2/index.html": `SSR`, - "page-data/path/2/page-data.json": `SSR`, + expect(process.send).toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_ROUTE`, + payload: { + routes: { + "index.html": `DSR`, + "page-data/index/page-data.json": `DSR`, + "path/1/index.html": `DSR`, + "page-data/path/1/page-data.json": `DSR`, + "path/2/index.html": `SSR`, + "page-data/path/2/page-data.json": `SSR`, + }, }, }, }, - }) + expect.any(Function) + ) - expect(process.send).not.toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "path/3/index.html": `SSG`, - "page-data/path/3/page-data.json": `SSG`, + expect(process.send).not.toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_ROUTE`, + payload: { + routes: { + "path/3/index.html": `SSG`, + "page-data/path/3/page-data.json": `SSG`, + }, }, }, }, - }) + expect.any(Function) + ) } if (os.platform() === `win32`) { - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "index.html": `DSR`, - "page-data\\index\\page-data.json": `DSR`, - }, - }, - }, - }) - - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "path\\1\\index.html": `DSR`, - "page-data\\path\\1\\page-data.json": `DSR`, - }, - }, - }, - }) - - expect(process.send).toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "path\\2\\index.html": `SSR`, - "page-data\\path\\2\\page-data.json": `SSR`, + expect(process.send).toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_ROUTE`, + payload: { + routes: { + "index.html": `DSR`, + "page-data\\index\\page-data.json": `DSR`, + "path\\1\\index.html": `DSR`, + "page-data\\path\\1\\page-data.json": `DSR`, + "path\\2\\index.html": `SSR`, + "page-data\\path\\2\\page-data.json": `SSR`, + }, }, }, }, - }) + expect.any(Function) + ) - expect(process.send).not.toHaveBeenCalledWith({ - type: `LOG_ACTION`, - action: { - type: `CREATE_ROUTE`, - payload: { - routes: { - "path\\3\\index.html": `SSG`, - "page-data\\path\\3\\page-data.json": `SSG`, + expect(process.send).not.toHaveBeenCalledWith( + { + type: `LOG_ACTION`, + action: { + type: `CREATE_ROUTE`, + payload: { + routes: { + "path\\3\\index.html": `SSG`, + "page-data\\path\\3\\page-data.json": `SSG`, + }, }, }, }, - }) + expect.any(Function) + ) } }) }) diff --git a/packages/gatsby-plugin-gatsby-cloud/src/build-headers-program.js b/packages/gatsby-plugin-gatsby-cloud/src/build-headers-program.js index 942315442ab4b..f65f147685e86 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/build-headers-program.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/build-headers-program.js @@ -310,40 +310,50 @@ const applyTransfromHeaders = headers => _.mapValues(headers, transformHeaders) -const writeHeadersFile = - ({ publicFolder }) => - contents => - new Promise((resolve, reject) => { - /** - * Emit Headers via IPC - */ - Object.entries(contents).map(([k, val]) => { - emitHeaders({ - url: k, - headers: val, - }) - }) +const sendHeadersViaIPC = async headers => { + /** + * Emit Headers via IPC + */ + let lastMessage + Object.entries(headers).forEach(([k, val]) => { + lastMessage = emitHeaders({ + url: k, + headers: val, + }) + }) + await lastMessage +} - const contentsStr = JSON.stringify(contents) - const writeStream = createWriteStream(publicFolder(HEADERS_FILENAME)) - const chunkSize = 10000 - const numChunks = Math.ceil(contentsStr.length / chunkSize) - - for (let i = 0; i < numChunks; i++) { - writeStream.write( - contentsStr.slice( - i * chunkSize, - Math.min((i + 1) * chunkSize, contentsStr.length) - ) +const writeHeadersFile = async (publicFolder, contents) => + new Promise((resolve, reject) => { + const contentsStr = JSON.stringify(contents) + const writeStream = createWriteStream(publicFolder(HEADERS_FILENAME)) + const chunkSize = 10000 + const numChunks = Math.ceil(contentsStr.length / chunkSize) + + for (let i = 0; i < numChunks; i++) { + writeStream.write( + contentsStr.slice( + i * chunkSize, + Math.min((i + 1) * chunkSize, contentsStr.length) ) - } + ) + } - writeStream.end() - writeStream.on(`finish`, () => { - resolve() - }) - writeStream.on(`error`, reject) + writeStream.end() + writeStream.on(`finish`, () => { + resolve() }) + writeStream.on(`error`, reject) + }) + +const saveHeaders = + ({ publicFolder }) => + contents => + Promise.all([ + sendHeadersViaIPC(contents), + writeHeadersFile(publicFolder, contents), + ]) export default function buildHeadersProgram(pluginData, pluginOptions) { return _.flow( @@ -353,6 +363,6 @@ export default function buildHeadersProgram(pluginData, pluginOptions) { mapUserLinkAllPageHeaders(pluginData, pluginOptions), applyLinkHeaders(pluginData, pluginOptions), applyTransfromHeaders(pluginOptions), - writeHeadersFile(pluginData) + saveHeaders(pluginData) )(pluginOptions.headers) } diff --git a/packages/gatsby-plugin-gatsby-cloud/src/create-redirects.js b/packages/gatsby-plugin-gatsby-cloud/src/create-redirects.js index a9d26a292545c..534cb29eaea36 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/create-redirects.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/create-redirects.js @@ -29,17 +29,21 @@ export default async function writeRedirectsFile( /** * IPC Emit for redirects */ + let lastMessageSent redirects.forEach(redirect => { - emitRedirects(redirect) + lastMessageSent = emitRedirects(redirect) }) /** * IPC Emit for rewrites */ rewrites.forEach(rewrite => { - emitRewrites(rewrite) + lastMessageSent = emitRewrites(rewrite) }) + // This prevents process from exiting before handling the last IPC message + await lastMessageSent + // Is it ok to pass through the data or should we format it so that we don't have dependencies // between the redirects and rewrites formats? What are the chances those will change? const FILE_PATH = publicFolder(REDIRECTS_FILENAME) diff --git a/packages/gatsby-plugin-gatsby-cloud/src/gatsby-node.js b/packages/gatsby-plugin-gatsby-cloud/src/gatsby-node.js index cd3b40a70871c..20130773f2e0f 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/gatsby-node.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/gatsby-node.js @@ -48,14 +48,23 @@ exports.onPostBuild = async ( /** * Emit via IPC routes for which pages are non SSG */ + let index = 0 + let batch = {} for (const [pathname, page] of pages) { if (page.mode && page.mode !== `SSG`) { - emitRoutes({ - [generateHtmlPath(``, pathname)]: page.mode, - [generatePageDataPath(``, pathname)]: page.mode, - }) + index++ + batch[generateHtmlPath(``, pathname)] = page.mode + batch[generatePageDataPath(``, pathname)] = page.mode + + if (index % 1000 === 0) { + await emitRoutes(batch) + batch = {} + } } } + if (Object.keys(batch).length > 0) { + await emitRoutes(batch) + } let nodesCount @@ -90,8 +99,9 @@ exports.onPostBuild = async ( const fileNodes = getNodesByType(`File`) // TODO: This is missing the cacheLocations .cache/caches + .cache/caches-lmdb + let fileNodesEmitted for (const file of fileNodes) { - emitFileNodes({ + fileNodesEmitted = emitFileNodes({ path: file.absolutePath, }) } @@ -115,6 +125,7 @@ exports.onPostBuild = async ( } await Promise.all([ + fileNodesEmitted, buildHeadersProgram(pluginData, pluginOptions), createSiteConfig(pluginData, pluginOptions), createRedirects(pluginData, redirects, rewrites), diff --git a/packages/gatsby-plugin-gatsby-cloud/src/ipc.js b/packages/gatsby-plugin-gatsby-cloud/src/ipc.js index 9521309dcd1bb..eb998ff310145 100644 --- a/packages/gatsby-plugin-gatsby-cloud/src/ipc.js +++ b/packages/gatsby-plugin-gatsby-cloud/src/ipc.js @@ -1,9 +1,36 @@ -export function emitRoutes(routes) { +function sendOrPromise(msg) { if (!process.send) { - return + return false } + let resolve + let reject + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + // `process.send` can buffer messages and delay their delivery. In this case it returns false. + // if process exists before all messages in the buffer are processed - remaining messages will be lost. + // + // The function returns sync `true` when message is handled immediately and Promise when + // it is buffered (this is essentially a backpressure for lots of sync calls to `process.send`). + // + // Callers must `await` results of this function to ensure process doesn't exit early. + // + // See also https://github.com/nodejs/node/issues/7657#issuecomment-253744600 + // and https://nodejs.org/dist/latest-v14.x/docs/api/child_process.html#child_process_subprocess_send_message_sendhandle_options_callback + // + const sent = process.send(msg, error => { + if (error) { + reject(error) + } else { + resolve(true) + } + }) + return sent === false ? promise : true +} - process.send({ +export function emitRoutes(routes) { + return sendOrPromise({ type: `LOG_ACTION`, action: { type: `CREATE_ROUTE`, @@ -15,11 +42,7 @@ export function emitRoutes(routes) { } export function emitRedirects(redirect) { - if (!process.send) { - return - } - - process.send({ + return sendOrPromise({ type: `LOG_ACTION`, action: { type: `CREATE_REDIRECT_ENTRY`, @@ -29,11 +52,7 @@ export function emitRedirects(redirect) { } export function emitRewrites(rewrite) { - if (!process.send) { - return - } - - process.send({ + return sendOrPromise({ type: `LOG_ACTION`, action: { type: `CREATE_REWRITE_ENTRY`, @@ -43,11 +62,7 @@ export function emitRewrites(rewrite) { } export function emitHeaders(header) { - if (!process.send) { - return - } - - process.send({ + return sendOrPromise({ type: `LOG_ACTION`, action: { type: `CREATE_HEADER_ENTRY`, @@ -57,11 +72,7 @@ export function emitHeaders(header) { } export function emitFileNodes(file) { - if (!process.send) { - return - } - - process.send({ + return sendOrPromise({ type: `LOG_ACTION`, action: { type: `CREATE_FILE_NODE`,