diff --git a/packages/gatsby-source-contentful/README.md b/packages/gatsby-source-contentful/README.md index 9bcc639294b73..3eec2f865f4f0 100644 --- a/packages/gatsby-source-contentful/README.md +++ b/packages/gatsby-source-contentful/README.md @@ -181,6 +181,12 @@ Number of entries to retrieve from Contentful at a time. Due to some technical l Number of workers to use when downloading Contentful assets. Due to technical limitations, opening too many concurrent requests can cause stalled downloads. If you encounter this issue you can set this param to a lower number than 50, e.g 25. +**`contentfulClientConfig`** [object][optional] [default: `{}`] + +Additional config which will get passed to [Contentfuls JS SDK](https://github.com/contentful/contentful.js#configuration). + +Use this with caution, you might override values this plugin does set for you to connect to Contentful. + ## Notes on Contentful Content Models There are currently some things to keep in mind when building your content models at Contentful. diff --git a/packages/gatsby-source-contentful/package.json b/packages/gatsby-source-contentful/package.json index c21f775b3ab2f..850a8f7c40fd2 100644 --- a/packages/gatsby-source-contentful/package.json +++ b/packages/gatsby-source-contentful/package.json @@ -31,7 +31,8 @@ "@babel/cli": "^7.12.1", "@babel/core": "^7.12.3", "babel-preset-gatsby-package": "^0.12.0", - "cross-env": "^7.0.3" + "cross-env": "^7.0.3", + "nock": "^13.0.6" }, "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-contentful#readme", "keywords": [ diff --git a/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js b/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js new file mode 100644 index 0000000000000..27c7045dadd87 --- /dev/null +++ b/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js @@ -0,0 +1,219 @@ +/** + * @jest-environment node + */ + +import nock from "nock" +import fetchData from "../fetch" +import { createPluginConfig } from "../plugin-options" + +nock.disableNetConnect() + +const host = `localhost` +const options = { + spaceId: `12345`, + accessToken: `67890`, + host, + contentfulClientConfig: { + retryLimit: 2, + }, +} +const baseURI = `https://${host}` + +const start = jest.fn() +const end = jest.fn() +const mockActivity = { + start, + end, + tick: jest.fn(), + done: end, +} + +const reporter = { + info: jest.fn(), + verbose: jest.fn(), + panic: jest.fn(e => { + throw e + }), + activityTimer: jest.fn(() => mockActivity), + createProgress: jest.fn(() => mockActivity), +} + +const pluginConfig = createPluginConfig(options) + +describe(`fetch-retry`, () => { + afterEach(() => { + nock.cleanAll() + reporter.verbose.mockClear() + reporter.panic.mockClear() + }) + + test(`request retries when network timeout happens`, async () => { + const scope = nock(baseURI) + // Space + .get(`/spaces/${options.spaceId}/`) + .reply(200, { items: [] }) + // Locales + .get(`/spaces/${options.spaceId}/environments/master/locales`) + .reply(200, { items: [{ code: `en`, default: true }] }) + // Sync + .get( + `/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=100` + ) + .times(1) + .replyWithError({ code: `ETIMEDOUT` }) + .get( + `/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=100` + ) + .reply(200, { items: [] }) + // Content types + .get( + `/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=100&order=sys.createdAt` + ) + .reply(200, { items: [] }) + + await fetchData({ pluginConfig, reporter }) + + expect(reporter.panic).not.toBeCalled() + expect(scope.isDone()).toBeTruthy() + }) + + test(`request should fail after to many retries`, async () => { + // Due to the retries, this can take up to 10 seconds + jest.setTimeout(10000) + + const scope = nock(baseURI) + // Space + .get(`/spaces/${options.spaceId}/`) + .reply(200, { items: [] }) + // Locales + .get(`/spaces/${options.spaceId}/environments/master/locales`) + .reply(200, { items: [{ code: `en`, default: true }] }) + // Sync + .get( + `/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=100` + ) + .times(3) + .reply( + 500, + { + sys: { + type: `Error`, + id: `MockedContentfulError`, + }, + message: `Mocked message of Contentful error`, + }, + { [`x-contentful-request-id`]: `123abc` } + ) + + try { + await fetchData({ pluginConfig, reporter }) + jest.fail() + } catch (e) { + const msg = expect(e.context.sourceMessage) + msg.toEqual( + expect.stringContaining( + `Fetching contentful data failed: 500 MockedContentfulError` + ) + ) + msg.toEqual(expect.stringContaining(`Request ID: 123abc`)) + msg.toEqual( + expect.stringContaining(`The request was sent with 3 attempts`) + ) + } + expect(reporter.panic).toBeCalled() + expect(scope.isDone()).toBeTruthy() + }) +}) + +describe(`fetch-network-errors`, () => { + test(`catches plain network error`, async () => { + const scope = nock(baseURI) + // Space + .get(`/spaces/${options.spaceId}/`) + .replyWithError({ code: `ECONNRESET` }) + try { + await fetchData({ + pluginConfig: createPluginConfig({ + ...options, + contentfulClientConfig: { retryOnError: false }, + }), + reporter, + }) + jest.fail() + } catch (e) { + expect(e.context.sourceMessage).toEqual( + expect.stringContaining( + `Accessing your Contentful space failed: ECONNRESET` + ) + ) + } + + expect(reporter.panic).toBeCalled() + expect(scope.isDone()).toBeTruthy() + }) + + test(`catches error with response string`, async () => { + const scope = nock(baseURI) + // Space + .get(`/spaces/${options.spaceId}/`) + .reply(502, `Bad Gateway`) + + try { + await fetchData({ + pluginConfig: createPluginConfig({ + ...options, + contentfulClientConfig: { retryOnError: false }, + }), + reporter, + }) + jest.fail() + } catch (e) { + expect(e.context.sourceMessage).toEqual( + expect.stringContaining( + `Accessing your Contentful space failed: Bad Gateway` + ) + ) + } + + expect(reporter.panic).toBeCalled() + expect(scope.isDone()).toBeTruthy() + }) + + test(`catches error with response object`, async () => { + const scope = nock(baseURI) + // Space + .get(`/spaces/${options.spaceId}/`) + .reply(429, { + sys: { + type: `Error`, + id: `MockedContentfulError`, + }, + message: `Mocked message of Contentful error`, + requestId: `123abc`, + }) + + try { + await fetchData({ + pluginConfig: createPluginConfig({ + ...options, + contentfulClientConfig: { retryOnError: false }, + }), + reporter, + }) + jest.fail() + } catch (e) { + const msg = expect(e.context.sourceMessage) + + msg.toEqual( + expect.stringContaining( + `Accessing your Contentful space failed: MockedContentfulError` + ) + ) + msg.toEqual(expect.stringContaining(`Mocked message of Contentful error`)) + msg.toEqual(expect.stringContaining(`Request ID: 123abc`)) + } + + expect(reporter.panic).toBeCalled() + expect(scope.isDone()).toBeTruthy() + }) +}) diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index b66d027ddb61c..f949b275b5fd3 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -4,6 +4,74 @@ const chalk = require(`chalk`) const { formatPluginOptionsForCLI } = require(`./plugin-options`) const { CODES } = require(`./report`) +/** + * Generate a user friendly error message. + * + * Contentful's API has its own error message structure, which might change depending of internal server or authentification errors. + * + * Additionally the SDK strips the error object, sometimes: + * https://github.com/contentful/contentful.js/blob/b67b77ac8c919c4ec39203f8cac2043854ab0014/lib/create-contentful-api.js#L89-L99 + * + * This code tries to work around this. + */ +const createContentfulErrorMessage = e => { + if (typeof e === `string`) { + return e + } + // If we got a response, it is very likely that it is a Contentful API error. + if (e.response) { + let parsedContentfulErrorData = null + + // Parse JSON response data, and add it to the object. + if (typeof e.response.data === `string`) { + try { + parsedContentfulErrorData = JSON.parse(e.response.data) + } catch (err) { + e.message = e.response.data + } + // If response data was parsed already, just add it. + } else if (typeof e.response.data === `object`) { + parsedContentfulErrorData = e.response.data + } + + e = { ...e, ...e.response, ...parsedContentfulErrorData } + } + + let errorMessage = [ + // Generic error values + e.code && String(e.code), + e.status && String(e.status), + e.statusText, + // Contentful API error response values + e.sys?.id, + ] + .filter(Boolean) + .join(` `) + + // Add message if it exists. Usually error default or Contentful's error message + if (e.message) { + errorMessage += `\n\n${e.message}` + } + + // Get request ID from headers or Contentful's error data + const requestId = + (e.headers && + typeof e.headers === `object` && + e.headers[`x-contentful-request-id`]) || + e.requestId + + if (requestId) { + errorMessage += `\n\nRequest ID: ${requestId}` + } + + // Tell the user about how many request attempts Contentful SDK made + if (e.attempts) { + errorMessage += `\n\nThe request was sent with ${e.attempts} attempts` + } + + return errorMessage +} + module.exports = async function contentfulFetch({ syncToken, pluginConfig, @@ -11,7 +79,6 @@ module.exports = async function contentfulFetch({ }) { // Fetch articles. let syncProgress - let syncItemCount = 0 const pageLimit = pluginConfig.get(`pageLimit`) const contentfulClientOptions = { space: pluginConfig.get(`spaceId`), @@ -22,8 +89,8 @@ module.exports = async function contentfulFetch({ integration: `gatsby-source-contentful`, responseLogger: response => { function createMetadataLog(response) { - if (process.env.gatsby_log_level === `verbose`) { - return `` + if (!response.headers) { + return null } return [ response?.headers[`content-length`] && @@ -37,42 +104,30 @@ module.exports = async function contentfulFetch({ .join(` `) } - // Log error and throw it in an extended shape - if (response.isAxiosError) { - reporter.verbose( - `${response.config.method} /${response.config.url}: ${ - response.response.status - } ${response.response.statusText} (${createMetadataLog( - response.response - )})` - ) - let errorMessage = `${response.response.status} ${response.response.statusText}` - if (response.response?.data?.message) { - errorMessage += `\n\n${response.response.data.message}` - } - const contentfulApiError = new Error(errorMessage) - // Special response naming to ensure the error object is not touched by - // https://github.com/contentful/contentful.js/commit/41039afa0c1462762514c61458556e6868beba61 - contentfulApiError.responseData = response.response - contentfulApiError.request = response.request - contentfulApiError.config = response.config - - throw contentfulApiError - } - // Sync progress - if (response.config.url === `sync`) { - syncItemCount += response.data.items.length - syncProgress.total = syncItemCount + if ( + response.config.url === `sync` && + !response.isAxiosError && + response?.data.items + ) { syncProgress.tick(response.data.items.length) } + const metadataLog = createMetadataLog(response) + reporter.verbose( - `${response.config.method} /${response.config.url}: ${ - response.status - } ${response.statusText} (${createMetadataLog(response)})` + [ + `${response.config.method} /${response.config.url}:`, + response.status, + response.statusText, + metadataLog && `(${metadataLog})`, + ] + .filter(Boolean) + .join(` `) ) }, + // Allow passing of custom configuration to the Contentful SDK like headers + ...(pluginConfig.get(`contentfulClientConfig`) || {}), } const client = contentful.createClient(contentfulClientOptions) @@ -98,15 +153,12 @@ module.exports = async function contentfulFetch({ if (e.code === `ENOTFOUND`) { details = `You seem to be offline` } else if (e.code === `SELF_SIGNED_CERT_IN_CHAIN`) { - reporter.panic( - { - id: CODES.SelfSignedCertificate, - context: { - sourceMessage: `We couldn't make a secure connection to your contentful space. Please check if you have any self-signed SSL certificates installed.`, - }, + reporter.panic({ + id: CODES.SelfSignedCertificate, + context: { + sourceMessage: `We couldn't make a secure connection to your contentful space. Please check if you have any self-signed SSL certificates installed.`, }, - e - ) + }) } else if (e.responseData) { if (e.responseData.status === 404) { // host and space used to generate url @@ -131,7 +183,10 @@ module.exports = async function contentfulFetch({ reporter.panic({ context: { - sourceMessage: `Accessing your Contentful space failed: ${e.message} + sourceMessage: `Accessing your Contentful space failed: ${createContentfulErrorMessage( + e + )} + Try setting GATSBY_CONTENTFUL_OFFLINE=true to see if we can serve from cache. ${details ? `\n${details}\n` : ``} Used options: @@ -158,15 +213,14 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, : { initial: true, ...basicSyncConfig } currentSyncData = await client.sync(query) } catch (e) { - reporter.panic( - { - id: CODES.SyncError, - context: { - sourceMessage: `Fetching contentful data failed: ${e.message}`, - }, + reporter.panic({ + id: CODES.SyncError, + context: { + sourceMessage: `Fetching contentful data failed: ${createContentfulErrorMessage( + e + )}`, }, - e - ) + }) } finally { syncProgress.done() } @@ -177,15 +231,14 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, try { contentTypes = await pagedGet(client, `getContentTypes`, pageLimit) } catch (e) { - reporter.panic( - { - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Error fetching content types: ${e.message}`, - }, + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Error fetching content types: ${createContentfulErrorMessage( + e + )}`, }, - e - ) + }) } reporter.verbose(`Content types fetched ${contentTypes.items.length}`) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index ae0ce31b87a8d..0c229d3685d02 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -45,7 +45,7 @@ const validateContentfulAccess = async pluginOptions => { pluginOptions.spaceId )}" on environment "${ pluginOptions.environment - } with access token "${maskText( + }" with access token "${maskText( pluginOptions.accessToken )}". Make sure to double check them!` @@ -134,6 +134,14 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo If you are confident your Content Types will have natural-language IDs (e.g. \`blogPost\`), then you should set this option to \`false\`. If you are unable to ensure this, then you should leave this option set to \`true\` (the default).` ) .default(true), + contentfulClientConfig: Joi.object() + .description( + `Additional config which will get passed to [Contentfuls JS SDK](https://github.com/contentful/contentful.js#configuration). + + Use this with caution, you might override values this plugin does set for you to connect to Contentful.` + ) + .unknown(true) + .default({}), // default plugins passed by gatsby plugins: Joi.array(), }) diff --git a/yarn.lock b/yarn.lock index 98ab5133b1930..5f8581479143a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18039,6 +18039,16 @@ no-case@^3.0.3: lower-case "^2.0.1" tslib "^1.10.0" +nock@^13.0.6: + version "13.0.11" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.0.11.tgz#ba733252e720897ca50033205c39db0c7470f331" + integrity sha512-sKZltNkkWblkqqPAsjYW0bm3s9DcHRPiMOyKO/PkfJ+ANHZ2+LA2PLe22r4lLrKgXaiSaDQwW3qGsJFtIpQIeQ== + dependencies: + debug "^4.1.0" + json-stringify-safe "^5.0.1" + lodash.set "^4.3.2" + propagate "^2.0.0" + node-abi@^2.7.0: version "2.8.0" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-2.8.0.tgz#bd2e88dbe6a6871e6dd08553e0605779325737ec" @@ -20380,6 +20390,11 @@ prop-types@^15.5.10, prop-types@^15.6.0, prop-types@^15.6.1, prop-types@^15.6.2, object-assign "^4.1.1" react-is "^16.8.1" +propagate@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" + integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== + proper-lockfile@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.1.tgz#284cf9db9e30a90e647afad69deb7cb06881262c"