From 3f8c55b087db6ef0a6e77872f6fda14b75182720 Mon Sep 17 00:00:00 2001 From: Matt Kane Date: Thu, 27 Apr 2023 06:58:40 +0100 Subject: [PATCH] feat(gatsby-source-drupal): add `typePrefix` option (#37967) --- examples/using-drupal/gatsby-config.js | 1 + examples/using-drupal/src/pages/index.js | 12 +- examples/using-drupal/src/pages/recipes.js | 4 +- ...pes.title}.js => {DrupalRecipes.title}.js} | 4 +- .../babel-preset-gatsby-package/package.json | 2 +- packages/gatsby-source-drupal/README.md | 31 ++++ .../gatsby-source-drupal/src/gatsby-node.ts | 25 +-- .../gatsby-source-drupal/src/normalize.ts | 148 +++++++++++------- packages/gatsby-source-drupal/src/utils.ts | 109 ++++++------- 9 files changed, 205 insertions(+), 131 deletions(-) rename examples/using-drupal/src/pages/{{Recipes.title}.js => {DrupalRecipes.title}.js} (97%) diff --git a/examples/using-drupal/gatsby-config.js b/examples/using-drupal/gatsby-config.js index 6994c5410cc1d..b02c0a9ce1869 100644 --- a/examples/using-drupal/gatsby-config.js +++ b/examples/using-drupal/gatsby-config.js @@ -8,6 +8,7 @@ module.exports = { options: { baseUrl: `https://live-contentacms.pantheonsite.io/`, apiBase: `api`, + typePrefix: "Drupal", }, }, { diff --git a/examples/using-drupal/src/pages/index.js b/examples/using-drupal/src/pages/index.js index 182b9478489f4..47f163e6dee1a 100644 --- a/examples/using-drupal/src/pages/index.js +++ b/examples/using-drupal/src/pages/index.js @@ -220,11 +220,11 @@ export default IndexPage export const pageQuery = graphql` { - topRecipe: allRecipes(sort: { createdAt: ASC }, limit: 1) { + topRecipe: allDrupalRecipes(sort: { createdAt: ASC }, limit: 1) { edges { node { title - gatsbyPath(filePath: "/{Recipes.title}") + gatsbyPath(filePath: "/{DrupalRecipes.title}") relationships { image { relationships { @@ -243,7 +243,7 @@ export const pageQuery = graphql` } } } - nextTwoPromotedRecipes: allRecipes( + nextTwoPromotedRecipes: allDrupalRecipes( sort: { createdAt: ASC } limit: 2 skip: 1 @@ -251,7 +251,7 @@ export const pageQuery = graphql` edges { node { title - gatsbyPath(filePath: "/{Recipes.title}") + gatsbyPath(filePath: "/{DrupalRecipes.title}") relationships { category { name @@ -273,7 +273,7 @@ export const pageQuery = graphql` } } } - nextFourPromotedRecipes: allRecipes( + nextFourPromotedRecipes: allDrupalRecipes( sort: { createdAt: ASC } limit: 4 skip: 3 @@ -282,7 +282,7 @@ export const pageQuery = graphql` node { id title - gatsbyPath(filePath: "/{Recipes.title}") + gatsbyPath(filePath: "/{DrupalRecipes.title}") relationships { category { name diff --git a/examples/using-drupal/src/pages/recipes.js b/examples/using-drupal/src/pages/recipes.js index ce53bc0ada115..b183c1f38a731 100644 --- a/examples/using-drupal/src/pages/recipes.js +++ b/examples/using-drupal/src/pages/recipes.js @@ -23,11 +23,11 @@ export default Recipes export const query = graphql` query { - recipes: allRecipes(limit: 1000) { + recipes: allDrupalRecipes(limit: 1000) { edges { node { title - gatsbyPath(filePath: "/{Recipes.title}") + gatsbyPath(filePath: "/{DrupalRecipes.title}") } } } diff --git a/examples/using-drupal/src/pages/{Recipes.title}.js b/examples/using-drupal/src/pages/{DrupalRecipes.title}.js similarity index 97% rename from examples/using-drupal/src/pages/{Recipes.title}.js rename to examples/using-drupal/src/pages/{DrupalRecipes.title}.js index f7327868f0725..996b9adb82bc6 100644 --- a/examples/using-drupal/src/pages/{Recipes.title}.js +++ b/examples/using-drupal/src/pages/{DrupalRecipes.title}.js @@ -83,8 +83,8 @@ const RecipeTemplate = ({ data }) => ( export default RecipeTemplate export const query = graphql` - query($id: String!) { - recipe: recipes(id: { eq: $id }) { + query ($id: String!) { + recipe: drupalRecipes(id: { eq: $id }) { title preparationTime difficulty diff --git a/packages/babel-preset-gatsby-package/package.json b/packages/babel-preset-gatsby-package/package.json index deda410445124..f1b97b8b0df4d 100644 --- a/packages/babel-preset-gatsby-package/package.json +++ b/packages/babel-preset-gatsby-package/package.json @@ -44,6 +44,6 @@ "scripts": { "build": "", "prepare": "cross-env NODE_ENV=production npm run build", - "watch": "babel -w src --out-dir dist/ --ignore \"**/__tests__\" --extensions \".ts,.js\"" + "watch": "" } } diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md index c6f4c5185d187..8b480fdc7e689 100644 --- a/packages/gatsby-source-drupal/README.md +++ b/packages/gatsby-source-drupal/README.md @@ -486,6 +486,37 @@ module.exports = { Some entities are not translatable like Drupal files and will return null result when language code from parent entity doesn't match up. These items can be specified as nonTranslatableEntities and receive the defaultLanguage as fallback. +## Type prefix + +By default, types are created with names that match the types in Drupal. However you can use the `typePrefix` option to add a prefix to all types. This is useful if you have multiple Drupal sources and want to differentiate between them, or if you have Drupal types that conflict with other types in your site. + +```javascript +// In your gatsby-config.js +module.exports = { + plugins: [ + { + resolve: `gatsby-source-drupal`, + options: { + baseUrl: `https://live-contentacms.pantheonsite.io/`, + typePrefix: `Drupal`, + }, + }, + ], +} +``` + +You would then query for `allDrupalArticle` instead of `allArticle`. + +```graphql +{ + allDrupalArticle { + nodes { + title + } + } +} +``` + ## Gatsby Preview (experimental) You will need to have the Drupal module installed, more information on that here: https://www.drupal.org/project/gatsby diff --git a/packages/gatsby-source-drupal/src/gatsby-node.ts b/packages/gatsby-source-drupal/src/gatsby-node.ts index b5a51b50a0156..bbef738ac1c79 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.ts +++ b/packages/gatsby-source-drupal/src/gatsby-node.ts @@ -23,15 +23,14 @@ import { imageCDNState, } from "./normalize" -const { +import { handleReferences, handleWebhookUpdate, createNodeIfItDoesNotExist, handleDeletedNode, drupalCreateNodeManifest, getExtendedFileNodeData, -} = require(`./utils`) - +} from "./utils" const imageCdnDocs = `https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-source-drupal#readme` const agent = { @@ -243,6 +242,7 @@ ${JSON.stringify(webhookBody, null, 4)}` createNodeId, createContentDigest, entityReferenceRevisions, + pluginOptions, }) reporter.log(`Deleted node: ${deletedNode.id}`) } @@ -264,6 +264,7 @@ ${JSON.stringify(webhookBody, null, 4)}` createContentDigest, getNode, reporter, + pluginOptions, }) } @@ -278,8 +279,6 @@ ${JSON.stringify(webhookBody, null, 4)}` getCache, getNode, reporter, - store, - languageConfig, }, pluginOptions ) @@ -402,6 +401,7 @@ ${JSON.stringify(webhookBody, null, 4)}` createContentDigest, getNode, reporter, + pluginOptions, }) } } @@ -416,6 +416,7 @@ ${JSON.stringify(webhookBody, null, 4)}` createNodeId, createContentDigest, entityReferenceRevisions, + pluginOptions, }) } else { // The data could be a single Drupal entity or an array of Drupal @@ -436,8 +437,6 @@ ${JSON.stringify(webhookBody, null, 4)}` getCache, getNode, reporter, - store, - languageConfig, }, pluginOptions ) @@ -773,6 +772,7 @@ ${JSON.stringify(webhookBody, null, 4)}` createNodeId, cache, entityReferenceRevisions, + pluginOptions, }) } @@ -782,7 +782,9 @@ ${JSON.stringify(webhookBody, null, 4)}` reporter.info(`Downloading remote files from Drupal`) // Download all files (await for each pool to complete to fix concurrency issues) - const fileNodes = [...nodes.values()].filter(isFileNode) + const fileNodes = [...nodes.values()].filter(node => + isFileNode(node, pluginOptions.typePrefix) + ) if (fileNodes.length) { const downloadingFilesActivity = reporter.activityTimer( @@ -795,12 +797,10 @@ ${JSON.stringify(webhookBody, null, 4)}` await downloadFile( { node, - store, cache, createNode, createNodeId, getCache, - reporter, }, pluginOptions ) @@ -833,7 +833,6 @@ exports.onCreateDevServer = ( createNodeId, getNode, actions, - store, cache, createContentDigest, getCache, @@ -873,7 +872,6 @@ exports.onCreateDevServer = ( getCache, getNode, reporter, - store, }, pluginOptions ) @@ -913,6 +911,9 @@ exports.pluginOptionsSchema = ({ Joi }) => disallowedLinkTypes: Joi.array().items(Joi.string()), skipFileDownloads: Joi.boolean(), imageCDN: Joi.boolean().default(true), + typePrefix: Joi.string() + .description(`Prefix for Drupal node types`) + .default(``), fastBuilds: Joi.boolean(), entityReferenceRevisions: Joi.array().items(Joi.string()), secret: Joi.string().description( diff --git a/packages/gatsby-source-drupal/src/normalize.ts b/packages/gatsby-source-drupal/src/normalize.ts index f5883d40d5d6e..d9241bf7a8da3 100644 --- a/packages/gatsby-source-drupal/src/normalize.ts +++ b/packages/gatsby-source-drupal/src/normalize.ts @@ -1,6 +1,7 @@ const { URL } = require(`url`) const { createRemoteFileNode } = require(`gatsby-source-filesystem`) const path = require(`path`) +const { capitalize } = require(`lodash`) const probeImageSize = require(`probe-image-size`) import { getOptions } from "./plugin-options" @@ -33,11 +34,14 @@ const getGatsbyImageCdnFields = async ({ return {} } - const isFile = isFileNode({ - internal: { - type, + const isFile = isFileNode( + { + internal: { + type, + }, }, - }) + pluginOptions.typePrefix + ) if (!isFile) { return {} @@ -139,6 +143,14 @@ const getGatsbyImageCdnFields = async ({ return {} } +export const generateTypeName = (type: string, typePrefix = ``) => { + const prefixed = typePrefix + ? `${capitalize(typePrefix)}${capitalize(type)}` + : type + + return prefixed.replace(/-|__|:|\.|\s/g, `_`) +} + export const nodeFromData = async ( datum, createNodeId, @@ -154,7 +166,9 @@ export const nodeFromData = async ( typeof attributeId !== `undefined` ? { _attributes_id: attributeId } : {} const langcode = attributes.langcode || `und` - const type = datum.type.replace(/-|__|:|\.|\s/g, `_`) + const { typePrefix = `` } = pluginOptions + + const type = generateTypeName(datum.type, typePrefix) const gatsbyImageCdnFields = await getGatsbyImageCdnFields({ node: datum, @@ -164,13 +178,14 @@ export const nodeFromData = async ( reporter, }) - const versionedId = createNodeIdWithVersion( - datum.id, - datum.type, + const versionedId = createNodeIdWithVersion({ + id: datum.id, + type: datum.type, langcode, - attributes.drupal_internal__revision_id, - entityReferenceRevisions - ) + revisionId: attributes.drupal_internal__revision_id, + entityReferenceRevisions, + typePrefix, + }) const gatsbyId = createNodeId(versionedId) @@ -191,18 +206,29 @@ export const nodeFromData = async ( } } -const isEntityReferenceRevision = (type, entityReferenceRevisions = []) => +const isEntityReferenceRevision = ( + type, + entityReferenceRevisions: Array = [] +) => entityReferenceRevisions.findIndex( revisionType => type.indexOf(revisionType) === 0 ) !== -1 -export const createNodeIdWithVersion = ( - id: string, - type: string, - langcode: string, - revisionId: string, - entityReferenceRevisions = [] -) => { +export const createNodeIdWithVersion = ({ + id, + type, + langcode, + revisionId, + entityReferenceRevisions = [], + typePrefix, +}: { + id: string + type: string + langcode: string + revisionId: string + entityReferenceRevisions: Array + typePrefix: string +}) => { const options = getOptions() // Fallback to default language for entities that don't translate. @@ -245,12 +271,23 @@ export const createNodeIdWithVersion = ( ? `${langcodeNormalized}.${id}.${revisionId || 0}` : `${langcodeNormalized}.${id}` - return idVersion + return typePrefix ? `${typePrefix}.${idVersion}` : idVersion } -export const isFileNode = node => { - const type = node?.internal?.type - return type === `files` || type === `file__file` +const fileNodeTypes = new Map>() + +export const isFileNode = (node, typePrefix = ``) => { + if (!fileNodeTypes.has(typePrefix)) { + fileNodeTypes.set( + typePrefix, + new Set([ + generateTypeName(`files`, typePrefix), + generateTypeName(`file--file`, typePrefix), + ]) + ) + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + return fileNodeTypes.get(typePrefix)!.has(node?.internal?.type) } const getFileUrl = (node, baseUrl) => { @@ -269,41 +306,42 @@ const getFileUrl = (node, baseUrl) => { export const downloadFile = async ( { node, cache, createNode, createNodeId, getCache }, - { basicAuth, baseUrl } + { basicAuth, baseUrl, typePrefix }: Record ) => { // handle file downloads - if (isFileNode(node)) { - let fileType + if (!isFileNode(node, typePrefix)) { + return + } + let fileType - if (typeof node.uri === `object`) { - // get file type from uri prefix ("S3:", "public:", etc.) - const uriPrefix = node.uri.value.match(/^\w*:/) - fileType = uriPrefix ? uriPrefix[0] : null - } + if (typeof node.uri === `object`) { + // get file type from uri prefix ("S3:", "public:", etc.) + const uriPrefix = node.uri.value.match(/^\w*:/) + fileType = uriPrefix ? uriPrefix[0] : null + } - const url = getFileUrl(node, baseUrl) - - // If we have basicAuth credentials, add them to the request. - const basicAuthFileSystems = [`public:`, `private:`, `temporary:`] - const auth = - typeof basicAuth === `object` && basicAuthFileSystems.includes(fileType) - ? { - htaccess_user: basicAuth.username, - htaccess_pass: basicAuth.password, - } - : {} - const fileNode = await createRemoteFileNode({ - url: url.href, - name: path.parse(decodeURIComponent(url.pathname)).name, - cache, - createNode, - createNodeId, - getCache, - parentNodeId: node.id, - auth, - }) - if (fileNode) { - node.localFile___NODE = fileNode.id - } + const url = getFileUrl(node, baseUrl) + + // If we have basicAuth credentials, add them to the request. + const basicAuthFileSystems = [`public:`, `private:`, `temporary:`] + const auth = + typeof basicAuth === `object` && basicAuthFileSystems.includes(fileType) + ? { + htaccess_user: basicAuth.username, + htaccess_pass: basicAuth.password, + } + : {} + const fileNode = await createRemoteFileNode({ + url: url.href, + name: path.parse(decodeURIComponent(url.pathname)).name, + cache, + createNode, + createNodeId, + getCache, + parentNodeId: node.id, + auth, + }) + if (fileNode) { + node.localFile___NODE = fileNode.id } } diff --git a/packages/gatsby-source-drupal/src/utils.ts b/packages/gatsby-source-drupal/src/utils.ts index 047b7877ad0a5..0084d47f11b0f 100644 --- a/packages/gatsby-source-drupal/src/utils.ts +++ b/packages/gatsby-source-drupal/src/utils.ts @@ -1,13 +1,13 @@ -const _ = require(`lodash`) +import _ from "lodash" -const { +import { nodeFromData, downloadFile, isFileNode, createNodeIdWithVersion, -} = require(`./normalize`) +} from "./normalize" -const { getOptions } = require(`./plugin-options`) +import { getOptions } from "./plugin-options" import { getGatsbyVersion } from "gatsby-core-utils" import { lt, prerelease } from "semver" @@ -20,7 +20,7 @@ function makeRefNodesKey(id) { return `refnodes-${id}` } -async function handleReferences( +export async function handleReferences( node, { getNode, @@ -28,14 +28,15 @@ async function handleReferences( createNodeId, entityReferenceRevisions = [], cache, + pluginOptions, } ) { const relationships = node.relationships const rootNodeLanguage = getOptions().languageConfig ? node.langcode : `und` - const backReferencedNodes = [] + const backReferencedNodes: Array = [] if (node.drupal_relationships) { - const referencedNodes = [] + const referencedNodes: Array = [] _.each(node.drupal_relationships, (v, k) => { if (!v.data) return @@ -44,13 +45,14 @@ async function handleReferences( relationships[nodeFieldName] = _.compact( v.data.map(data => { const referencedNodeId = createNodeId( - createNodeIdWithVersion( - data.id, - data.type, - rootNodeLanguage, - data.meta?.target_version, - entityReferenceRevisions - ) + createNodeIdWithVersion({ + id: data.id, + type: data.type, + langcode: rootNodeLanguage, + revisionId: data.meta?.target_version, + entityReferenceRevisions, + typePrefix: pluginOptions.typePrefix, + }) ) if (!getNode(referencedNodeId)) { return null @@ -72,13 +74,14 @@ async function handleReferences( } } else { const referencedNodeId = createNodeId( - createNodeIdWithVersion( - v.data.id, - v.data.type, - rootNodeLanguage, - v.data.meta?.target_revision_id, - entityReferenceRevisions - ) + createNodeIdWithVersion({ + id: v.data.id, + type: v.data.type, + langcode: rootNodeLanguage, + revisionId: v.data.meta?.target_revision_id, + entityReferenceRevisions, + typePrefix: pluginOptions.typePrefix, + }) ) if (getNode(referencedNodeId)) { relationships[nodeFieldName] = referencedNodeId @@ -133,9 +136,7 @@ async function handleReferences( return backReferencedNodes } -exports.handleReferences = handleReferences - -const handleDeletedNode = async ({ +export const handleDeletedNode = async ({ actions, node, getNode, @@ -143,16 +144,20 @@ const handleDeletedNode = async ({ createContentDigest, cache, entityReferenceRevisions, + pluginOptions, }) => { let deletedNode = getNode( createNodeId( - createNodeIdWithVersion( - node.id, - node.type, - getOptions().languageConfig ? node.attributes?.langcode : `und`, - node.attributes?.drupal_internal__revision_id, - entityReferenceRevisions - ) + createNodeIdWithVersion({ + id: node.id, + type: node.type, + langcode: getOptions().languageConfig + ? node.attributes?.langcode + : `und`, + revisionId: node.attributes?.drupal_internal__revision_id, + entityReferenceRevisions, + typePrefix: pluginOptions.typePrefix, + }) ) ) @@ -183,7 +188,9 @@ const handleDeletedNode = async ({ if (referencedNodes?.includes(deletedNode.id)) { // Loop over relationships and cleanup references. - Object.entries(node.relationships).forEach(([key, value]) => { + Object.entries( + node.relationships as Record> + ).forEach(([key, value]) => { // If a string ref matches, delete it. if (_.isString(value) && value === deletedNode.id) { delete node.relationships[key] @@ -191,7 +198,7 @@ const handleDeletedNode = async ({ // If it's an array, filter, then check if the array is empty and then delete // if so - if (_.isArray(value)) { + if (Array.isArray(value)) { value = value.filter(v => v !== deletedNode.id) if (value.length === 0) { @@ -227,7 +234,7 @@ const handleDeletedNode = async ({ return deletedNode } -async function createNodeIfItDoesNotExist({ +export async function createNodeIfItDoesNotExist({ nodeToUpdate, actions, createNodeId, @@ -249,13 +256,14 @@ ${JSON.stringify(nodeToUpdate, null, 4)} const { createNode } = actions const newNodeId = createNodeId( - createNodeIdWithVersion( - nodeToUpdate.id, - nodeToUpdate.type, - getOptions().languageConfig ? nodeToUpdate.langcode : `und`, - nodeToUpdate.meta?.target_version, - getOptions().entityReferenceRevisions - ) + createNodeIdWithVersion({ + id: nodeToUpdate.id, + type: nodeToUpdate.type, + langcode: getOptions().languageConfig ? nodeToUpdate.langcode : `und`, + revisionId: nodeToUpdate.meta?.target_version, + entityReferenceRevisions: getOptions().entityReferenceRevisions, + typePrefix: pluginOptions.typePrefix, + }) ) const oldNode = getNode(newNodeId) @@ -275,7 +283,7 @@ ${JSON.stringify(nodeToUpdate, null, 4)} } } -const handleWebhookUpdate = async ( +export const handleWebhookUpdate = async ( { nodeToUpdate, actions, @@ -285,9 +293,8 @@ const handleWebhookUpdate = async ( getCache, getNode, reporter, - store, }, - pluginOptions = {} + pluginOptions: Record = {} ) => { if (!nodeToUpdate) { reporter.warn( @@ -333,6 +340,7 @@ ${JSON.stringify(nodeToUpdate, null, 4)} createNodeId, cache, entityReferenceRevisions: pluginOptions.entityReferenceRevisions, + pluginOptions, }) nodesToUpdate.push(...backReferencedNodes) @@ -383,12 +391,11 @@ ${JSON.stringify(nodeToUpdate, null, 4)} } // Download file. - const { skipFileDownloads } = pluginOptions - if (isFileNode(newNode) && !skipFileDownloads) { + const { skipFileDownloads, typePrefix } = pluginOptions + if (isFileNode(newNode, typePrefix) && !skipFileDownloads) { await downloadFile( { node: newNode, - store, cache, createNode, createNodeId, @@ -467,14 +474,10 @@ export function drupalCreateNodeManifest({ } } -exports.handleWebhookUpdate = handleWebhookUpdate -exports.handleDeletedNode = handleDeletedNode -exports.createNodeIfItDoesNotExist = createNodeIfItDoesNotExist - /** * This FN returns a Map with additional file node information that Drupal doesn't return on actual file nodes (namely the width/height of images) */ -exports.getExtendedFileNodeData = allData => { +export const getExtendedFileNodeData = allData => { const fileNodesExtendedData = new Map() for (const contentType of allData) { @@ -490,7 +493,7 @@ exports.getExtendedFileNodeData = allData => { const { relationships } = node if (relationships) { - for (const relationship of Object.values(relationships)) { + for (const relationship of Object.values(relationships)) { const relationshipNodes = Array.isArray(relationship.data) ? relationship.data : [relationship.data]