feat: support deploy config API with Blobs (#5565)

* feat: support deploy config API with Blobs

* feat: set `experimentalRegion`

packages/build/src/plugins_core/blobs_upload/index.ts

@@ -5,7 +5,7 @@ import pMap from 'p-map'
 import semver from 'semver'
 
 import { log, logError } from '../../log/logger.js'
-import { anyBlobsToUpload, getBlobsDir } from '../../utils/blobs.js'
+import { scanForBlobs } from '../../utils/blobs.js'
 import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js'
 
 import { getKeysToUpload, getFileWithMetadata } from './utils.js'
@@ -26,22 +26,41 @@ const coreStep: CoreStepFunction = async function ({
   // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined
   const apiHost = NETLIFY_API_HOST || 'api.netlify.com'
 
-  const storeOpts: { siteID: string; deployID: string; token: string; apiURL: string; fetch?: any } = {
+  const storeOpts: Parameters<typeof getDeployStore>[0] = {
     siteID: SITE_ID,
     deployID: deployId,
     token: NETLIFY_API_TOKEN,
     apiURL: `https://${apiHost}`,
   }
+
+  // If we don't have native `fetch` in the global scope, add a polyfill.
   if (semver.lt(nodeVersion, '18.0.0')) {
     const nodeFetch = await import('node-fetch')
+
+    // @ts-expect-error The types between `node-fetch` and the native `fetch`
+    // are not a 100% match, even though the APIs are mostly compatible.
     storeOpts.fetch = nodeFetch.default
   }
 
-  const blobStore = getDeployStore(storeOpts)
-  const blobsDir = getBlobsDir(buildDir, packagePath)
-  const keys = await getKeysToUpload(blobsDir)
+  const blobs = await scanForBlobs(buildDir, packagePath)
 
   // We checked earlier, but let's be extra safe
+  if (blobs === null) {
+    if (!quiet) {
+      log(logs, 'No blobs to upload to deploy store.')
+    }
+    return {}
+  }
+
+  // If using the deploy config API, configure the store to use the region that
+  // was configured for the deploy.
+  if (!blobs.isLegacyDirectory) {
+    storeOpts.experimentalRegion = 'auto'
+  }
+
+  const blobStore = getDeployStore(storeOpts)
+  const keys = await getKeysToUpload(blobs.directory)
+
   if (keys.length === 0) {
     if (!quiet) {
       log(logs, 'No blobs to upload to deploy store.')
@@ -57,7 +76,7 @@ const coreStep: CoreStepFunction = async function ({
     if (debug && !quiet) {
       log(logs, `- Uploading blob ${key}`, { indent: true })
     }
-    const { data, metadata } = await getFileWithMetadata(blobsDir, key)
+    const { data, metadata } = await getFileWithMetadata(blobs.directory, key)
     await blobStore.set(key, data, { metadata })
   }
 
@@ -81,7 +100,7 @@ const deployAndBlobsPresent: CoreStepCondition = async ({
   buildDir,
   packagePath,
   constants: { NETLIFY_API_TOKEN },
-}) => Boolean(NETLIFY_API_TOKEN && deployId && (await anyBlobsToUpload(buildDir, packagePath)))
+}) => Boolean(NETLIFY_API_TOKEN && deployId && (await scanForBlobs(buildDir, packagePath)))
 
 export const uploadBlobs: CoreStep = {
   event: 'onPostBuild',

packages/build/src/plugins_core/pre_cleanup/index.ts

@@ -1,20 +1,21 @@
 import { rm } from 'node:fs/promises'
 
-import { anyBlobsToUpload, getBlobsDir } from '../../utils/blobs.js'
+import { scanForBlobs, getBlobsDirs } from '../../utils/blobs.js'
 import { CoreStep, CoreStepCondition, CoreStepFunction } from '../types.js'
 
 const coreStep: CoreStepFunction = async ({ buildDir, packagePath }) => {
-  const blobsDir = getBlobsDir(buildDir, packagePath)
+  const blobsDirs = getBlobsDirs(buildDir, packagePath)
   try {
-    await rm(blobsDir, { recursive: true, force: true })
+    await Promise.all(blobsDirs.map((dir) => rm(dir, { recursive: true, force: true })))
   } catch {
     // Ignore errors if it fails, we can continue anyway.
   }
 
   return {}
 }
 
-const blobsPresent: CoreStepCondition = ({ buildDir, packagePath }) => anyBlobsToUpload(buildDir, packagePath)
+const blobsPresent: CoreStepCondition = async ({ buildDir, packagePath }) =>
+  Boolean(await scanForBlobs(buildDir, packagePath))
 
 export const preCleanup: CoreStep = {
   event: 'onPreBuild',

packages/build/src/utils/blobs.ts

@@ -2,19 +2,44 @@ import { resolve } from 'node:path'
 
 import { fdir } from 'fdir'
 
-const BLOBS_PATH = '.netlify/blobs/deploy'
+const LEGACY_BLOBS_PATH = '.netlify/blobs/deploy'
+const DEPLOY_CONFIG_BLOBS_PATH = '.netlify/deploy/v1/blobs/deploy'
 
-/** Retrieve the absolute path of the deploy scoped internal blob directory */
-export const getBlobsDir = (buildDir: string, packagePath?: string) => resolve(buildDir, packagePath || '', BLOBS_PATH)
+/** Retrieve the absolute path of the deploy scoped internal blob directories */
+export const getBlobsDirs = (buildDir: string, packagePath?: string) => [
+  resolve(buildDir, packagePath || '', DEPLOY_CONFIG_BLOBS_PATH),
+  resolve(buildDir, packagePath || '', LEGACY_BLOBS_PATH),
+]
 
 /**
- * Detect if there are any blobs to upload
+ * Detect if there are any blobs to upload, and if so, what directory they're
+ * in and whether that directory is the legacy `.netlify/blobs` path or the
+ * newer deploy config API endpoint.
+ *
  * @param buildDir The build directory. (current working directory where the build is executed)
  * @param packagePath An optional package path for mono repositories
  * @returns
  */
-export const anyBlobsToUpload = async function (buildDir: string, packagePath?: string) {
-  const blobsDir = getBlobsDir(buildDir, packagePath)
-  const { files } = await new fdir().onlyCounts().crawl(blobsDir).withPromise()
-  return files > 0
+export const scanForBlobs = async function (buildDir: string, packagePath?: string) {
+  const blobsDir = resolve(buildDir, packagePath || '', DEPLOY_CONFIG_BLOBS_PATH)
+  const blobsDirScan = await new fdir().onlyCounts().crawl(blobsDir).withPromise()
+
+  if (blobsDirScan.files > 0) {
+    return {
+      directory: blobsDir,
+      isLegacyDirectory: false,
+    }
+  }
+
+  const legacyBlobsDir = resolve(buildDir, packagePath || '', LEGACY_BLOBS_PATH)
+  const legacyBlobsDirScan = await new fdir().onlyCounts().crawl(legacyBlobsDir).withPromise()
+
+  if (legacyBlobsDirScan.files > 0) {
+    return {
+      directory: legacyBlobsDir,
+      isLegacyDirectory: true,
+    }
+  }
+
+  return null
 }

packages/build/tests/blobs_upload/fixtures/src_with_blobs/build.mjs

@@ -1,11 +1,11 @@
 import { mkdir, writeFile } from 'node:fs/promises'
 
-await mkdir('.netlify/blobs/deploy/nested', { recursive: true })
+await mkdir('.netlify/deploy/v1/blobs/deploy/nested', { recursive: true })
 
 await Promise.all([
-  writeFile('.netlify/blobs/deploy/something.txt', 'some value'),
-  writeFile('.netlify/blobs/deploy/with-metadata.txt', 'another value'),
-  writeFile('.netlify/blobs/deploy/$with-metadata.txt.json', JSON.stringify({ "meta": "data", "number": 1234 })),
-  writeFile('.netlify/blobs/deploy/nested/file.txt', 'file value'),
-  writeFile('.netlify/blobs/deploy/nested/$file.txt.json', JSON.stringify({ "some": "metadata" })),
+  writeFile('.netlify/deploy/v1/blobs/deploy/something.txt', 'some value'),
+  writeFile('.netlify/deploy/v1/blobs/deploy/with-metadata.txt', 'another value'),
+  writeFile('.netlify/deploy/v1/blobs/deploy/$with-metadata.txt.json', JSON.stringify({ "meta": "data", "number": 1234 })),
+  writeFile('.netlify/deploy/v1/blobs/deploy/nested/file.txt', 'file value'),
+  writeFile('.netlify/deploy/v1/blobs/deploy/nested/$file.txt.json', JSON.stringify({ "some": "metadata" })),
 ])

packages/build/tests/blobs_upload/fixtures/src_with_blobs_legacy/build.mjs

@@ -0,0 +1,11 @@
+import { mkdir, writeFile } from 'node:fs/promises'
+
+await mkdir('.netlify/blobs/deploy/nested', { recursive: true })
+
+await Promise.all([
+  writeFile('.netlify/blobs/deploy/something.txt', 'some value'),
+  writeFile('.netlify/blobs/deploy/with-metadata.txt', 'another value'),
+  writeFile('.netlify/blobs/deploy/$with-metadata.txt.json', JSON.stringify({ "meta": "data", "number": 1234 })),
+  writeFile('.netlify/blobs/deploy/nested/file.txt', 'file value'),
+  writeFile('.netlify/blobs/deploy/nested/$file.txt.json', JSON.stringify({ "some": "metadata" })),
+])

packages/build/tests/blobs_upload/fixtures/src_with_blobs_legacy/netlify.toml

@@ -0,0 +1,4 @@
+[build]
+command = "node build.mjs"
+base = "/"
+publish = "/dist"

packages/build/tests/blobs_upload/tests.js

@@ -14,23 +14,24 @@ const TOKEN = 'test'
 
 test.beforeEach(async (t) => {
   const port = await getPort()
-  t.context.blobRequestCount = { set: 0, get: 0 }
+  t.context.blobRequests = {}
 
   const tmpDir = await tmp.dir()
   t.context.blobServer = new BlobsServer({
     port,
     token: TOKEN,
     directory: tmpDir.path,
-    onRequest: ({ type }) => {
-      t.context.blobRequestCount[type] = (t.context.blobRequestCount[type] || 0) + 1
+    onRequest: ({ type, url }) => {
+      t.context.blobRequests[type] = t.context.blobRequests[type] || []
+      t.context.blobRequests[type].push(url)
     },
   })
 
   await t.context.blobServer.start()
 
   process.env.NETLIFY_BLOBS_CONTEXT = Buffer.from(
     JSON.stringify({
-      edgeURL: `http://localhost:${port}`,
+      apiURL: `http://localhost:${port}`,
     }),
   ).toString('base64')
 })
@@ -50,27 +51,74 @@ test.serial("blobs upload, don't run when deploy id is provided and no files in
     .runBuildProgrammatic()
 
   t.true(success)
-  t.is(t.context.blobRequestCount.set, 0)
+  t.is(t.context.blobRequests.set, undefined)
 
   t.false(stdout.join('\n').includes('Uploading blobs to deploy store'))
 })
 
-test.serial("blobs upload, don't run when there are files but deploy id is not provided", async (t) => {
-  const fixture = await new Fixture('./fixtures/src_with_blobs').withCopyRoot({ git: false })
+test.serial(
+  "blobs upload, don't run when there are files but deploy id is not provided using legacy API",
+  async (t) => {
+    const fixture = await new Fixture('./fixtures/src_with_blobs_legacy').withCopyRoot({ git: false })
+
+    const {
+      success,
+      logs: { stdout },
+    } = await fixture.withFlags({ token: TOKEN, offline: true, cwd: fixture.repositoryRoot }).runBuildProgrammatic()
+
+    t.true(success)
+
+    const blobsDir = join(fixture.repositoryRoot, '.netlify', 'blobs', 'deploy')
+    await t.notThrowsAsync(access(blobsDir))
+
+    t.is(t.context.blobRequests.set, undefined)
+
+    t.false(stdout.join('\n').includes('Uploading blobs to deploy store'))
+  },
+)
+
+test.serial('blobs upload, uploads files to deploy store using legacy API', async (t) => {
+  const fixture = await new Fixture('./fixtures/src_with_blobs_legacy').withCopyRoot({ git: false })
 
   const {
     success,
     logs: { stdout },
-  } = await fixture.withFlags({ token: TOKEN, offline: true, cwd: fixture.repositoryRoot }).runBuildProgrammatic()
+  } = await fixture
+    .withFlags({ deployId: 'abc123', siteId: 'test', token: TOKEN, offline: true, cwd: fixture.repositoryRoot })
+    .runBuildProgrammatic()
 
   t.true(success)
+  t.is(t.context.blobRequests.set.length, 6)
 
-  const blobsDir = join(fixture.repositoryRoot, '.netlify', 'blobs', 'deploy')
-  await t.notThrowsAsync(access(blobsDir))
+  const regionRequests = t.context.blobRequests.set.filter((urlPath) => {
+    const url = new URL(urlPath, 'http://localhost')
 
-  t.is(t.context.blobRequestCount.set, 0)
+    return url.searchParams.has('region')
+  })
 
-  t.false(stdout.join('\n').includes('Uploading blobs to deploy store'))
+  t.is(regionRequests.length, 0)
+
+  const storeOpts = { deployID: 'abc123', siteID: 'test', token: TOKEN }
+  if (semver.lt(nodeVersion, '18.0.0')) {
+    const nodeFetch = await import('node-fetch')
+    storeOpts.fetch = nodeFetch.default
+  }
+
+  const store = getDeployStore(storeOpts)
+
+  const blob1 = await store.getWithMetadata('something.txt')
+  t.is(blob1.data, 'some value')
+  t.deepEqual(blob1.metadata, {})
+
+  const blob2 = await store.getWithMetadata('with-metadata.txt')
+  t.is(blob2.data, 'another value')
+  t.deepEqual(blob2.metadata, { meta: 'data', number: 1234 })
+
+  const blob3 = await store.getWithMetadata('nested/file.txt')
+  t.is(blob3.data, 'file value')
+  t.deepEqual(blob3.metadata, { some: 'metadata' })
+
+  t.true(stdout.join('\n').includes('Uploading blobs to deploy store'))
 })
 
 test.serial('blobs upload, uploads files to deploy store', async (t) => {
@@ -84,7 +132,17 @@ test.serial('blobs upload, uploads files to deploy store', async (t) => {
     .runBuildProgrammatic()
 
   t.true(success)
-  t.is(t.context.blobRequestCount.set, 3)
+
+  // 3 requests for getting pre-signed URLs + 3 requests for hitting them.
+  t.is(t.context.blobRequests.set.length, 6)
+
+  const regionAutoRequests = t.context.blobRequests.set.filter((urlPath) => {
+    const url = new URL(urlPath, 'http://localhost')
+
+    return url.searchParams.get('region') === 'auto'
+  })
+
+  t.is(regionAutoRequests.length, 3)
 
   const storeOpts = { deployID: 'abc123', siteID: 'test', token: TOKEN }
   if (semver.lt(nodeVersion, '18.0.0')) {
@@ -118,7 +176,7 @@ test.serial('blobs upload, cancels deploy if blob metadata is malformed', async
   const blobsDir = join(fixture.repositoryRoot, '.netlify', 'blobs', 'deploy')
   await t.notThrowsAsync(access(blobsDir))
 
-  t.is(t.context.blobRequestCount.set, 0)
+  t.is(t.context.blobRequests.set, undefined)
 
   t.false(success)
   t.is(severityCode, 4)
@@ -136,7 +194,7 @@ if (semver.gte(nodeVersion, '16.9.0')) {
       .runBuildProgrammatic()
 
     t.true(success)
-    t.is(t.context.blobRequestCount.set, 3)
+    t.is(t.context.blobRequests.set.length, 6)
 
     const storeOpts = { deployID: 'abc123', siteID: 'test', token: TOKEN }
     if (semver.lt(nodeVersion, '18.0.0')) {