feat: populate Blobs context in build plugins (#5571)

* feat: populate Blobs context in build plugins

* chore: fix test

* chore: update test

* chore: fix test

* chore: fix formatting

* chore: fix test

* chore: update test

* chore: update tests

Author: eduardoboucas

packages/build/src/core/build.ts

@@ -14,6 +14,7 @@ import { reportStatuses } from '../status/report.js'
 import { getDevSteps, getSteps } from '../steps/get.js'
 import { runSteps } from '../steps/run_steps.js'
 import { initTimers, measureDuration } from '../time/main.js'
+import { getBlobsEnvironmentContext } from '../utils/blobs.js'
 
 import { getConfigOpts, loadConfig } from './config.js'
 import { getConstants } from './constants.js'
@@ -197,6 +198,7 @@ const tExecBuild = async function ({
     dry,
     mode,
     api,
+    token,
     errorMonitor,
     deployId,
     errorParams,
@@ -258,6 +260,7 @@ export const runAndReportBuild = async function ({
   dry,
   mode,
   api,
+  token,
   errorMonitor,
   deployId,
   errorParams,
@@ -310,6 +313,7 @@ export const runAndReportBuild = async function ({
       dry,
       mode,
       api,
+      token,
       errorMonitor,
       deployId,
       errorParams,
@@ -414,6 +418,7 @@ const initAndRunBuild = async function ({
   dry,
   mode,
   api,
+  token,
   errorMonitor,
   deployId,
   errorParams,
@@ -459,6 +464,10 @@ const initAndRunBuild = async function ({
     systemLog,
   })
 
+  const pluginsEnv = featureFlags.build_inject_blobs_context
+    ? { ...childEnv, ...getBlobsEnvironmentContext({ api, deployId: deployId, siteId: siteInfo?.id, token }) }
+    : childEnv
+
   if (pluginsOptionsA?.length) {
     const buildPlugins = {}
     for (const plugin of pluginsOptionsA) {
@@ -475,7 +484,7 @@ const initAndRunBuild = async function ({
   const { childProcesses, timers: timersB } = await startPlugins({
     pluginsOptions: pluginsOptionsA,
     buildDir,
-    childEnv,
+    childEnv: pluginsEnv,
     logs,
     debug,
     timers: timersA,

packages/build/src/core/feature_flags.ts

@@ -15,6 +15,7 @@ const getFeatureFlag = function (name: string): FeatureFlags {
 
 // Default values for feature flags
 export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
+  build_inject_blobs_context: false,
   buildbot_zisi_trace_nft: false,
   buildbot_zisi_esbuild_parser: false,
   buildbot_zisi_system_log: false,

packages/build/src/core/normalize_flags.ts

@@ -6,6 +6,7 @@ import { removeFalsy } from '../utils/remove_falsy.js'
 import { DEFAULT_FEATURE_FLAGS } from './feature_flags.js'
 import type { BuildFlags, Mode, TestOptions } from './types.js'
 
+export const DEFAULT_API_HOST = 'api.netlify.com'
 const REQUIRE_MODE: Mode = 'require'
 const DEFAULT_EDGE_FUNCTIONS_DIST = '.netlify/edge-functions-dist/'
 const DEFAULT_FUNCTIONS_DIST = '.netlify/functions/'
@@ -91,7 +92,7 @@ const getDefaultFlags = function ({ env: envOpt = {} }, combinedEnv) {
     bugsnagKey: combinedEnv.BUGSNAG_KEY,
     sendStatus: false,
     saveConfig: false,
-    apiHost: 'api.netlify.com',
+    apiHost: DEFAULT_API_HOST,
     testOpts: {},
     featureFlags: DEFAULT_FEATURE_FLAGS,
     statsd: { port: DEFAULT_STATSD_PORT },

packages/build/src/plugins_core/blobs_upload/index.ts

@@ -4,6 +4,7 @@ import { getDeployStore } from '@netlify/blobs'
 import pMap from 'p-map'
 import semver from 'semver'
 
+import { DEFAULT_API_HOST } from '../../core/normalize_flags.js'
 import { log, logError } from '../../log/logger.js'
 import { getFileWithMetadata, getKeysToUpload, scanForBlobs } from '../../utils/blobs.js'
 import { type CoreStep, type CoreStepCondition, type CoreStepFunction } from '../types.js'
@@ -22,7 +23,7 @@ const coreStep: CoreStepFunction = async function ({
     return {}
   }
   // for cli deploys with `netlify deploy --build` the `NETLIFY_API_HOST` is undefined
-  const apiHost = NETLIFY_API_HOST || 'api.netlify.com'
+  const apiHost = NETLIFY_API_HOST || DEFAULT_API_HOST
 
   const storeOpts: Parameters<typeof getDeployStore>[0] = {
     siteID: SITE_ID,

packages/build/src/utils/blobs.ts

@@ -3,6 +3,8 @@ import path from 'node:path'
 
 import { fdir } from 'fdir'
 
+import { DEFAULT_API_HOST } from '../core/normalize_flags.js'
+
 const LEGACY_BLOBS_PATH = '.netlify/blobs/deploy'
 const DEPLOY_CONFIG_BLOBS_PATH = '.netlify/deploy/v1/blobs/deploy'
 
@@ -12,6 +14,40 @@ export const getBlobsDirs = (buildDir: string, packagePath?: string) => [
   path.resolve(buildDir, packagePath || '', LEGACY_BLOBS_PATH),
 ]
 
+interface EnvironmentContext {
+  api?: {
+    host: string
+    scheme: string
+  }
+  deployId?: string
+  siteId?: string
+  token?: string
+}
+
+// TODO: Move this work to a method exported by `@netlify/blobs`.
+export const getBlobsEnvironmentContext = ({
+  api = { host: DEFAULT_API_HOST, scheme: 'https' },
+  deployId,
+  siteId,
+  token,
+}: EnvironmentContext) => {
+  if (!deployId || !siteId || !token) {
+    return {}
+  }
+
+  const payload = {
+    apiURL: `${api.scheme}://${api.host}`,
+    deployID: deployId,
+    siteID: siteId,
+    token,
+  }
+  const encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64')
+
+  return {
+    NETLIFY_BLOBS_CONTEXT: encodedPayload,
+  }
+}
+
 /**
  * Detect if there are any blobs to upload, and if so, what directory they're
  * in and whether that directory is the legacy `.netlify/blobs` path or the

packages/build/tests/install/snapshots/tests.js.md

@@ -598,73 +598,6 @@ Generated by [AVA](https://avajs.dev).
     (Netlify Build completed in 1ms)␊
     Build step duration: Netlify Build completed in 1ms`
 
-## Functions: install dependencies handles errors
-
-> Snapshot 1
-
-    `␊
-    Netlify Build                                                 ␊
-    ────────────────────────────────────────────────────────────────␊
-    ␊
-    > Version␊
-      @netlify/build 1.0.0␊
-    ␊
-    > Flags␊
-      debug: true␊
-      repositoryRoot: packages/build/tests/install/fixtures/functions_error␊
-      testOpts:␊
-        pluginsListUrl: test␊
-        silentLingeringProcesses: true␊
-    ␊
-    > Current directory␊
-      packages/build/tests/install/fixtures/functions_error␊
-    ␊
-    > Config file␊
-      packages/build/tests/install/fixtures/functions_error/netlify.toml␊
-    ␊
-    > Resolved config␊
-      build:␊
-        publish: packages/build/tests/install/fixtures/functions_error␊
-        publishOrigin: default␊
-      functionsDirectory: packages/build/tests/install/fixtures/functions_error/functions␊
-      plugins:␊
-        - inputs: {}␊
-          origin: config␊
-          package: '@netlify/plugin-functions-install-core'␊
-    ␊
-    > Context␊
-      production␊
-    ␊
-    @netlify/plugin-functions-install-core (onPreBuild event)     ␊
-    ────────────────────────────────────────────────────────────────␊
-    ␊
-    Installing functions dependencies␊
-    ␊
-    Dependencies installation error                               ␊
-    ────────────────────────────────────────────────────────────────␊
-    ␊
-      Error message␊
-      Error while installing dependencies in packages/build/tests/install/fixtures/functions_error/functions␊
-      npm ERR! code ENOVERSIONS␊
-      npm ERR! No versions available for math-avg-does-not-exist␊
-    ␊
-      Plugin details␊
-      Package:        @netlify/plugin-functions-install-core␊
-      Version:        1.0.0␊
-      Repository:     git+https://github.com/netlify/build.git␊
-      npm link:       https://www.npmjs.com/package/@netlify/build␊
-      Report issues:  https://github.com/netlify/build/issues␊
-    ␊
-      Resolved config␊
-      build:␊
-        publish: packages/build/tests/install/fixtures/functions_error␊
-        publishOrigin: default␊
-      functionsDirectory: packages/build/tests/install/fixtures/functions_error/functions␊
-      plugins:␊
-        - inputs: {}␊
-          origin: config␊
-          package: '@netlify/plugin-functions-install-core'`
-
 ## Install local plugin dependencies: with npm
 
 > Snapshot 1
@@ -846,68 +779,6 @@ Generated by [AVA](https://avajs.dev).
     (Netlify Build completed in 1ms)␊
     Build step duration: Netlify Build completed in 1ms`
 
-## Install local plugin dependencies: propagate errors
-
-> Snapshot 1
-
-    `␊
-    Netlify Build                                                 ␊
-    ────────────────────────────────────────────────────────────────␊
-    ␊
-    > Version␊
-      @netlify/build 1.0.0␊
-    ␊
-    > Flags␊
-      debug: true␊
-      repositoryRoot: packages/build/tests/install/fixtures/error␊
-      testOpts:␊
-        pluginsListUrl: test␊
-        silentLingeringProcesses: true␊
-    ␊
-    > Current directory␊
-      packages/build/tests/install/fixtures/error␊
-    ␊
-    > Config file␊
-      packages/build/tests/install/fixtures/error/netlify.toml␊
-    ␊
-    > Resolved config␊
-      build:␊
-        publish: packages/build/tests/install/fixtures/error␊
-        publishOrigin: default␊
-      plugins:␊
-        - inputs: {}␊
-          origin: config␊
-          package: '@netlify/plugin-local-install-core'␊
-        - inputs: {}␊
-          origin: config␊
-          package: ./plugin/main.js␊
-    ␊
-    > Context␊
-      production␊
-    ␊
-    > Installing local plugins dependencies␊
-       - ./plugin/main.js␊
-    ␊
-    Dependencies installation error                               ␊
-    ────────────────────────────────────────────────────────────────␊
-    ␊
-      Error message␊
-      Error while installing dependencies in packages/build/tests/install/fixtures/error/plugin␊
-      npm ERR! code ENOVERSIONS␊
-      npm ERR! No versions available for this-dependency-does-not-exist␊
-    ␊
-      Resolved config␊
-      build:␊
-        publish: packages/build/tests/install/fixtures/error␊
-        publishOrigin: default␊
-      plugins:␊
-        - inputs: {}␊
-          origin: config␊
-          package: '@netlify/plugin-local-install-core'␊
-        - inputs: {}␊
-          origin: config␊
-          package: ./plugin/main.js`
-
 ## Install local plugin dependencies: already installed
 
 > Snapshot 1

packages/build/tests/install/snapshots/tests.js.snap

@@ -1,3 +1,4 @@
+import { join } from 'path'
 import { fileURLToPath } from 'url'
 
 import { Fixture, normalizeOutput, removeDir } from '@netlify/testing'
@@ -11,18 +12,23 @@ const FIXTURES_DIR = fileURLToPath(new URL('fixtures', import.meta.url))
 //  - specific directories are removed before/after test
 // TODO: once we have a test runner that supports before and after this would be way nicer to read to remove dirs there
 
-const runInstallFixture = async (t, fixtureName, dirs = [], flags = {}, binary = false) => {
+const runInstallFixture = async (t, fixtureName, dirs = [], flags = {}, binary = false, useSnapshot = true) => {
   await removeDir(dirs)
   try {
     const fixture = new Fixture(`./fixtures/${fixtureName}`).withFlags(flags)
     const result = binary ? await fixture.runBuildBinary().then(({ output }) => output) : await fixture.runWithBuild()
 
-    t.snapshot(normalizeOutput(result))
+    if (useSnapshot) {
+      t.snapshot(normalizeOutput(result))
+    }
+
     await Promise.all(
       dirs.map(async (dir) => {
         t.true(await pathExists(dir))
       }),
     )
+
+    return { fixture, result }
   } finally {
     await removeDir(dirs)
   }
@@ -95,7 +101,10 @@ test('Functions: does not print warnings when dependency was local', async (t) =
 })
 
 test('Functions: install dependencies handles errors', async (t) => {
-  await runInstallFixture(t, 'functions_error')
+  const { fixture, result } = await runInstallFixture(t, 'functions_error', [], {}, false, false)
+  const functionsPath = join(fixture.repositoryRoot, 'functions')
+
+  t.true(result.includes(`Error while installing dependencies in ${functionsPath}`))
 })
 
 test('Install local plugin dependencies: with npm', async (t) => {
@@ -114,8 +123,12 @@ test('Install local plugin dependencies: with yarn in CI', async (t) => {
 })
 
 test('Install local plugin dependencies: propagate errors', async (t) => {
-  const output = await new Fixture('./fixtures/error').runWithBuild()
-  t.snapshot(normalizeOutput(output))
+  const fixture = new Fixture('./fixtures/error')
+  const { success, output } = await fixture.runWithBuildAndIntrospect()
+  const pluginPath = join(fixture.repositoryRoot, 'plugin')
+
+  t.false(success)
+  t.true(output.includes(`Error while installing dependencies in ${pluginPath}`))
 })
 
 test('Install local plugin dependencies: already installed', async (t) => {

packages/build/tests/install/tests.js

@@ -0,0 +1,2 @@
+name: test
+inputs: []

packages/build/tests/plugins/fixtures/blobs_read/manifest.yml

@@ -0,0 +1,2 @@
+[[plugins]]
+package = "./plugin"

packages/build/tests/plugins/fixtures/blobs_read/netlify.toml

@@ -0,0 +1,18 @@
+import { version as nodeVersion } from "process"
+
+import { getDeployStore } from '@netlify/blobs'
+import semver from 'semver'
+
+export const onPreBuild = async function ({netlifyConfig}) {
+  const storeOptions = {}
+
+  if (semver.lt(nodeVersion, '18.0.0')) {
+    const nodeFetch = await import('node-fetch')
+    storeOptions.fetch = nodeFetch.default
+  }
+
+  const store = getDeployStore(storeOptions)
+  const value = await store.get("my-key")
+
+  netlifyConfig.build.command = `echo "${value}"`
+}

packages/build/tests/plugins/fixtures/blobs_read/plugin.js

@@ -2,8 +2,9 @@ import * as fs from 'fs/promises'
 import { platform } from 'process'
 import { fileURLToPath } from 'url'
 
-import { Fixture, normalizeOutput, removeDir } from '@netlify/testing'
+import { Fixture, normalizeOutput, removeDir, startServer } from '@netlify/testing'
 import test from 'ava'
+import getPort from 'get-port'
 import tmp, { tmpName } from 'tmp-promise'
 
 import { DEFAULT_FEATURE_FLAGS } from '../../lib/core/feature_flags.js'
@@ -366,3 +367,38 @@ test('Plugin errors that occur during the loading phase are piped to system logs
 
   t.snapshot(normalizeOutput(output))
 })
+
+test('Plugins have a pre-populated Blobs context', async (t) => {
+  const serverPort = await getPort()
+  const deployId = 'deploy123'
+  const siteId = 'site321'
+  const token = 'some-token'
+  const { scheme, host, stopServer } = await startServer(
+    [
+      {
+        response: { url: `http://localhost:${serverPort}/some-signed-url` },
+        path: `/api/v1/blobs/${siteId}/deploy:${deployId}/my-key`,
+      },
+      {
+        response: 'Hello there',
+        path: `/some-signed-url`,
+      },
+    ],
+    serverPort,
+  )
+
+  const { netlifyConfig } = await new Fixture('./fixtures/blobs_read')
+    .withFlags({
+      apiHost: host,
+      deployId,
+      featureFlags: { build_inject_blobs_context: true },
+      testOpts: { scheme },
+      siteId,
+      token,
+    })
+    .runWithBuildAndIntrospect()
+
+  await stopServer()
+
+  t.is(netlifyConfig.build.command, `echo ""Hello there""`)
+})

packages/build/tests/plugins/tests.js

@@ -26,12 +26,12 @@ const setTimeoutPromise = promisify(setTimeout)
 // response: json payload response (defaults to {})
 // status: http status code (defaults to 200)
 // wait: number used to induce a certain time delay in milliseconds in the response (defaults to undefined)
-export const startServer = async (handler: ServerHandler) => {
+export const startServer = async (handler: ServerHandler, port = 0) => {
   const handlers = Array.isArray(handler) ? handler : [handler]
   const requests: Request[] = []
 
   const server = createServer((req, res) => requestHandler(req, res, requests, handlers))
-  await promisify(server.listen.bind(server))(0)
+  await promisify(server.listen.bind(server))(port)
 
   const host = getHost(server)