feat: add local dev experience for scheduled functions (#3689)

* feat: add local dev experience for scheduled functions

* feat: use synchronous function's timeout

* chore: rename to scheduled.js

* feat: return 400 when trying to invoke scheduled function via http

* chore: use testMatrix pattern

* fix: config-based schedule without esbuild

* feat: add support for ISC-declared flags

* fix: node v12 doesn't understand optional chaining

* fix: allow esbuild to read mock files by writing them to disk

* fix: wrong import

* feat: use listFunction to detect ISC schedule

* feat: remove unused feature flag

* chore: update zisi

* fix: enable parseISC hook

* feat: give full command

* refactor: move clockwork simulation to calling side

* chore: remove one changed line

* refactor: extract clockwork useragent into constants

* feat: improve error message

co-authored-by: Eduardo Boucas <eduardo@netlify.com>

* feat: print friendly error screen

* chore: trim down diff to npm-shrinkwrap

* chore: remove mock-require (not used anymore)

* fix: optional chaining doesnt exist

* chore: add test for helpful tips and tricks

* fix: correct tests

* fix: add missing property to test

* Update src/lib/functions/runtimes/js/builders/zisi.js

Co-authored-by: Lukas Holzer <lukas.holzer@netlify.com>

* refactor: extract help response into its own testable function

* chore: add some unit tests

* fix: replaceAll not available on node v12

co-authored-by: Lukas Holzer <lukas.holzer@netlify.com>

* fix: remove unneeded level field

* refactor: remove unused test matrix

* fix: increase file change delay for macOS and windows

Co-authored-by: Netlify Team Account 1 <netlify-team-account-1@users.noreply.github.com>
Co-authored-by: Eduardo Boucas <eduardo@netlify.com>
Co-authored-by: Lukas Holzer <lukas.holzer@netlify.com>
Co-authored-by: Simon Knott <info@simonknott.de>

Author: 4 people

npm-shrinkwrap.json

@@ -89,6 +89,7 @@
     "@sindresorhus/slugify": "^1.1.0",
     "ansi-escapes": "^5.0.0",
     "ansi-styles": "^5.0.0",
+    "ansi2html": "^0.0.1",
     "ascii-table": "0.0.9",
     "backoff": "^2.5.0",
     "better-opn": "^3.0.0",
@@ -103,6 +104,7 @@
     "content-type": "^1.0.4",
     "cookie": "^0.4.0",
     "copy-template-dir": "^1.4.0",
+    "cron-parser": "^4.2.1",
     "debug": "^4.1.1",
     "decache": "^4.6.0",
     "del": "^6.0.0",
@@ -197,7 +199,6 @@
     "ini": "^2.0.0",
     "jsonwebtoken": "^8.5.1",
     "mock-fs": "^5.1.2",
-    "mock-require": "^3.0.3",
     "p-timeout": "^4.0.0",
     "proxyquire": "^2.1.3",
     "seedrandom": "^3.0.5",

package.json

@@ -3,10 +3,11 @@ const fs = require('fs')
 const path = require('path')
 const process = require('process')
 
+const CronParser = require('cron-parser')
 const inquirer = require('inquirer')
 const fetch = require('node-fetch')
 
-const { BACKGROUND, NETLIFYDEVWARN, chalk, error, exit, getFunctions } = require('../../utils')
+const { BACKGROUND, CLOCKWORK_USERAGENT, NETLIFYDEVWARN, chalk, error, exit, getFunctions } = require('../../utils')
 
 // https://www.netlify.com/docs/functions/#event-triggered-functions
 const events = [
@@ -130,6 +131,13 @@ const getFunctionToTrigger = function (options, argumentName) {
   return argumentName
 }
 
+const getNextRun = function (schedule) {
+  const cron = CronParser.parseExpression(schedule, {
+    tz: 'Etc/UTC',
+  })
+  return cron.next().toDate()
+}
+
 /**
  * The functions:invoke command
  * @param {string} nameArgument
@@ -150,11 +158,18 @@ const functionsInvoke = async (nameArgument, options, command) => {
 
   const functions = await getFunctions(functionsDir)
   const functionToTrigger = await getNameFromArgs(functions, options, nameArgument)
+  const functionObj = functions.find((func) => func.name === functionToTrigger)
 
   let headers = {}
   let body = {}
 
-  if (eventTriggeredFunctions.has(functionToTrigger)) {
+  if (functionObj.schedule) {
+    body.next_run = getNextRun(functionObj.schedule)
+    headers = {
+      'user-agent': CLOCKWORK_USERAGENT,
+      'X-NF-Event': 'schedule',
+    }
+  } else if (eventTriggeredFunctions.has(functionToTrigger)) {
     /** handle event triggered fns  */
     // https://www.netlify.com/docs/functions/#event-triggered-functions
     const [name, event] = functionToTrigger.split('-')

src/commands/functions/functions-invoke.js

@@ -32,6 +32,7 @@ class NetlifyFunction {
     // Determines whether this is a background function based on the function
     // name.
     this.isBackground = name.endsWith(BACKGROUND_SUFFIX)
+    this.schedule = null
 
     // List of the function's source files. This starts out as an empty set
     // and will get populated on every build.
@@ -44,6 +45,12 @@ class NetlifyFunction {
     return /^[A-Za-z0-9_-]+$/.test(this.name)
   }
 
+  async isScheduled() {
+    await this.buildQueue
+
+    return Boolean(this.schedule)
+  }
+
   // The `build` method transforms source files into invocable functions. Its
   // return value is an object with:
   //
@@ -61,12 +68,13 @@ class NetlifyFunction {
     this.buildQueue = buildFunction({ cache })
 
     try {
-      const { srcFiles, ...buildData } = await this.buildQueue
+      const { schedule, srcFiles, ...buildData } = await this.buildQueue
       const srcFilesSet = new Set(srcFiles)
       const srcFilesDiff = this.getSrcFilesDiff(srcFilesSet)
 
       this.buildData = buildData
       this.srcFiles = srcFilesSet
+      this.schedule = schedule
 
       return { srcFilesDiff }
     } catch (error) {

src/lib/functions/netlify-function.js

@@ -1,7 +1,7 @@
 const { mkdir, writeFile } = require('fs').promises
 const path = require('path')
 
-const { zipFunction } = require('@netlify/zip-it-and-ship-it')
+const { listFunction, zipFunction } = require('@netlify/zip-it-and-ship-it')
 const decache = require('decache')
 const readPkgUp = require('read-pkg-up')
 const sourceMapSupport = require('source-map-support')
@@ -35,7 +35,11 @@ const buildFunction = async ({ cache, config, directory, func, hasTypeModule, pr
   // root of the functions directory (e.g. `functions/my-func.js`). In
   // this case, we use `mainFile` as the function path of `zipFunction`.
   const entryPath = functionDirectory === directory ? func.mainFile : functionDirectory
-  const { inputs, path: functionPath } = await memoizedBuild({
+  const {
+    inputs,
+    path: functionPath,
+    schedule,
+  } = await memoizedBuild({
     cache,
     cacheKey: `zisi-${entryPath}`,
     command: () => zipFunction(entryPath, targetDirectory, zipOptions),
@@ -56,7 +60,22 @@ const buildFunction = async ({ cache, config, directory, func, hasTypeModule, pr
 
   clearFunctionsCache(targetDirectory)
 
-  return { buildPath, srcFiles }
+  return { buildPath, srcFiles, schedule }
+}
+
+/**
+ * @param {object} params
+ * @param {unknown} params.config
+ * @param {string} params.mainFile
+ * @param {string} params.projectRoot
+ */
+const parseForSchedule = async ({ config, mainFile, projectRoot }) => {
+  const listedFunction = await listFunction(mainFile, {
+    config: netlifyConfigToZisiConfig({ config, projectRoot }),
+    parseISC: true,
+  })
+
+  return listedFunction && listedFunction.schedule
 }
 
 // Clears the cache for any files inside the directory from which functions are
@@ -79,10 +98,11 @@ const getTargetDirectory = async ({ errorExit }) => {
   return targetDirectory
 }
 
+const netlifyConfigToZisiConfig = ({ config, projectRoot }) =>
+  addFunctionsConfigDefaults(normalizeFunctionsConfig({ functionsConfig: config.functions, projectRoot }))
+
 module.exports = async ({ config, directory, errorExit, func, projectRoot }) => {
-  const functionsConfig = addFunctionsConfigDefaults(
-    normalizeFunctionsConfig({ functionsConfig: config.functions, projectRoot }),
-  )
+  const functionsConfig = netlifyConfigToZisiConfig({ config, projectRoot })
 
   const packageJson = await readPkgUp(func.mainFile)
   const hasTypeModule = packageJson && packageJson.packageJson.type === 'module'
@@ -115,3 +135,5 @@ module.exports = async ({ config, directory, errorExit, func, projectRoot }) =>
     target: targetDirectory,
   }
 }
+
+module.exports.parseForSchedule = parseForSchedule

src/lib/functions/runtimes/js/builders/zisi.js

@@ -46,8 +46,9 @@ const getBuildFunction = async ({ config, directory, errorExit, func, projectRoo
   // main file otherwise.
   const functionDirectory = dirname(func.mainFile)
   const srcFiles = functionDirectory === directory ? [func.mainFile] : [functionDirectory]
+  const schedule = await detectZisiBuilder.parseForSchedule({ mainFile: func.mainFile, config, projectRoot })
 
-  return () => ({ srcFiles })
+  return () => ({ schedule, srcFiles })
 }
 
 const invokeFunction = async ({ context, event, func, timeout }) => {

src/lib/functions/runtimes/js/index.js

@@ -0,0 +1,98 @@
+const ansi2html = require('ansi2html')
+
+const { CLOCKWORK_USERAGENT } = require('../../utils')
+
+const { formatLambdaError } = require('./utils')
+
+const buildHelpResponse = ({ error, headers, path, result }) => {
+  const acceptsHtml = headers.accept && headers.accept.includes('text/html')
+
+  const paragraph = (text) => {
+    text = text.trim()
+
+    if (acceptsHtml) {
+      return ansi2html(`<p>${text}</p>`)
+    }
+
+    text = text
+      .replace(/<pre><code>/gm, '```\n')
+      .replace(/<\/code><\/pre>/gm, '\n```')
+      .replace(/<code>/gm, '`')
+      .replace(/<\/code>/gm, '`')
+
+    return `${text}\n\n`
+  }
+
+  const isSimulatedRequest = headers['user-agent'] === CLOCKWORK_USERAGENT
+
+  let message = ''
+
+  if (!isSimulatedRequest) {
+    message += paragraph(`
+You performed an HTTP request to <code>${path}</code>, which is a scheduled function.
+You can do this to test your functions locally, but it won't work in production.
+    `)
+  }
+
+  if (error) {
+    message += paragraph(`
+There was an error during execution of your scheduled function:
+
+<pre><code>${formatLambdaError(error)}</code></pre>`)
+  }
+
+  if (result) {
+    // lambda emulator adds level field, which isn't user-provided
+    const returnValue = { ...result }
+    delete returnValue.level
+
+    const { statusCode } = returnValue
+    if (statusCode >= 500) {
+      message += paragraph(`
+Your function returned a status code of <code>${statusCode}</code>.
+At the moment, Netlify does nothing about that. In the future, there might be a retry mechanism based on this.
+`)
+    }
+
+    const allowedKeys = new Set(['statusCode'])
+    const returnedKeys = Object.keys(returnValue)
+    const ignoredKeys = returnedKeys.filter((key) => !allowedKeys.has(key))
+
+    if (ignoredKeys.length !== 0) {
+      message += paragraph(
+        `Your function returned ${ignoredKeys
+          .map((key) => `<code>${key}</code>`)
+          .join(', ')}. Is this an accident? It won't be interpreted by Netlify.`,
+      )
+    }
+  }
+
+  const statusCode = error ? 500 : 200
+  return acceptsHtml
+    ? {
+        statusCode,
+        contentType: 'text/html',
+        message: `<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/water.css@2/out/water.css">\n
+                ${message}`,
+      }
+    : {
+        statusCode,
+        contentType: 'text/plain',
+        message,
+      }
+}
+
+const handleScheduledFunction = ({ error, request, response, result }) => {
+  const { contentType, message, statusCode } = buildHelpResponse({
+    error,
+    headers: request.headers,
+    path: request.path,
+    result,
+  })
+
+  response.status(statusCode)
+  response.set('Content-Type', contentType)
+  response.send(message)
+}
+
+module.exports = { handleScheduledFunction, buildHelpResponse }

src/lib/functions/scheduled.js

@@ -0,0 +1,58 @@
+const test = require('ava')
+
+const { buildHelpResponse } = require('./scheduled')
+
+const withAccept = (accept) =>
+  buildHelpResponse({
+    error: undefined,
+    headers: {
+      accept,
+    },
+    path: '/',
+    result: {
+      statusCode: 200,
+    },
+  })
+
+test('buildHelpResponse does content negotiation', (t) => {
+  const html = withAccept('text/html')
+  t.is(html.contentType, 'text/html')
+  t.true(html.message.includes('<link rel='))
+  t.true(html.message.includes('<p>'))
+
+  const plain = withAccept('text/plain')
+  t.is(plain.contentType, 'text/plain')
+  t.false(plain.message.includes('<link rel='))
+  t.false(plain.message.includes('<p>'))
+})
+
+test('buildHelpResponse prints errors', (t) => {
+  const response = buildHelpResponse({
+    error: new Error('test'),
+    headers: {},
+    path: '/',
+    result: {
+      statusCode: 200,
+    },
+  })
+
+  t.true(response.message.includes('There was an error'))
+})
+
+const withUserAgent = (userAgent) =>
+  buildHelpResponse({
+    error: new Error('test'),
+    headers: {
+      accept: 'text/plain',
+      'user-agent': userAgent,
+    },
+    path: '/',
+    result: {
+      statusCode: 200,
+    },
+  })
+
+test('buildHelpResponse conditionally prints notice about HTTP x scheduled functions', (t) => {
+  t.true(withUserAgent('').message.includes("it won't work in production"))
+  t.false(withUserAgent('Netlify Clockwork').message.includes("it won't work in production"))
+})

src/lib/functions/scheduled.test.js

@@ -6,6 +6,7 @@ const { NETLIFYDEVERR, NETLIFYDEVLOG, error: errorExit, getInternalFunctionsDir,
 const { handleBackgroundFunction, handleBackgroundFunctionResult } = require('./background')
 const { createFormSubmissionHandler } = require('./form-submissions-handler')
 const { FunctionsRegistry } = require('./registry')
+const { handleScheduledFunction } = require('./scheduled')
 const { handleSynchronousFunction } = require('./synchronous')
 const { shouldBase64Encode } = require('./utils')
 
@@ -105,6 +106,15 @@ const createHandler = function ({ functionsRegistry }) {
       const { error } = await func.invoke(event, clientContext)
 
       handleBackgroundFunctionResult(functionName, error)
+    } else if (await func.isScheduled()) {
+      const { error, result } = await func.invoke(event, clientContext)
+
+      handleScheduledFunction({
+        error,
+        result,
+        request,
+        response,
+      })
     } else {
       const { error, result } = await func.invoke(event, clientContext)
 

src/lib/functions/server.js

@@ -1,58 +1,37 @@
-const fs = require('fs')
-const { platform } = require('os')
+const { mkdirSync, mkdtempSync, writeFileSync } = require('fs')
+const { tmpdir } = require('os')
 const { join } = require('path')
 
-const zisi = require('@netlify/zip-it-and-ship-it')
 const test = require('ava')
 const express = require('express')
-const mockRequire = require('mock-require')
-const sinon = require('sinon')
 const request = require('supertest')
 
-const projectRoot = platform() === 'win32' ? 'C:\\my-functions' : `/my-functions`
-const functionsPath = `functions`
-
-// mock mkdir for the functions folder
-sinon.stub(fs.promises, 'mkdir').withArgs(join(projectRoot, functionsPath)).returns(Promise.resolve())
-
 const { FunctionsRegistry } = require('./registry')
 const { createHandler } = require('./server')
 
 /** @type { express.Express} */
 let app
 
-test.before(async (t) => {
-  const mainFile = join(projectRoot, functionsPath, 'hello.js')
-  t.context.zisiStub = sinon.stub(zisi, 'listFunctions').returns(
-    Promise.resolve([
-      {
-        name: 'hello',
-        mainFile,
-        runtime: 'js',
-        extension: '.js',
-      },
-    ]),
-  )
+test.before(async () => {
+  const projectRoot = mkdtempSync(join(tmpdir(), 'functions-server-project-root'))
+  const functionsDirectory = join(projectRoot, 'functions')
+  mkdirSync(functionsDirectory)
+
+  const mainFile = join(functionsDirectory, 'hello.js')
+  writeFileSync(mainFile, `exports.handler = (event) => ({ statusCode: 200, body: event.rawUrl })`)
 
-  mockRequire(mainFile, {
-    handler: (event) => ({ statusCode: 200, body: event.rawUrl }),
-  })
   const functionsRegistry = new FunctionsRegistry({
     projectRoot,
     config: {},
     timeouts: { syncFunctions: 1, backgroundFunctions: 1 },
     // eslint-disable-next-line no-magic-numbers
     settings: { port: 8888 },
   })
-  await functionsRegistry.scan([functionsPath])
+  await functionsRegistry.scan([functionsDirectory])
   app = express()
   app.all('*', createHandler({ functionsRegistry }))
 })
 
-test.after((t) => {
-  t.context.zisiStub.restore()
-})
-
 test('should get the url as the `rawUrl` inside the function', async (t) => {
   await request(app)
     .get('/hello')

src/lib/functions/server.test.js

@@ -0,0 +1,5 @@
+const CLOCKWORK_USERAGENT = 'Netlify Clockwork'
+
+module.exports = {
+  CLOCKWORK_USERAGENT,
+}

src/utils/functions/constants.js

@@ -5,10 +5,10 @@ const getUrlPath = (functionName) => `/.netlify/functions/${functionName}`
 
 const BACKGROUND = '-background'
 
-const addFunctionProps = ({ mainFile, name, runtime }) => {
+const addFunctionProps = ({ mainFile, name, runtime, schedule }) => {
   const urlPath = getUrlPath(name)
   const isBackground = name.endsWith(BACKGROUND)
-  return { mainFile, name, runtime, urlPath, isBackground }
+  return { mainFile, name, runtime, urlPath, isBackground, schedule }
 }
 
 const JS = 'js'
@@ -21,7 +21,9 @@ const getFunctions = async (functionsSrcDir) => {
   // performance optimization, load '@netlify/zip-it-and-ship-it' on demand
   // eslint-disable-next-line node/global-require
   const { listFunctions } = require('@netlify/zip-it-and-ship-it')
-  const functions = await listFunctions(functionsSrcDir)
+  const functions = await listFunctions(functionsSrcDir, {
+    parseISC: true,
+  })
   const functionsWithProps = functions.filter(({ runtime }) => runtime === JS).map((func) => addFunctionProps(func))
   return functionsWithProps
 }

src/utils/functions/get-functions.js

@@ -37,6 +37,7 @@ test('should return object with function details for a directory with js files',
         mainFile: path.join(builder.directory, 'functions', 'index.js'),
         isBackground: false,
         runtime: 'js',
+        schedule: undefined,
         urlPath: '/.netlify/functions/index',
       },
     ])
@@ -64,13 +65,15 @@ test('should mark background functions based on filenames', async (t) => {
         mainFile: path.join(builder.directory, 'functions', 'bar-background', 'bar-background.js'),
         isBackground: true,
         runtime: 'js',
+        schedule: undefined,
         urlPath: '/.netlify/functions/bar-background',
       },
       {
         name: 'foo-background',
         mainFile: path.join(builder.directory, 'functions', 'foo-background.js'),
         isBackground: true,
         runtime: 'js',
+        schedule: undefined,
         urlPath: '/.netlify/functions/foo-background',
       },
     ])

src/utils/functions/get-functions.test.js

@@ -1,8 +1,10 @@
+const constants = require('./constants')
 const edgeHandlers = require('./edge-handlers')
 const functions = require('./functions')
 const getFunctions = require('./get-functions')
 
 module.exports = {
+  ...constants,
   ...functions,
   ...edgeHandlers,
   ...getFunctions,

src/utils/functions/index.js

@@ -13,6 +13,7 @@ const { withDevServer } = require('./utils/dev-server')
 const got = require('./utils/got')
 const { CONFIRM, DOWN, answerWithValue, handleQuestions } = require('./utils/handle-questions')
 const { withMockApi } = require('./utils/mock-api')
+const { pause } = require('./utils/pause')
 const { killProcess } = require('./utils/process')
 const { withSiteBuilder } = require('./utils/site-builder')
 
@@ -582,6 +583,129 @@ test('should trigger background function from event', async (t) => {
   })
 })
 
+test('should serve helpful tips and tricks', async (t) => {
+  await withSiteBuilder('site-with-isc-ping-function', async (builder) => {
+    await builder
+      .withNetlifyToml({
+        config: { functions: { directory: 'functions' } },
+      })
+      // mocking until https://github.com/netlify/functions/pull/226 landed
+      .withContentFile({
+        path: 'node_modules/@netlify/functions/package.json',
+        content: `{}`,
+      })
+      .withContentFile({
+        path: 'node_modules/@netlify/functions/index.js',
+        content: `
+          module.exports.schedule = (schedule, handler) => handler
+          `,
+      })
+      .withContentFile({
+        path: 'functions/hello-world.js',
+        content: `
+          const { schedule } = require('@netlify/functions')
+
+          module.exports.handler = schedule('@daily', () => {
+            return {
+              statusCode: 200,
+              body: "hello world"
+            }
+          })
+          `.trim(),
+      })
+      .buildAsync()
+
+    await withDevServer({ cwd: builder.directory }, async (server) => {
+      const plainTextResponse = await got(`http://localhost:${server.port}/.netlify/functions/hello-world`, {
+        throwHttpErrors: false,
+        retry: null,
+      })
+      const youReturnedBodyRegex = /.*Your function returned `body`. Is this an accident\?.*/
+      t.regex(plainTextResponse.body, youReturnedBodyRegex)
+      t.regex(plainTextResponse.body, /.*You performed an HTTP request.*/)
+      t.is(plainTextResponse.statusCode, 200)
+
+      const htmlResponse = await got(`http://localhost:${server.port}/.netlify/functions/hello-world`, {
+        throwHttpErrors: false,
+        retry: null,
+        headers: {
+          accept: 'text/html',
+        },
+      })
+      t.regex(htmlResponse.body, /.*<link.*/)
+      t.is(htmlResponse.statusCode, 200)
+
+      const stdout = await callCli(['functions:invoke', 'hello-world', '--identity', `--port=${server.port}`], {
+        cwd: builder.directory,
+      })
+      t.regex(stdout, youReturnedBodyRegex)
+    })
+  })
+})
+
+test('should detect file changes to scheduled function', async (t) => {
+  await withSiteBuilder('site-with-isc-ping-function', async (builder) => {
+    await builder
+      .withNetlifyToml({
+        config: { functions: { directory: 'functions' } },
+      })
+      // mocking until https://github.com/netlify/functions/pull/226 landed
+      .withContentFile({
+        path: 'node_modules/@netlify/functions/package.json',
+        content: `{}`,
+      })
+      .withContentFile({
+        path: 'node_modules/@netlify/functions/index.js',
+        content: `
+          module.exports.schedule = (schedule, handler) => handler
+          `,
+      })
+      .withContentFile({
+        path: 'functions/hello-world.js',
+        content: `
+          module.exports.handler = () => {
+            return {
+              statusCode: 200
+            }
+          }
+          `.trim(),
+      })
+      .buildAsync()
+
+    await withDevServer({ cwd: builder.directory }, async (server) => {
+      const helloWorldBody = () =>
+        got(`http://localhost:${server.port}/.netlify/functions/hello-world`, {
+          throwHttpErrors: false,
+          retry: null,
+        }).then((response) => response.body)
+
+      t.is(await helloWorldBody(), '')
+
+      await builder
+        .withContentFile({
+          path: 'functions/hello-world.js',
+          content: `
+          const { schedule } = require('@netlify/functions')
+
+          module.exports.handler = schedule("@daily", () => {
+            return {
+              statusCode: 200,
+              body: "test"
+            }
+          })
+          `.trim(),
+        })
+        .buildAsync()
+
+      const DETECT_FILE_CHANGE_DELAY = 500
+      await pause(DETECT_FILE_CHANGE_DELAY)
+
+      const warningMessage = await helloWorldBody()
+      t.true(warningMessage.includes('Your function returned `body`'))
+    })
+  })
+})
+
 test('should inject env variables', async (t) => {
   await withSiteBuilder('site-with-env-function', async (builder) => {
     await builder