feat(gatsby): lazily compile functions in development (#31508) (#31579)

* feat(gatsby): lazily compile functions in development

* Throttle restarting the webpack server so tests don't fail

* Fix lint

* move comment

* Run tests serially now that we deleted old functions on startup

* fix lint

* Pull out throttling stuff and just run dev tests serially

* Fix watching functions in subdirectories

* fix

(cherry picked from commit d38f4d9)

Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com>

integration-tests/functions/package.json

@@ -4,17 +4,17 @@
   "private": true,
   "description": "functions",
   "author": "Kyle Mathews",
-  "keywords": [
-    "gatsby"
-  ],
+  "keywords": ["gatsby"],
   "scripts": {
     "build": "gatsby build",
     "develop": "gatsby develop",
     "serve": "gatsby serve",
-    "start-servers-and-test": "start-server-and-test servers \"8000|9000\" test:jest",
-    "servers": "run-p develop serve",
+    "test-prod": "start-server-and-test serve 9000 jest:prod",
+    "test-dev": "start-server-and-test develop 8000 jest:dev",
+    "jest:prod": "jest __tests__/functions-prod.js",
+    "jest:dev": "jest --runInBand __tests__/functions-dev.js",
     "test:jest": "jest",
-    "test": "npm-run-all -s build start-servers-and-test"
+    "test": "npm-run-all -s build test-prod test-dev"
   },
   "devDependencies": {
     "babel-jest": "^24.0.0",

packages/gatsby/src/internal-plugins/functions/gatsby-node.ts

@@ -43,6 +43,45 @@ interface IGlobPattern {
   globPattern: string
 }
 
+// During development, we lazily compile functions only when they're requested.
+// Here we keep track of which functions have been requested so are "active"
+const activeDevelopmentFunctions = new Set<IFunctionData>()
+let activeEntries = {}
+
+async function ensureFunctionIsCompiled(
+  functionObj: IFunctionData,
+  compiledFunctionsDir: string
+): any {
+  // stat the compiled function. If it's there, then return.
+  let compiledFileExists = false
+  try {
+    compiledFileExists = !!(await fs.stat(functionObj.absoluteCompiledFilePath))
+  } catch (e) {
+    // ignore
+  }
+  if (compiledFileExists) {
+    return
+  } else {
+    // Otherwise, restart webpack by touching the file and watch for the file to be
+    // compiled.
+    const time = new Date()
+    fs.utimesSync(functionObj.originalAbsoluteFilePath, time, time)
+    await new Promise(resolve => {
+      const watcher = chokidar
+        // Watch the root of the compiled function directory in .cache as chokidar
+        // can't watch files in directories that don't yet exist.
+        .watch(compiledFunctionsDir)
+        .on(`add`, async _path => {
+          if (_path === functionObj.absoluteCompiledFilePath) {
+            await watcher.close()
+
+            resolve(null)
+          }
+        })
+    })
+  }
+}
+
 // Create glob type w/ glob, plugin name, root path
 const createGlobArray = (siteDirectoryPath, plugins): Array<IGlobPattern> => {
   const globs: Array<IGlobPattern> = []
@@ -212,7 +251,10 @@ const createWebpackConfig = async ({
   )
 
   const entries = {}
-  knownFunctions.forEach(functionObj => {
+  const functionsList = isProductionEnv
+    ? knownFunctions
+    : activeDevelopmentFunctions
+  functionsList.forEach(functionObj => {
     // Get path without the extension (as it could be ts or js)
     const parsedFile = path.parse(functionObj.originalRelativeFilePath)
     const compiledNameWithoutExtension = path.join(
@@ -223,6 +265,8 @@ const createWebpackConfig = async ({
     entries[compiledNameWithoutExtension] = functionObj.originalAbsoluteFilePath
   })
 
+  activeEntries = entries
+
   const stage = isProductionEnv
     ? `functions-production`
     : `functions-development`
@@ -307,6 +351,7 @@ export async function onPreBootstrap({
   )
 
   await fs.ensureDir(compiledFunctionsDir)
+  await fs.emptyDir(compiledFunctionsDir)
 
   try {
     // We do this ungainly thing as we need to make accessible
@@ -350,7 +395,7 @@ export async function onPreBootstrap({
           }
         }
 
-        return resolve()
+        return resolve(null)
       }
 
       if (isProductionEnv) {
@@ -373,9 +418,14 @@ export async function onPreBootstrap({
             ],
             { ignoreInitial: true }
           )
-          .on(`all`, (event, path) => {
-            // Ignore change events from the API directory
-            if (event === `change` && path.includes(`/src/api/`)) {
+          .on(`all`, async (event, path) => {
+            // Ignore change events from the API directory for functions we're
+            // already watching.
+            if (
+              event === `change` &&
+              Object.values(activeEntries).includes(path) &&
+              path.includes(`/src/api/`)
+            ) {
               return
             }
 
@@ -410,6 +460,16 @@ export async function onCreateDevServer({
 }: CreateDevServerArgs): Promise<void> {
   reporter.verbose(`Attaching functions to development server`)
 
+  const {
+    program: { directory: siteDirectoryPath },
+  } = store.getState()
+
+  const compiledFunctionsDir = path.join(
+    siteDirectoryPath,
+    `.cache`,
+    `functions`
+  )
+
   app.use(
     `/api/*`,
     multer().any(),
@@ -466,6 +526,10 @@ export async function onCreateDevServer({
       }
 
       if (functionObj) {
+        activeDevelopmentFunctions.add(functionObj)
+
+        await ensureFunctionIsCompiled(functionObj, compiledFunctionsDir)
+
         reporter.verbose(`Running ${functionObj.functionRoute}`)
         const start = Date.now()
         const pathToFunction = functionObj.absoluteCompiledFilePath