fix(gatsby): Persist manifest on cached builds (#36693)

* fix(gatsby): Persist manifest on cached builds

* Clarify manifest path

* Error handling

Author: tyhopp

packages/gatsby-cli/src/structured-errors/error-map.ts

@@ -783,6 +783,21 @@ const errors = {
     docsUrl: `https://gatsby.dev/partial-hydration-error`,
     category: ErrorCategory.USER,
   },
+  "80001": {
+    text: (): string =>
+      stripIndents(
+        `
+        Failed to restore previous client module manifest.
+        
+        This can happen if the manifest is corrupted or is not compatible with the current version of Gatsby.
+
+        Please run "gatsby clean" and try again. If the issue persists, please open an issue with a reproduction at https://github.com/gatsbyjs/gatsby/issues/new for more help.
+        `
+      ),
+    level: Level.ERROR,
+    docsUrl: `https://gatsby.dev/partial-hydration-error`,
+    category: ErrorCategory.USER,
+  },
 }
 
 export type ErrorId = string | keyof typeof errors

packages/gatsby/src/utils/webpack.config.js

@@ -10,7 +10,7 @@ const { store } = require(`../redux`)
 const { actions } = require(`../redux/actions`)
 const { getPublicPath } = require(`./get-public-path`)
 const debug = require(`debug`)(`gatsby:webpack-config`)
-const report = require(`gatsby-cli/lib/reporter`)
+const reporter = require(`gatsby-cli/lib/reporter`)
 import { withBasePath, withTrailingSlash } from "./path"
 import { getGatsbyDependents } from "./gatsby-dependents"
 const apiRunnerNode = require(`./api-runner-node`)
@@ -74,7 +74,7 @@ module.exports = async (
       parsed = dotenv.parse(fs.readFileSync(envFile, { encoding: `utf8` }))
     } catch (err) {
       if (err.code !== `ENOENT`) {
-        report.error(
+        reporter.error(
           `There was a problem processing the .env file (${envFile})`,
           err
         )
@@ -232,7 +232,7 @@ module.exports = async (
       plugins.virtualModules(),
       new BabelConfigItemsCacheInvalidatorPlugin(),
       process.env.GATSBY_WEBPACK_LOGGING?.split(`,`)?.includes(stage) &&
-        new WebpackLoggingPlugin(program.directory, report, program.verbose),
+        new WebpackLoggingPlugin(program.directory, reporter, program.verbose),
     ].filter(Boolean)
 
     switch (stage) {
@@ -285,8 +285,13 @@ module.exports = async (
             new StaticQueryMapper(store),
             isPartialHydrationEnabled
               ? new PartialHydrationPlugin(
-                  `../.cache/partial-hydration/manifest.json`,
-                  path.join(directory, `.cache`, `public-page-renderer-prod.js`)
+                  path.join(
+                    directory,
+                    `.cache`,
+                    `partial-hydration`,
+                    `manifest.json`
+                  ),
+                  reporter
                 )
               : null,
           ])

packages/gatsby/src/utils/webpack/plugins/partial-hydration.ts

@@ -1,9 +1,11 @@
 import * as path from "path"
+import fs from "fs-extra"
 import Template from "webpack/lib/Template"
 import ModuleDependency from "webpack/lib/dependencies/ModuleDependency"
 import NullDependency from "webpack/lib/dependencies/NullDependency"
 import { createNormalizedModuleKey } from "../utils/create-normalized-module-key"
 import webpack, { Module, NormalModule, Dependency, javascript } from "webpack"
+import type Reporter from "gatsby-cli/lib/reporter"
 
 interface IModuleExport {
   id: string
@@ -33,14 +35,17 @@ class ClientReferenceDependency extends ModuleDependency {
  */
 export class PartialHydrationPlugin {
   name = `PartialHydrationPlugin`
-  _manifestPath: string
-  _rootFilePath: string
+
+  _manifestPath: string // Absolute path to where the manifest file should be written
+  _reporter: typeof Reporter
+
   _references: Array<ClientReferenceDependency> = []
   _clientModules = new Set<webpack.NormalModule>()
+  _previousManifest = {}
 
-  constructor(manifestPath: string, rootFilePath: string) {
+  constructor(manifestPath: string, reporter: typeof Reporter) {
     this._manifestPath = manifestPath
-    this._rootFilePath = rootFilePath
+    this._reporter = reporter
   }
 
   _generateClientReferenceChunk(
@@ -188,6 +193,27 @@ export class PartialHydrationPlugin {
   }
 
   apply(compiler: webpack.Compiler): void {
+    // Restore manifest from the previous compilation, otherwise it will be wiped since files aren't visited on cached builds
+    compiler.hooks.beforeCompile.tap(this.name, () => {
+      try {
+        const previousManifest = fs.existsSync(this._manifestPath)
+
+        if (!previousManifest) {
+          return
+        }
+
+        this._previousManifest = JSON.parse(
+          fs.readFileSync(this._manifestPath, `utf-8`)
+        )
+      } catch (error) {
+        this._reporter.panic({
+          id: `80001`,
+          context: {},
+          error,
+        })
+      }
+    })
+
     compiler.hooks.thisCompilation.tap(
       this.name,
       (compilation, { normalModuleFactory }) => {
@@ -298,10 +324,23 @@ export class PartialHydrationPlugin {
               compilation.options.context as string
             )
 
+            /**
+             * `emitAsset` is unclear about what the path should be relative to and absolute paths don't work. This works so we'll go with that.
+             * @see {@link https://webpack.js.org/api/compilation-object/#emitasset}
+             */
+            const emitManifestPath = `..${this._manifestPath.replace(
+              compiler.context,
+              ``
+            )}`
+
             compilation.emitAsset(
-              this._manifestPath,
+              emitManifestPath,
               new webpack.sources.RawSource(
-                JSON.stringify(manifest, null, 2),
+                JSON.stringify(
+                  { ...this._previousManifest, ...manifest },
+                  null,
+                  2
+                ),
                 false
               )
             )