fix: reverting migration to electron-rebuild since Cxx flags were back-ported to latest versions of electron (#7668)

Author: mmaietta

.changeset/fast-plums-play.md

@@ -0,0 +1,6 @@
+---
+"app-builder-lib": patch
+"electron-builder": patch
+---
+
+fix: reverting migration to electron-rebuild to resolve native prebuilt modules issue

packages/app-builder-lib/package.json

@@ -50,7 +50,6 @@
     "@develar/schema-utils": "~2.6.5",
     "@electron/notarize": "^1.2.3",
     "@electron/osx-sign": "^1.0.4",
-    "@electron/rebuild": "3.2.13",
     "@electron/universal": "1.3.4",
     "@malept/flatpak-bundler": "^0.4.0",
     "@types/fs-extra": "9.0.13",

packages/app-builder-lib/src/electron/electronVersion.ts

@@ -1,4 +1,3 @@
-import { getProjectRootPath } from "@electron/rebuild/lib/search-module"
 import { InvalidConfigurationError, log } from "builder-util"
 import { parseXml } from "builder-util-runtime"
 import { httpExecutor } from "builder-util/out/nodeHttpExecutor"
@@ -57,15 +56,8 @@ export async function computeElectronVersion(projectDir: string): Promise<string
     return result
   }
 
-  const potentialRootDirs = [projectDir, await getProjectRootPath(projectDir)]
-  let dependency: NameAndVersion | null = null
-  for await (const dir of potentialRootDirs) {
-    const metadata = await orNullIfFileNotExist(readJson(path.join(dir, "package.json")))
-    dependency = metadata ? findFromPackageMetadata(metadata) : null
-    if (dependency) {
-      break
-    }
-  }
+  const metadata = await orNullIfFileNotExist(readJson(path.join(projectDir, "package.json")))
+  const dependency = metadata ? findFromPackageMetadata(metadata) : null
   if (dependency?.name === "electron-nightly") {
     log.info("You are using a nightly version of electron, be warned that those builds are highly unstable.")
     const feedXml = await httpExecutor.request({

packages/app-builder-lib/src/packager.ts

@@ -495,7 +495,7 @@ export class Packager {
     const frameworkInfo = { version: this.framework.version, useCustomDist: true }
     const config = this.config
     if (config.nodeGypRebuild === true) {
-      await nodeGypRebuild(frameworkInfo, arch, platform)
+      await nodeGypRebuild(platform.nodeName, Arch[arch], frameworkInfo)
     }
 
     if (config.npmRebuild === false) {
@@ -526,6 +526,7 @@ export class Packager {
         frameworkInfo,
         platform: platform.nodeName,
         arch: Arch[arch],
+        productionDeps: this.getNodeDependencyInfo(null),
       })
     }
   }

packages/app-builder-lib/src/util/packageMetadata.ts

@@ -68,9 +68,9 @@ export function checkMetadata(metadata: Metadata, devMetadata: any | null, appPa
   }
 
   const devDependencies = (metadata as any).devDependencies
-  if (devDependencies != null && "@electron/rebuild" in devDependencies) {
+  if (devDependencies != null && ("electron-rebuild" in devDependencies || "@electron/rebuild" in devDependencies)) {
     log.info(
-      '@electron/rebuild is already incorporated into electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
+      '@electron/rebuild not required if you use electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
     )
   }
 
@@ -110,7 +110,7 @@ function checkDependencies(dependencies: { [key: string]: string } | null | unde
     errors.push(`At least electron-builder-squirrel-windows 20.32.0 is required by current electron-builder version. Please set electron-builder-squirrel-windows to "^20.32.0"`)
   }
 
-  const deps = ["electron", "electron-prebuilt", "@electron/rebuild"]
+  const deps = ["electron", "electron-prebuilt", "electron-rebuild"]
   if (process.env.ALLOW_ELECTRON_BUILDER_AS_PRODUCTION_DEPENDENCY !== "true") {
     deps.push("electron-builder")
   }

packages/app-builder-lib/src/util/yarn.ts

@@ -1,33 +1,32 @@
-import { Arch, archFromString, asArray, log, spawn } from "builder-util"
+import { asArray, log, spawn } from "builder-util"
 import { pathExists } from "fs-extra"
+import { Lazy } from "lazy-val"
 import { homedir } from "os"
 import * as path from "path"
 import { Configuration } from "../configuration"
-import * as electronRebuild from "@electron/rebuild/lib/rebuild"
-import * as searchModule from "@electron/rebuild/lib/search-module"
-import { EventEmitter } from "events"
-import { Platform } from "../core"
+import { executeAppBuilderAndWriteJson } from "./appBuilder"
+import { NodeModuleDirInfo } from "./packageDependencies"
 
 export async function installOrRebuild(config: Configuration, appDir: string, options: RebuildOptions, forceInstall = false) {
+  const effectiveOptions = {
+    buildFromSource: config.buildDependenciesFromSource === true,
+    additionalArgs: asArray(config.npmArgs),
+    ...options,
+  }
   let isDependenciesInstalled = false
+
   for (const fileOrDir of ["node_modules", ".pnp.js"]) {
     if (await pathExists(path.join(appDir, fileOrDir))) {
       isDependenciesInstalled = true
+
       break
     }
   }
 
   if (forceInstall || !isDependenciesInstalled) {
-    const effectiveOptions: RebuildOptions = {
-      buildFromSource: config.buildDependenciesFromSource === true,
-      additionalArgs: asArray(config.npmArgs),
-      ...options,
-    }
     await installDependencies(appDir, effectiveOptions)
   } else {
-    const arch = archFromString(options.arch || process.arch)
-    const platform = Platform.fromString(options.platform || process.platform)
-    await rebuild(appDir, config.buildDependenciesFromSource === true, options.frameworkInfo, arch, platform)
+    await rebuild(appDir, effectiveOptions)
   }
 }
 
@@ -120,8 +119,23 @@ function installDependencies(appDir: string, options: RebuildOptions): Promise<a
   })
 }
 
-export async function nodeGypRebuild(frameworkInfo: DesktopFrameworkInfo, arch: Arch, platform: Platform) {
-  return rebuild(process.cwd(), false, frameworkInfo, arch, platform)
+export async function nodeGypRebuild(platform: NodeJS.Platform, arch: string, frameworkInfo: DesktopFrameworkInfo) {
+  log.info({ platform, arch }, "executing node-gyp rebuild")
+  // this script must be used only for electron
+  const nodeGyp = `node-gyp${process.platform === "win32" ? ".cmd" : ""}`
+  const args = ["rebuild"]
+  // headers of old Electron versions do not have a valid config.gypi file
+  // and --force-process-config must be passed to node-gyp >= 8.4.0 to
+  // correctly build modules for them.
+  // see also https://github.com/nodejs/node-gyp/pull/2497
+  const [major, minor] = frameworkInfo.version
+    .split(".")
+    .slice(0, 2)
+    .map(n => parseInt(n, 10))
+  if (major <= 13 || (major == 14 && minor <= 1) || (major == 15 && minor <= 2)) {
+    args.push("--force-process-config")
+  }
+  await spawn(nodeGyp, args, { env: getGypEnv(frameworkInfo, platform, arch, true) })
 }
 
 function getPackageToolPath() {
@@ -139,6 +153,7 @@ function isRunningYarn(execPath: string | null | undefined) {
 
 export interface RebuildOptions {
   frameworkInfo: DesktopFrameworkInfo
+  productionDeps?: Lazy<Array<NodeModuleDirInfo>>
 
   platform?: NodeJS.Platform
   arch?: string
@@ -149,21 +164,17 @@ export interface RebuildOptions {
 }
 
 /** @internal */
-export async function rebuild(appDir: string, buildFromSource: boolean, frameworkInfo: DesktopFrameworkInfo, arch: Arch, platform: Platform) {
-  log.info({ arch: Arch[arch], platform: platform.name, version: frameworkInfo.version, appDir }, "executing @electron/rebuild")
-  const rootPath = await searchModule.getProjectRootPath(appDir)
-  const rebuilderOptions: electronRebuild.RebuilderOptions = {
-    buildPath: appDir,
-    electronVersion: frameworkInfo.version,
-    arch: Arch[arch],
-    projectRootPath: rootPath,
-    disablePreGypCopy: true,
-    lifecycle: new EventEmitter(),
-  }
-  if (buildFromSource) {
-    rebuilderOptions.prebuildTagPrefix = "totally-not-a-real-prefix-to-force-rebuild"
+export async function rebuild(appDir: string, options: RebuildOptions) {
+  const configuration: any = {
+    dependencies: await options.productionDeps!.value,
+    nodeExecPath: process.execPath,
+    platform: options.platform || process.platform,
+    arch: options.arch || process.arch,
+    additionalArgs: options.additionalArgs,
+    execPath: process.env.npm_execpath || process.env.NPM_CLI_JS,
+    buildFromSource: options.buildFromSource === true,
   }
-  const rebuilder = new electronRebuild.Rebuilder(rebuilderOptions)
-  rebuilder.platform = platform.nodeName
-  return rebuilder.rebuild()
+
+  const env = getGypEnv(options.frameworkInfo, configuration.platform, configuration.arch, options.buildFromSource === true)
+  await executeAppBuilderAndWriteJson(["rebuild-node-modules"], configuration, { env, cwd: appDir })
 }

packages/electron-builder/src/cli/cli.ts

@@ -78,5 +78,5 @@ async function checkIsOutdated() {
 async function rebuildAppNativeCode(args: any) {
   const projectDir = process.cwd()
   // this script must be used only for electron
-  return nodeGypRebuild({ version: await getElectronVersion(projectDir), useCustomDist: true }, args.arch, args.platform)
+  return nodeGypRebuild(args.platform, args.arch, { version: await getElectronVersion(projectDir), useCustomDist: true })
 }

pnpm-lock.yaml

@@ -957,6 +957,130 @@ Object {
           },
           "unpacked": true,
         },
+        "lzma-native": Object {
+          "files": Object {
+            "LICENSE": Object {
+              "size": "<size>",
+            },
+            "bin": Object {
+              "files": Object {
+                "lzmajs": Object {
+                  "executable": true,
+                  "size": "<size>",
+                },
+              },
+            },
+            "index.js": Object {
+              "size": "<size>",
+            },
+            "liblzma-build.sh": Object {
+              "executable": true,
+              "size": "<size>",
+            },
+            "liblzma-config.sh": Object {
+              "executable": true,
+              "size": "<size>",
+            },
+            "package.json": Object {
+              "size": "<size>",
+            },
+            "prebuilds": Object {
+              "files": Object {
+                "darwin-arm64": Object {
+                  "files": Object {
+                    "electron.napi.node": Object {
+                      "size": "<size>",
+                    },
+                    "node.napi.node": Object {
+                      "size": "<size>",
+                    },
+                  },
+                },
+                "darwin-x64": Object {
+                  "files": Object {
+                    "electron.napi.node": Object {
+                      "size": "<size>",
+                    },
+                    "node.napi.node": Object {
+                      "size": "<size>",
+                    },
+                  },
+                },
+                "linux-arm64": Object {
+                  "files": Object {
+                    "electron.napi.node": Object {
+                      "size": "<size>",
+                    },
+                    "node.napi.node": Object {
+                      "size": "<size>",
+                    },
+                  },
+                },
+                "linux-x64": Object {
+                  "files": Object {
+                    "electron.napi.node": Object {
+                      "size": "<size>",
+                    },
+                    "node.napi.node": Object {
+                      "size": "<size>",
+                    },
+                  },
+                },
+                "win32-ia32": Object {
+                  "files": Object {
+                    "electron.napi.node": Object {
+                      "size": "<size>",
+                    },
+                    "node.napi.node": Object {
+                      "size": "<size>",
+                    },
+                  },
+                },
+                "win32-x64": Object {
+                  "files": Object {
+                    "electron.napi.node": Object {
+                      "size": "<size>",
+                    },
+                    "node.napi.node": Object {
+                      "size": "<size>",
+                    },
+                  },
+                },
+              },
+            },
+            "src": Object {
+              "files": Object {
+                "filter-array.cpp": Object {
+                  "size": "<size>",
+                },
+                "index-parser.cpp": Object {
+                  "size": "<size>",
+                },
+                "index-parser.h": Object {
+                  "size": "<size>",
+                },
+                "liblzma-functions.cpp": Object {
+                  "size": "<size>",
+                },
+                "liblzma-node.hpp": Object {
+                  "size": "<size>",
+                },
+                "lzma-stream.cpp": Object {
+                  "size": "<size>",
+                },
+                "module.cpp": Object {
+                  "size": "<size>",
+                },
+                "mt-options.cpp": Object {
+                  "size": "<size>",
+                },
+                "util.cpp": Object {
+                  "size": "<size>",
+                },
+              },
+            },
+          },
+        },
         "mimic-response": Object {
           "files": Object {
             "index.js": Object {
@@ -1197,6 +1321,91 @@ Object {
             },
           },
         },
+        "node-addon-api": Object {
+          "files": Object {
+            "LICENSE.md": Object {
+              "size": "<size>",
+            },
+            "common.gypi": Object {
+              "size": "<size>",
+            },
+            "except.gypi": Object {
+              "size": "<size>",
+            },
+            "index.js": Object {
+              "size": "<size>",
+            },
+            "napi-inl.deprecated.h": Object {
+              "size": "<size>",
+            },
+            "napi-inl.h": Object {
+              "size": "<size>",
+            },
+            "napi.h": Object {
+              "size": "<size>",
+            },
+            "node_api.gyp": Object {
+              "size": "<size>",
+            },
+            "noexcept.gypi": Object {
+              "size": "<size>",
+            },
+            "nothing.c": Object {
+              "size": "<size>",
+            },
+            "package-support.json": Object {
+              "size": "<size>",
+            },
+            "package.json": Object {
+              "size": "<size>",
+            },
+            "tools": Object {
+              "files": Object {
+                "README.md": Object {
+                  "size": "<size>",
+                },
+                "check-napi.js": Object {
+                  "size": "<size>",
+                },
+                "clang-format.js": Object {
+                  "size": "<size>",
+                },
+                "conversion.js": Object {
+                  "executable": true,
+                  "size": "<size>",
+                },
+              },
+            },
+          },
+        },
+        "node-gyp-build": Object {
+          "files": Object {
+            "LICENSE": Object {
+              "size": "<size>",
+            },
+            "bin.js": Object {
+              "executable": true,
+              "size": "<size>",
+            },
+            "build-test.js": Object {
+              "executable": true,
+              "size": "<size>",
+            },
+            "index.js": Object {
+              "size": "<size>",
+            },
+            "node-gyp-build.js": Object {
+              "size": "<size>",
+            },
+            "optional.js": Object {
+              "executable": true,
+              "size": "<size>",
+            },
+            "package.json": Object {
+              "size": "<size>",
+            },
+          },
+        },
         "noop-logger": Object {
           "files": Object {
             "History.md": Object {

test/snapshots/BuildTest.js.snap

@@ -345,8 +345,7 @@ test.ifDevOrLinuxCi(
         it.dependencies = {
           debug: "4.1.1",
           "edge-cs": "1.2.1",
-          // no prebuilt for electron 3
-          // "lzma-native": "3.0.10",
+          "lzma-native": "8.0.6",
           keytar: "5.6.0",
         }
       }),