feat(cli): support @napi-rs/cross-toolchain (#1838)

.github/workflows/test-release.yaml

@@ -394,11 +394,6 @@ jobs:
           toolchain: stable
           targets: armv7-unknown-linux-gnueabihf
 
-      - name: Install ziglang
-        uses: goto-bus-stop/setup-zig@v2
-        with:
-          version: 0.11.0
-
       - name: Cache cargo
         uses: actions/cache@v3
         with:
@@ -414,14 +409,14 @@ jobs:
           yarn install --immutable --mode=skip-build
 
       - name: Cross build native tests
-        run: yarn build:test -- --target armv7-unknown-linux-gnueabihf --cross-compile
+        run: yarn build:test -- --target armv7-unknown-linux-gnueabihf --use-napi-cross
 
       - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
       - name: Setup and run tests
         uses: addnab/docker-run-action@v3
         with:
-          image: node:lts-bullseye-slim
+          image: node:lts-slim
           options: --platform linux/arm/v7 -v ${{ github.workspace }}:/build -w /build
           run: yarn test
 

cli/codegen/commands.ts

@@ -243,6 +243,12 @@ const BUILD_OPTIONS: CommandSchema = {
       description:
         '[experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo`',
     },
+    {
+      name: 'useNapiCross',
+      type: 'boolean',
+      description:
+        '[experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets.',
+    },
     {
       name: 'watch',
       type: 'boolean',

cli/docs/build.md

@@ -47,6 +47,7 @@ new NapiCli().build({
 | profile           | --profile             | string   | false    |         | Build artifacts with the specified profile                                                                                |
 | crossCompile      | --cross-compile,-x    | boolean  | false    |         | [experimental] cross-compile for the specified target with `cargo-xwin` on windows and `cargo-zigbuild` on other platform |
 | useCross          | --use-cross           | boolean  | false    |         | [experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo`                                          |
+| useNapiCross      | --use-napi-cross      | boolean  | false    |         | [experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets                              |
 | watch             | --watch,-w            | boolean  | false    |         | watch the crate changes and build continiously with `cargo-watch` crates                                                  |
 | features          | --features,-F         | string[] | false    |         | Space-separated list of features to activate                                                                              |
 | allFeatures       | --all-features        | boolean  | false    |         | Activate all available features                                                                                           |

cli/esbuild.mjs

@@ -5,6 +5,7 @@ await esbuild.build({
   outfile: './dist/index.cjs',
   bundle: true,
   platform: 'node',
+  external: ['@napi-rs/lzma', '@napi-rs/tar'],
   define: {
     'import.meta.url': '__filename',
   },

cli/package.json

@@ -67,6 +67,7 @@
     "url": "https://github.com/napi-rs/napi-rs/issues"
   },
   "dependencies": {
+    "@napi-rs/cross-toolchain": "^0.0.12",
     "@octokit/rest": "^20.0.2",
     "@tybys/wasm-util": "0.8.0",
     "clipanion": "^3.2.1",

cli/src/api/build.ts

@@ -1,7 +1,8 @@
 import { spawn } from 'node:child_process'
 import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync } from 'node:fs'
 import { createRequire } from 'node:module'
-import { tmpdir } from 'node:os'
+import { tmpdir, homedir } from 'node:os'
 import { parse, join, resolve } from 'node:path'
 
 import * as colors from 'colorette'
@@ -143,11 +144,103 @@ class Builder {
       .setPackage()
       .setFeatures()
       .setTarget()
+      .pickCrossToolchain()
       .setEnvs()
       .setBypassArgs()
       .exec()
   }
 
+  private pickCrossToolchain() {
+    if (!this.options.useNapiCross) {
+      return this
+    }
+    if (this.options.useCross) {
+      debug.warn(
+        'You are trying to use both `--cross` and `--use-napi-cross` options, `--use-cross` will be ignored.',
+      )
+    }
+
+    if (this.options.crossCompile) {
+      debug.warn(
+        'You are trying to use both `--cross-compile` and `--use-napi-cross` options, `--cross-compile` will be ignored.',
+      )
+    }
+
+    try {
+      const { version, download } = require('@napi-rs/cross-toolchain')
+
+      const toolchainPath = join(
+        homedir(),
+        '.napi-rs',
+        'cross-toolchain',
+        version,
+        this.target.triple,
+      )
+      mkdirSync(toolchainPath, { recursive: true })
+      if (existsSync(join(toolchainPath, 'package.json'))) {
+        debug(`Toolchain ${toolchainPath} exists, skip extracting`)
+      } else {
+        const tarArchive = download(process.arch, this.target.triple)
+        tarArchive.unpack(toolchainPath)
+      }
+      const upperCaseTarget = targetToEnvVar(this.target.triple)
+      const linkerEnv = `CARGO_TARGET_${upperCaseTarget}_LINKER`
+      this.envs[linkerEnv] = join(
+        toolchainPath,
+        'bin',
+        `${this.target.triple}-gcc`,
+      )
+      if (!process.env.TARGET_SYSROOT) {
+        this.envs[`TARGET_SYSROOT`] = join(
+          toolchainPath,
+          this.target.triple,
+          'sysroot',
+        )
+      }
+      if (!process.env.TARGET_AR) {
+        this.envs[`TARGET_AR`] = join(
+          toolchainPath,
+          'bin',
+          `${this.target.triple}-ar`,
+        )
+      }
+      if (!process.env.TARGET_RANLIB) {
+        this.envs[`TARGET_RANLIB`] = join(
+          toolchainPath,
+          'bin',
+          `${this.target.triple}-ranlib`,
+        )
+      }
+      if (!process.env.TARGET_READELF) {
+        this.envs[`TARGET_READELF`] = join(
+          toolchainPath,
+          'bin',
+          `${this.target.triple}-readelf`,
+        )
+      }
+      if (!process.env.TARGET_C_INCLUDE_PATH) {
+        this.envs[`TARGET_C_INCLUDE_PATH`] = join(
+          toolchainPath,
+          this.target.triple,
+          'sysroot',
+          'usr',
+          'include/',
+        )
+      }
+      if (
+        (process.env.CC === 'clang' &&
+          (process.env.TARGET_CC === 'clang' || !process.env.TARGET_CC)) ||
+        process.env.TARGET_CC === 'clang'
+      ) {
+        this.envs.C_FLAGS = `--sysroot=${this.envs.TARGET_SYSROOT}`
+      }
+    } catch (e) {
+      debug.warn('Pick cross toolchain failed', e as Error)
+      // ignore, do nothing
+    }
+    return this
+  }
+
   private exec() {
     debug(`Start building crate: ${this.crate.name}`)
     debug('  %i', `cargo ${this.args.join(' ')}`)
@@ -156,6 +249,11 @@ class Builder {
 
     const watch = this.options.watch
     const buildTask = new Promise<void>((resolve, reject) => {
+      if (this.options.useCross && this.options.crossCompile) {
+        throw new Error(
+          '`--use-cross` and `--cross-compile` can not be used together',
+        )
+      }
       const command =
         process.env.CARGO ?? (this.options.useCross ? 'cross' : 'cargo')
       const buildProcess = spawn(command, this.args, {

cli/src/def/build.ts

@@ -110,6 +110,11 @@ export abstract class BaseBuildCommand extends Command {
       '[experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo`',
   })
 
+  useNapiCross?: boolean = Option.Boolean('--use-napi-cross', {
+    description:
+      '[experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets',
+  })
+
   watch?: boolean = Option.Boolean('--watch,-w', {
     description:
       'watch the crate changes and build continiously with `cargo-watch` crates',
@@ -151,6 +156,7 @@ export abstract class BaseBuildCommand extends Command {
       profile: this.profile,
       crossCompile: this.crossCompile,
       useCross: this.useCross,
+      useNapiCross: this.useNapiCross,
       watch: this.watch,
       features: this.features,
       allFeatures: this.allFeatures,
@@ -251,6 +257,10 @@ export interface BuildOptions {
    * [experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo`
    */
   useCross?: boolean
+  /**
+   * [experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets
+   */
+  useNapiCross?: boolean
   /**
    * watch the crate changes and build continiously with `cargo-watch` crates
    */

cli/src/utils/config.ts

@@ -120,7 +120,7 @@ export async function readNapiConfig(path: string): Promise<NapiConfig> {
 
   // compatible with old config
   if (userNapiConfig?.name) {
-    napiConfig.packageName = userNapiConfig.name
+    napiConfig.binaryName = userNapiConfig.name
   }
 
   if (!targets.length) {

cli/src/utils/target.ts

@@ -1,4 +1,4 @@
-import { execSync } from 'child_process'
+import { execSync } from 'node:child_process'
 
 export type Platform = NodeJS.Platform | 'wasm' | 'wasi'
 

yarn.lock

@@ -556,6 +556,7 @@ __metadata:
   dependencies:
     "@emnapi/core": "npm:0.44.0"
     "@emnapi/runtime": "npm:0.44.0"
+    "@napi-rs/cross-toolchain": "npm:^0.0.12"
     "@octokit/rest": "npm:^20.0.2"
     "@tybys/wasm-util": "npm:0.8.0"
     "@types/debug": "npm:^4.1.12"
@@ -595,6 +596,315 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@napi-rs/cross-toolchain@npm:^0.0.12":
+  version: 0.0.12
+  resolution: "@napi-rs/cross-toolchain@npm:0.0.12"
+  dependencies:
+    "@napi-rs/lzma": "npm:^1.2.1"
+    "@napi-rs/tar": "npm:^0.0.1"
+    debug: "npm:^4.3.4"
+  peerDependencies:
+    "@napi-rs/cross-toolchain-arm64-target-aarch64": ^0.0.12
+    "@napi-rs/cross-toolchain-arm64-target-armv7": ^0.0.12
+    "@napi-rs/cross-toolchain-arm64-target-x86_64": ^0.0.12
+    "@napi-rs/cross-toolchain-x64-target-aarch64": ^0.0.12
+    "@napi-rs/cross-toolchain-x64-target-armv7": ^0.0.12
+    "@napi-rs/cross-toolchain-x64-target-x86_64": ^0.0.12
+  peerDependenciesMeta:
+    "@napi-rs/cross-toolchain-arm64-target-aarch64":
+      optional: true
+    "@napi-rs/cross-toolchain-arm64-target-armv7":
+      optional: true
+    "@napi-rs/cross-toolchain-arm64-target-x86_64":
+      optional: true
+    "@napi-rs/cross-toolchain-x64-target-aarch64":
+      optional: true
+    "@napi-rs/cross-toolchain-x64-target-armv7":
+      optional: true
+    "@napi-rs/cross-toolchain-x64-target-x86_64":
+      optional: true
+  checksum: de65d9ce93d9754d46322bd0ebda9c7e2bf63e1fa57777e843cac1f5a0c94ca57af208ff22fad15ef9cc9661575ffc1787e8d946c455dd04cecbb5c44f174655
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-android-arm-eabi@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-android-arm-eabi@npm:1.2.1"
+  conditions: os=android & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-android-arm64@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-android-arm64@npm:1.2.1"
+  conditions: os=android & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-darwin-arm64@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-darwin-arm64@npm:1.2.1"
+  conditions: os=darwin & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-darwin-x64@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-darwin-x64@npm:1.2.1"
+  conditions: os=darwin & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-freebsd-x64@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-freebsd-x64@npm:1.2.1"
+  conditions: os=freebsd & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-linux-arm-gnueabihf@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-linux-arm-gnueabihf@npm:1.2.1"
+  conditions: os=linux & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-linux-arm64-gnu@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-linux-arm64-gnu@npm:1.2.1"
+  conditions: os=linux & cpu=arm64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-linux-arm64-musl@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-linux-arm64-musl@npm:1.2.1"
+  conditions: os=linux & cpu=arm64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-linux-x64-gnu@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-linux-x64-gnu@npm:1.2.1"
+  conditions: os=linux & cpu=x64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-linux-x64-musl@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-linux-x64-musl@npm:1.2.1"
+  conditions: os=linux & cpu=x64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-win32-arm64-msvc@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-win32-arm64-msvc@npm:1.2.1"
+  conditions: os=win32 & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-win32-ia32-msvc@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-win32-ia32-msvc@npm:1.2.1"
+  conditions: os=win32 & cpu=ia32
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma-win32-x64-msvc@npm:1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma-win32-x64-msvc@npm:1.2.1"
+  conditions: os=win32 & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/lzma@npm:^1.2.1":
+  version: 1.2.1
+  resolution: "@napi-rs/lzma@npm:1.2.1"
+  dependencies:
+    "@napi-rs/lzma-android-arm-eabi": "npm:1.2.1"
+    "@napi-rs/lzma-android-arm64": "npm:1.2.1"
+    "@napi-rs/lzma-darwin-arm64": "npm:1.2.1"
+    "@napi-rs/lzma-darwin-x64": "npm:1.2.1"
+    "@napi-rs/lzma-freebsd-x64": "npm:1.2.1"
+    "@napi-rs/lzma-linux-arm-gnueabihf": "npm:1.2.1"
+    "@napi-rs/lzma-linux-arm64-gnu": "npm:1.2.1"
+    "@napi-rs/lzma-linux-arm64-musl": "npm:1.2.1"
+    "@napi-rs/lzma-linux-x64-gnu": "npm:1.2.1"
+    "@napi-rs/lzma-linux-x64-musl": "npm:1.2.1"
+    "@napi-rs/lzma-win32-arm64-msvc": "npm:1.2.1"
+    "@napi-rs/lzma-win32-ia32-msvc": "npm:1.2.1"
+    "@napi-rs/lzma-win32-x64-msvc": "npm:1.2.1"
+  dependenciesMeta:
+    "@napi-rs/lzma-android-arm-eabi":
+      optional: true
+    "@napi-rs/lzma-android-arm64":
+      optional: true
+    "@napi-rs/lzma-darwin-arm64":
+      optional: true
+    "@napi-rs/lzma-darwin-x64":
+      optional: true
+    "@napi-rs/lzma-freebsd-x64":
+      optional: true
+    "@napi-rs/lzma-linux-arm-gnueabihf":
+      optional: true
+    "@napi-rs/lzma-linux-arm64-gnu":
+      optional: true
+    "@napi-rs/lzma-linux-arm64-musl":
+      optional: true
+    "@napi-rs/lzma-linux-x64-gnu":
+      optional: true
+    "@napi-rs/lzma-linux-x64-musl":
+      optional: true
+    "@napi-rs/lzma-win32-arm64-msvc":
+      optional: true
+    "@napi-rs/lzma-win32-ia32-msvc":
+      optional: true
+    "@napi-rs/lzma-win32-x64-msvc":
+      optional: true
+  checksum: 8fc856aa1f547a2ca5477afd4aa7079069d7fb0ce5da64ba4a3d5eb243210a6bb591f1f0fa9584f0f272952bd1d11b99eb097a1d2be9f919c8a9dcd1d8bceb2c
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-android-arm-eabi@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-android-arm-eabi@npm:0.0.1"
+  conditions: os=android & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-android-arm64@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-android-arm64@npm:0.0.1"
+  conditions: os=android & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-darwin-arm64@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-darwin-arm64@npm:0.0.1"
+  conditions: os=darwin & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-darwin-x64@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-darwin-x64@npm:0.0.1"
+  conditions: os=darwin & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-freebsd-x64@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-freebsd-x64@npm:0.0.1"
+  conditions: os=freebsd & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-linux-arm-gnueabihf@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-linux-arm-gnueabihf@npm:0.0.1"
+  conditions: os=linux & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-linux-arm64-gnu@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-linux-arm64-gnu@npm:0.0.1"
+  conditions: os=linux & cpu=arm64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-linux-arm64-musl@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-linux-arm64-musl@npm:0.0.1"
+  conditions: os=linux & cpu=arm64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-linux-x64-gnu@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-linux-x64-gnu@npm:0.0.1"
+  conditions: os=linux & cpu=x64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-linux-x64-musl@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-linux-x64-musl@npm:0.0.1"
+  conditions: os=linux & cpu=x64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-win32-arm64-msvc@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-win32-arm64-msvc@npm:0.0.1"
+  conditions: os=win32 & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-win32-ia32-msvc@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-win32-ia32-msvc@npm:0.0.1"
+  conditions: os=win32 & cpu=ia32
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar-win32-x64-msvc@npm:0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar-win32-x64-msvc@npm:0.0.1"
+  conditions: os=win32 & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@napi-rs/tar@npm:^0.0.1":
+  version: 0.0.1
+  resolution: "@napi-rs/tar@npm:0.0.1"
+  dependencies:
+    "@napi-rs/tar-android-arm-eabi": "npm:0.0.1"
+    "@napi-rs/tar-android-arm64": "npm:0.0.1"
+    "@napi-rs/tar-darwin-arm64": "npm:0.0.1"
+    "@napi-rs/tar-darwin-x64": "npm:0.0.1"
+    "@napi-rs/tar-freebsd-x64": "npm:0.0.1"
+    "@napi-rs/tar-linux-arm-gnueabihf": "npm:0.0.1"
+    "@napi-rs/tar-linux-arm64-gnu": "npm:0.0.1"
+    "@napi-rs/tar-linux-arm64-musl": "npm:0.0.1"
+    "@napi-rs/tar-linux-x64-gnu": "npm:0.0.1"
+    "@napi-rs/tar-linux-x64-musl": "npm:0.0.1"
+    "@napi-rs/tar-win32-arm64-msvc": "npm:0.0.1"
+    "@napi-rs/tar-win32-ia32-msvc": "npm:0.0.1"
+    "@napi-rs/tar-win32-x64-msvc": "npm:0.0.1"
+  dependenciesMeta:
+    "@napi-rs/tar-android-arm-eabi":
+      optional: true
+    "@napi-rs/tar-android-arm64":
+      optional: true
+    "@napi-rs/tar-darwin-arm64":
+      optional: true
+    "@napi-rs/tar-darwin-x64":
+      optional: true
+    "@napi-rs/tar-freebsd-x64":
+      optional: true
+    "@napi-rs/tar-linux-arm-gnueabihf":
+      optional: true
+    "@napi-rs/tar-linux-arm64-gnu":
+      optional: true
+    "@napi-rs/tar-linux-arm64-musl":
+      optional: true
+    "@napi-rs/tar-linux-x64-gnu":
+      optional: true
+    "@napi-rs/tar-linux-x64-musl":
+      optional: true
+    "@napi-rs/tar-win32-arm64-msvc":
+      optional: true
+    "@napi-rs/tar-win32-ia32-msvc":
+      optional: true
+    "@napi-rs/tar-win32-x64-msvc":
+      optional: true
+  checksum: f38bc820f20be8e6c7b95bb846e1bedd2ab8b09da4565b18b1ac5396c9e63d5c409b7774d687568b45602f84484436fd86e5891d98db2d6ff8f8b8597194e158
+  languageName: node
+  linkType: hard
+
 "@napi-rs/triples@workspace:triples":
   version: 0.0.0-use.local
   resolution: "@napi-rs/triples@workspace:triples"