From 91c0eb8ce8bf452b4843dd33540185a10d73d53a Mon Sep 17 00:00:00 2001 From: LongYinan Date: Tue, 5 Dec 2023 17:27:23 +0800 Subject: [PATCH] feat(cli): support @napi-rs/cross-toolchain (#1838) --- .github/workflows/test-release.yaml | 9 +- cli/codegen/commands.ts | 6 + cli/docs/build.md | 1 + cli/esbuild.mjs | 1 + cli/package.json | 1 + cli/src/api/build.ts | 100 ++++++++- cli/src/def/build.ts | 10 + cli/src/utils/config.ts | 2 +- cli/src/utils/target.ts | 2 +- yarn.lock | 310 ++++++++++++++++++++++++++++ 10 files changed, 432 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index a3e3a0f9..36bec0ac 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -394,11 +394,6 @@ jobs: toolchain: stable targets: armv7-unknown-linux-gnueabihf - - name: Install ziglang - uses: goto-bus-stop/setup-zig@v2 - with: - version: 0.11.0 - - name: Cache cargo uses: actions/cache@v3 with: @@ -414,14 +409,14 @@ jobs: yarn install --immutable --mode=skip-build - name: Cross build native tests - run: yarn build:test -- --target armv7-unknown-linux-gnueabihf --cross-compile + run: yarn build:test -- --target armv7-unknown-linux-gnueabihf --use-napi-cross - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - name: Setup and run tests uses: addnab/docker-run-action@v3 with: - image: node:lts-bullseye-slim + image: node:lts-slim options: --platform linux/arm/v7 -v ${{ github.workspace }}:/build -w /build run: yarn test diff --git a/cli/codegen/commands.ts b/cli/codegen/commands.ts index 2ded547f..8e5e79a0 100644 --- a/cli/codegen/commands.ts +++ b/cli/codegen/commands.ts @@ -243,6 +243,12 @@ const BUILD_OPTIONS: CommandSchema = { description: '[experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo`', }, + { + name: 'useNapiCross', + type: 'boolean', + description: + '[experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets.', + }, { name: 'watch', type: 'boolean', diff --git a/cli/docs/build.md b/cli/docs/build.md index 9ecb3a9b..6b13d3a7 100644 --- a/cli/docs/build.md +++ b/cli/docs/build.md @@ -47,6 +47,7 @@ new NapiCli().build({ | profile | --profile | string | false | | Build artifacts with the specified profile | | crossCompile | --cross-compile,-x | boolean | false | | [experimental] cross-compile for the specified target with `cargo-xwin` on windows and `cargo-zigbuild` on other platform | | useCross | --use-cross | boolean | false | | [experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo` | +| useNapiCross | --use-napi-cross | boolean | false | | [experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets | | watch | --watch,-w | boolean | false | | watch the crate changes and build continiously with `cargo-watch` crates | | features | --features,-F | string[] | false | | Space-separated list of features to activate | | allFeatures | --all-features | boolean | false | | Activate all available features | diff --git a/cli/esbuild.mjs b/cli/esbuild.mjs index 14014aa0..1f048a13 100644 --- a/cli/esbuild.mjs +++ b/cli/esbuild.mjs @@ -5,6 +5,7 @@ await esbuild.build({ outfile: './dist/index.cjs', bundle: true, platform: 'node', + external: ['@napi-rs/lzma', '@napi-rs/tar'], define: { 'import.meta.url': '__filename', }, diff --git a/cli/package.json b/cli/package.json index 8e9a2ae1..82cf710f 100644 --- a/cli/package.json +++ b/cli/package.json @@ -67,6 +67,7 @@ "url": "https://github.com/napi-rs/napi-rs/issues" }, "dependencies": { + "@napi-rs/cross-toolchain": "^0.0.12", "@octokit/rest": "^20.0.2", "@tybys/wasm-util": "0.8.0", "clipanion": "^3.2.1", diff --git a/cli/src/api/build.ts b/cli/src/api/build.ts index 61bdcc1a..f60b1cbd 100644 --- a/cli/src/api/build.ts +++ b/cli/src/api/build.ts @@ -1,7 +1,8 @@ import { spawn } from 'node:child_process' import { createHash } from 'node:crypto' +import { existsSync, mkdirSync } from 'node:fs' import { createRequire } from 'node:module' -import { tmpdir } from 'node:os' +import { tmpdir, homedir } from 'node:os' import { parse, join, resolve } from 'node:path' import * as colors from 'colorette' @@ -143,11 +144,103 @@ class Builder { .setPackage() .setFeatures() .setTarget() + .pickCrossToolchain() .setEnvs() .setBypassArgs() .exec() } + private pickCrossToolchain() { + if (!this.options.useNapiCross) { + return this + } + if (this.options.useCross) { + debug.warn( + 'You are trying to use both `--cross` and `--use-napi-cross` options, `--use-cross` will be ignored.', + ) + } + + if (this.options.crossCompile) { + debug.warn( + 'You are trying to use both `--cross-compile` and `--use-napi-cross` options, `--cross-compile` will be ignored.', + ) + } + + try { + const { version, download } = require('@napi-rs/cross-toolchain') + + const toolchainPath = join( + homedir(), + '.napi-rs', + 'cross-toolchain', + version, + this.target.triple, + ) + mkdirSync(toolchainPath, { recursive: true }) + if (existsSync(join(toolchainPath, 'package.json'))) { + debug(`Toolchain ${toolchainPath} exists, skip extracting`) + } else { + const tarArchive = download(process.arch, this.target.triple) + tarArchive.unpack(toolchainPath) + } + const upperCaseTarget = targetToEnvVar(this.target.triple) + const linkerEnv = `CARGO_TARGET_${upperCaseTarget}_LINKER` + this.envs[linkerEnv] = join( + toolchainPath, + 'bin', + `${this.target.triple}-gcc`, + ) + if (!process.env.TARGET_SYSROOT) { + this.envs[`TARGET_SYSROOT`] = join( + toolchainPath, + this.target.triple, + 'sysroot', + ) + } + if (!process.env.TARGET_AR) { + this.envs[`TARGET_AR`] = join( + toolchainPath, + 'bin', + `${this.target.triple}-ar`, + ) + } + if (!process.env.TARGET_RANLIB) { + this.envs[`TARGET_RANLIB`] = join( + toolchainPath, + 'bin', + `${this.target.triple}-ranlib`, + ) + } + if (!process.env.TARGET_READELF) { + this.envs[`TARGET_READELF`] = join( + toolchainPath, + 'bin', + `${this.target.triple}-readelf`, + ) + } + if (!process.env.TARGET_C_INCLUDE_PATH) { + this.envs[`TARGET_C_INCLUDE_PATH`] = join( + toolchainPath, + this.target.triple, + 'sysroot', + 'usr', + 'include/', + ) + } + if ( + (process.env.CC === 'clang' && + (process.env.TARGET_CC === 'clang' || !process.env.TARGET_CC)) || + process.env.TARGET_CC === 'clang' + ) { + this.envs.C_FLAGS = `--sysroot=${this.envs.TARGET_SYSROOT}` + } + } catch (e) { + debug.warn('Pick cross toolchain failed', e as Error) + // ignore, do nothing + } + return this + } + private exec() { debug(`Start building crate: ${this.crate.name}`) debug(' %i', `cargo ${this.args.join(' ')}`) @@ -156,6 +249,11 @@ class Builder { const watch = this.options.watch const buildTask = new Promise((resolve, reject) => { + if (this.options.useCross && this.options.crossCompile) { + throw new Error( + '`--use-cross` and `--cross-compile` can not be used together', + ) + } const command = process.env.CARGO ?? (this.options.useCross ? 'cross' : 'cargo') const buildProcess = spawn(command, this.args, { diff --git a/cli/src/def/build.ts b/cli/src/def/build.ts index da09b45d..23e552b0 100644 --- a/cli/src/def/build.ts +++ b/cli/src/def/build.ts @@ -110,6 +110,11 @@ export abstract class BaseBuildCommand extends Command { '[experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo`', }) + useNapiCross?: boolean = Option.Boolean('--use-napi-cross', { + description: + '[experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets', + }) + watch?: boolean = Option.Boolean('--watch,-w', { description: 'watch the crate changes and build continiously with `cargo-watch` crates', @@ -151,6 +156,7 @@ export abstract class BaseBuildCommand extends Command { profile: this.profile, crossCompile: this.crossCompile, useCross: this.useCross, + useNapiCross: this.useNapiCross, watch: this.watch, features: this.features, allFeatures: this.allFeatures, @@ -251,6 +257,10 @@ export interface BuildOptions { * [experimental] use [cross](https://github.com/cross-rs/cross) instead of `cargo` */ useCross?: boolean + /** + * [experimental] use @napi-rs/cross-toolchain to cross-compile Linux arm/arm64/x64 gnu targets + */ + useNapiCross?: boolean /** * watch the crate changes and build continiously with `cargo-watch` crates */ diff --git a/cli/src/utils/config.ts b/cli/src/utils/config.ts index c83bd9be..7546f61a 100644 --- a/cli/src/utils/config.ts +++ b/cli/src/utils/config.ts @@ -120,7 +120,7 @@ export async function readNapiConfig(path: string): Promise { // compatible with old config if (userNapiConfig?.name) { - napiConfig.packageName = userNapiConfig.name + napiConfig.binaryName = userNapiConfig.name } if (!targets.length) { diff --git a/cli/src/utils/target.ts b/cli/src/utils/target.ts index fe5a18b3..679df9b0 100644 --- a/cli/src/utils/target.ts +++ b/cli/src/utils/target.ts @@ -1,4 +1,4 @@ -import { execSync } from 'child_process' +import { execSync } from 'node:child_process' export type Platform = NodeJS.Platform | 'wasm' | 'wasi' diff --git a/yarn.lock b/yarn.lock index f3bfdbfb..9b794afd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -556,6 +556,7 @@ __metadata: dependencies: "@emnapi/core": "npm:0.44.0" "@emnapi/runtime": "npm:0.44.0" + "@napi-rs/cross-toolchain": "npm:^0.0.12" "@octokit/rest": "npm:^20.0.2" "@tybys/wasm-util": "npm:0.8.0" "@types/debug": "npm:^4.1.12" @@ -595,6 +596,315 @@ __metadata: languageName: unknown linkType: soft +"@napi-rs/cross-toolchain@npm:^0.0.12": + version: 0.0.12 + resolution: "@napi-rs/cross-toolchain@npm:0.0.12" + dependencies: + "@napi-rs/lzma": "npm:^1.2.1" + "@napi-rs/tar": "npm:^0.0.1" + debug: "npm:^4.3.4" + peerDependencies: + "@napi-rs/cross-toolchain-arm64-target-aarch64": ^0.0.12 + "@napi-rs/cross-toolchain-arm64-target-armv7": ^0.0.12 + "@napi-rs/cross-toolchain-arm64-target-x86_64": ^0.0.12 + "@napi-rs/cross-toolchain-x64-target-aarch64": ^0.0.12 + "@napi-rs/cross-toolchain-x64-target-armv7": ^0.0.12 + "@napi-rs/cross-toolchain-x64-target-x86_64": ^0.0.12 + peerDependenciesMeta: + "@napi-rs/cross-toolchain-arm64-target-aarch64": + optional: true + "@napi-rs/cross-toolchain-arm64-target-armv7": + optional: true + "@napi-rs/cross-toolchain-arm64-target-x86_64": + optional: true + "@napi-rs/cross-toolchain-x64-target-aarch64": + optional: true + "@napi-rs/cross-toolchain-x64-target-armv7": + optional: true + "@napi-rs/cross-toolchain-x64-target-x86_64": + optional: true + checksum: de65d9ce93d9754d46322bd0ebda9c7e2bf63e1fa57777e843cac1f5a0c94ca57af208ff22fad15ef9cc9661575ffc1787e8d946c455dd04cecbb5c44f174655 + languageName: node + linkType: hard + +"@napi-rs/lzma-android-arm-eabi@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-android-arm-eabi@npm:1.2.1" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + +"@napi-rs/lzma-android-arm64@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-android-arm64@npm:1.2.1" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/lzma-darwin-arm64@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-darwin-arm64@npm:1.2.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/lzma-darwin-x64@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-darwin-x64@npm:1.2.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/lzma-freebsd-x64@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-freebsd-x64@npm:1.2.1" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/lzma-linux-arm-gnueabihf@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-linux-arm-gnueabihf@npm:1.2.1" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@napi-rs/lzma-linux-arm64-gnu@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-linux-arm64-gnu@npm:1.2.1" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"@napi-rs/lzma-linux-arm64-musl@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-linux-arm64-musl@npm:1.2.1" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"@napi-rs/lzma-linux-x64-gnu@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-linux-x64-gnu@npm:1.2.1" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"@napi-rs/lzma-linux-x64-musl@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-linux-x64-musl@npm:1.2.1" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"@napi-rs/lzma-win32-arm64-msvc@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-win32-arm64-msvc@npm:1.2.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/lzma-win32-ia32-msvc@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-win32-ia32-msvc@npm:1.2.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@napi-rs/lzma-win32-x64-msvc@npm:1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma-win32-x64-msvc@npm:1.2.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/lzma@npm:^1.2.1": + version: 1.2.1 + resolution: "@napi-rs/lzma@npm:1.2.1" + dependencies: + "@napi-rs/lzma-android-arm-eabi": "npm:1.2.1" + "@napi-rs/lzma-android-arm64": "npm:1.2.1" + "@napi-rs/lzma-darwin-arm64": "npm:1.2.1" + "@napi-rs/lzma-darwin-x64": "npm:1.2.1" + "@napi-rs/lzma-freebsd-x64": "npm:1.2.1" + "@napi-rs/lzma-linux-arm-gnueabihf": "npm:1.2.1" + "@napi-rs/lzma-linux-arm64-gnu": "npm:1.2.1" + "@napi-rs/lzma-linux-arm64-musl": "npm:1.2.1" + "@napi-rs/lzma-linux-x64-gnu": "npm:1.2.1" + "@napi-rs/lzma-linux-x64-musl": "npm:1.2.1" + "@napi-rs/lzma-win32-arm64-msvc": "npm:1.2.1" + "@napi-rs/lzma-win32-ia32-msvc": "npm:1.2.1" + "@napi-rs/lzma-win32-x64-msvc": "npm:1.2.1" + dependenciesMeta: + "@napi-rs/lzma-android-arm-eabi": + optional: true + "@napi-rs/lzma-android-arm64": + optional: true + "@napi-rs/lzma-darwin-arm64": + optional: true + "@napi-rs/lzma-darwin-x64": + optional: true + "@napi-rs/lzma-freebsd-x64": + optional: true + "@napi-rs/lzma-linux-arm-gnueabihf": + optional: true + "@napi-rs/lzma-linux-arm64-gnu": + optional: true + "@napi-rs/lzma-linux-arm64-musl": + optional: true + "@napi-rs/lzma-linux-x64-gnu": + optional: true + "@napi-rs/lzma-linux-x64-musl": + optional: true + "@napi-rs/lzma-win32-arm64-msvc": + optional: true + "@napi-rs/lzma-win32-ia32-msvc": + optional: true + "@napi-rs/lzma-win32-x64-msvc": + optional: true + checksum: 8fc856aa1f547a2ca5477afd4aa7079069d7fb0ce5da64ba4a3d5eb243210a6bb591f1f0fa9584f0f272952bd1d11b99eb097a1d2be9f919c8a9dcd1d8bceb2c + languageName: node + linkType: hard + +"@napi-rs/tar-android-arm-eabi@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-android-arm-eabi@npm:0.0.1" + conditions: os=android & cpu=arm + languageName: node + linkType: hard + +"@napi-rs/tar-android-arm64@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-android-arm64@npm:0.0.1" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/tar-darwin-arm64@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-darwin-arm64@npm:0.0.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/tar-darwin-x64@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-darwin-x64@npm:0.0.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/tar-freebsd-x64@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-freebsd-x64@npm:0.0.1" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/tar-linux-arm-gnueabihf@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-linux-arm-gnueabihf@npm:0.0.1" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@napi-rs/tar-linux-arm64-gnu@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-linux-arm64-gnu@npm:0.0.1" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"@napi-rs/tar-linux-arm64-musl@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-linux-arm64-musl@npm:0.0.1" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"@napi-rs/tar-linux-x64-gnu@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-linux-x64-gnu@npm:0.0.1" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"@napi-rs/tar-linux-x64-musl@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-linux-x64-musl@npm:0.0.1" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"@napi-rs/tar-win32-arm64-msvc@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-win32-arm64-msvc@npm:0.0.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@napi-rs/tar-win32-ia32-msvc@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-win32-ia32-msvc@npm:0.0.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@napi-rs/tar-win32-x64-msvc@npm:0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar-win32-x64-msvc@npm:0.0.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@napi-rs/tar@npm:^0.0.1": + version: 0.0.1 + resolution: "@napi-rs/tar@npm:0.0.1" + dependencies: + "@napi-rs/tar-android-arm-eabi": "npm:0.0.1" + "@napi-rs/tar-android-arm64": "npm:0.0.1" + "@napi-rs/tar-darwin-arm64": "npm:0.0.1" + "@napi-rs/tar-darwin-x64": "npm:0.0.1" + "@napi-rs/tar-freebsd-x64": "npm:0.0.1" + "@napi-rs/tar-linux-arm-gnueabihf": "npm:0.0.1" + "@napi-rs/tar-linux-arm64-gnu": "npm:0.0.1" + "@napi-rs/tar-linux-arm64-musl": "npm:0.0.1" + "@napi-rs/tar-linux-x64-gnu": "npm:0.0.1" + "@napi-rs/tar-linux-x64-musl": "npm:0.0.1" + "@napi-rs/tar-win32-arm64-msvc": "npm:0.0.1" + "@napi-rs/tar-win32-ia32-msvc": "npm:0.0.1" + "@napi-rs/tar-win32-x64-msvc": "npm:0.0.1" + dependenciesMeta: + "@napi-rs/tar-android-arm-eabi": + optional: true + "@napi-rs/tar-android-arm64": + optional: true + "@napi-rs/tar-darwin-arm64": + optional: true + "@napi-rs/tar-darwin-x64": + optional: true + "@napi-rs/tar-freebsd-x64": + optional: true + "@napi-rs/tar-linux-arm-gnueabihf": + optional: true + "@napi-rs/tar-linux-arm64-gnu": + optional: true + "@napi-rs/tar-linux-arm64-musl": + optional: true + "@napi-rs/tar-linux-x64-gnu": + optional: true + "@napi-rs/tar-linux-x64-musl": + optional: true + "@napi-rs/tar-win32-arm64-msvc": + optional: true + "@napi-rs/tar-win32-ia32-msvc": + optional: true + "@napi-rs/tar-win32-x64-msvc": + optional: true + checksum: f38bc820f20be8e6c7b95bb846e1bedd2ab8b09da4565b18b1ac5396c9e63d5c409b7774d687568b45602f84484436fd86e5891d98db2d6ff8f8b8597194e158 + languageName: node + linkType: hard + "@napi-rs/triples@workspace:triples": version: 0.0.0-use.local resolution: "@napi-rs/triples@workspace:triples"