Merge pull request #1006 from napi-rs/zig-cross-enhance

Zig cross enhance

.github/workflows/linux-aarch64-zig.yaml

@@ -1,69 +0,0 @@
-name: Linux-aarch64-zig
-
-env:
-  DEBUG: 'napi:*'
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-
-jobs:
-  build:
-    name: stable - aarch64-unknown-linux-gnu - node@16
-    runs-on: ubuntu-latest
-
-    steps:
-      - run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
-
-      - uses: actions/checkout@v2
-
-      - name: Setup node
-        uses: actions/setup-node@v2
-        with:
-          node-version: 16
-          check-latest: true
-          cache: 'yarn'
-
-      - name: Install
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          override: true
-          target: aarch64-unknown-linux-musl
-
-      - name: Install aarch64 toolchain
-        run: rustup target add aarch64-unknown-linux-gnu
-
-      - name: Install ziglang
-        uses: goto-bus-stop/setup-zig@v1
-        with:
-          version: 0.9.0
-
-      - name: Cache NPM dependencies
-        uses: actions/cache@v2
-        with:
-          path: node_modules
-          key: npm-cache-linux-aarch64-gnu-node@16-${{ hashFiles('yarn.lock') }}
-
-      - name: Install dependencies
-        run: yarn install --frozen-lockfile --ignore-platform --registry https://registry.npmjs.org --network-timeout 300000
-
-      - name: 'Build TypeScript'
-        run: yarn build
-
-      - name: Cross build native tests
-        run: |
-          yarn --cwd ./examples/napi-compat-mode build --target aarch64-unknown-linux-musl --zig
-          yarn --cwd ./examples/napi build --target aarch64-unknown-linux-musl --zig
-
-      - name: Setup and run tests
-        uses: docker://multiarch/alpine:aarch64-latest-stable
-        with:
-          args: >
-            sh -c "
-              apk add nodejs yarn && \
-              yarn test
-            "

.github/workflows/zig.yaml

@@ -0,0 +1,139 @@
+name: Zig-Cross-Compile
+
+env:
+  DEBUG: 'napi:*'
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    name: Zig-Cross-Compile-On-Linux
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          [
+            'x86_64-apple-darwin',
+            'x86_64-unknown-linux-musl',
+            'aarch64-unknown-linux-musl',
+          ]
+
+    steps:
+      - run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+      - uses: actions/checkout@v2
+      - name: Setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16
+          check-latest: true
+          cache: 'yarn'
+      - name: Install
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+          target: ${{ matrix.target }}
+      - name: Install aarch64 toolchain
+        run: rustup target add aarch64-unknown-linux-gnu
+      - name: Install ziglang
+        uses: goto-bus-stop/setup-zig@v1
+        with:
+          version: 0.9.0
+      - name: Cache NPM dependencies
+        uses: actions/cache@v2
+        with:
+          path: node_modules
+          key: npm-cache-linux-aarch64-gnu-node@16-${{ hashFiles('yarn.lock') }}
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile --ignore-platform --registry https://registry.npmjs.org --network-timeout 300000
+      - name: 'Build TypeScript'
+        run: yarn build
+      - name: Cross build native tests
+        run: |
+          yarn --cwd ./examples/napi-compat-mode build --target ${{ matrix.target }} --zig
+          yarn --cwd ./examples/napi build --target ${{ matrix.target }} --zig
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: compat-${{ matrix.target }}
+          path: ./examples/napi-compat-mode/index.node
+          if-no-files-found: error
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: napi-${{ matrix.target }}
+          path: ./examples/napi/index.node
+          if-no-files-found: error
+
+  test:
+    name: Test Zig Cross Compiled ${{ matrix.settings.target }}
+    runs-on: ${{ matrix.settings.host }}
+    needs:
+      - build
+    strategy:
+      fail-fast: false
+      matrix:
+        settings:
+          - host: ubuntu-latest
+            target: x86_64-unknown-linux-musl
+          - host: macos-latest
+            target: x86_64-apple-darwin
+          - host: ubuntu-latest
+            target: aarch64-unknown-linux-musl
+
+    steps:
+      - run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+        if: matrix.settings.host == 'ubuntu-latest'
+      - uses: actions/checkout@v2
+      - name: Setup node
+        uses: actions/setup-node@v2
+        with:
+          node-version: 16
+          check-latest: true
+          cache: 'yarn'
+      - name: Cache NPM dependencies
+        uses: actions/cache@v2
+        with:
+          path: node_modules
+          key: npm-cache-${{ matrix.settings.host }}-node@16-${{ hashFiles('yarn.lock') }}
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile --ignore-platform --registry https://registry.npmjs.org --network-timeout 300000
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: napi-${{ matrix.settings.target }}
+          path: ./examples/napi/
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: compat-${{ matrix.settings.target }}
+          path: ./examples/napi-compat-mode/
+      - name: List files
+        run: |
+          ls ./examples/napi
+          ls ./examples/napi-compat-mode
+      - name: Test
+        run: yarn test --verbose
+        if: matrix.settings.host == 'macos-latest'
+      - name: Test
+        uses: docker://multiarch/alpine:aarch64-latest-stable
+        if: matrix.settings.target == 'aarch64-unknown-linux-musl'
+        with:
+          args: >
+            sh -c "
+              apk add nodejs yarn && \
+              yarn test
+            "
+      - name: Test
+        uses: addnab/docker-run-action@v3
+        if: matrix.settings.target == 'x86_64-unknown-linux-musl'
+        with:
+          image: ghcr.io/${{ github.repository }}/nodejs-rust:lts-alpine
+          options: -v ${{ github.workspace }}:/napi-rs -w /napi-rs
+          run: yarn test

cli/src/build.ts

@@ -1,5 +1,5 @@
 import { execSync } from 'child_process'
-import { existsSync, mkdirSync, writeFileSync } from 'fs'
+import { existsSync, mkdirSync } from 'fs'
 import { join, parse, sep } from 'path'
 
 import { Instance } from 'chalk'
@@ -11,7 +11,7 @@ import toml from 'toml'
 import { getNapiConfig } from './consts'
 import { debugFactory } from './debug'
 import { createJsBinding } from './js-binding-template'
-import { getCpuArch, getDefaultTargetTriple, parseTriple } from './parse-triple'
+import { getDefaultTargetTriple, parseTriple } from './parse-triple'
 import {
   copyFileAsync,
   mkdirAsync,
@@ -23,6 +23,36 @@ import {
 const debug = debugFactory('build')
 const chalk = new Instance({ level: 1 })
 
+const ZIG_PLATFORM_TARGET_MAP = {
+  'x86_64-unknown-linux-musl': 'x86_64-linux-musl',
+  'x86_64-unknown-linux-gnu': 'x86_64-linux-gnu',
+  // Doesn't support Windows MSVC for now
+  // 'x86_64-pc-windows-gnu': 'x86_64-windows-gnu',
+  // https://github.com/ziglang/zig/issues/1759
+  // 'x86_64-unknown-freebsd': 'x86_64-freebsd',
+  'x86_64-apple-darwin': 'x86_64-macos-gnu',
+  'aarch64-apple-darwin': 'aarch64-macos-gnu',
+  'aarch64-unknown-linux-gnu': 'aarch64-linux-gnu',
+  'aarch64-unknown-linux-musl': 'aarch64-linux-musl',
+}
+
+function processZigLinkerArgs(platform: string, args: string[]) {
+  if (platform.includes('apple')) {
+    const newArgs = args.filter(
+      (arg) =>
+        !arg.startsWith('-Wl,-exported_symbols_list') &&
+        arg !== '-Wl,-dylib' &&
+        arg !== '-liconv',
+    )
+    newArgs.push('-Wl,"-undefined=dynamic_lookup"', '-dead_strip')
+    return newArgs
+  }
+  if (platform.includes('linux')) {
+    return args.filter((arg) => arg !== '-lgcc_s' && arg !== '-lunwind')
+  }
+  return args
+}
+
 export class BuildCommand extends Command {
   static usage = Command.Usage({
     description: 'Build and copy native module into specified dir',
@@ -119,7 +149,9 @@ export class BuildCommand extends Command {
   })
 
   useZig = Option.Boolean(`--zig`, false, {
-    description: `Use ${chalk.green('zig')} as linker`,
+    description: `Use ${chalk.green('zig')} as linker ${chalk.yellowBright(
+      '(Experimental)',
+    )}`,
   })
 
   async execute() {
@@ -127,6 +159,7 @@ export class BuildCommand extends Command {
       ? join(process.cwd(), this.cargoCwd)
       : process.cwd()
     const releaseFlag = this.isRelease ? `--release` : ''
+
     const targetFlag = this.targetTripleDir
       ? `--target ${this.targetTripleDir}`
       : ''
@@ -174,23 +207,50 @@ export class BuildCommand extends Command {
       }
     }
 
-    if (this.useZig && triple.platform === 'linux') {
+    if (this.useZig) {
+      const zigTarget = ZIG_PLATFORM_TARGET_MAP[triple.raw]
+      if (!zigTarget) {
+        throw new Error(`${triple.raw} can not be cross compiled by zig`)
+      }
       const paths = envPaths('napi-rs')
-      const cpuArch = getCpuArch(triple.arch)
-      const linkerWrapper = join(paths.cache, `zig-cc-${triple.abi}.sh`)
-      const zigTarget = `${cpuArch}-linux-${triple.abi}`
+      const linkerWrapperShell = join(
+        paths.cache,
+        `zig-cc-${triple.raw}.${process.platform === 'win32' ? 'bat' : 'sh'}`,
+      )
+      const linkerWrapper = join(paths.cache, `zig-cc-${triple.raw}.js`)
       mkdirSync(paths.cache, { recursive: true })
-      writeFileSync(
+      const forwardArgs = process.platform === 'win32' ? '%*' : '$@'
+      await writeFileAsync(
+        linkerWrapperShell,
+        `node ${linkerWrapper} ${forwardArgs}`,
+        {
+          mode: '777',
+        },
+      )
+      await writeFileAsync(
         linkerWrapper,
-        `#!/bin/bash\nzig cc \${@/-lgcc_s/-lunwind} -target ${zigTarget}\n`,
-        { mode: 0o700 },
+        `#!/usr/bin/env node\nconst{writeFileSync} = require('fs')\n${processZigLinkerArgs.toString()}\nconst {status} = require('child_process').spawnSync('zig', ['${
+          triple.platform === 'win32' ? 'c++' : 'cc'
+        }', ...processZigLinkerArgs('${
+          triple.raw
+        }', process.argv.slice(2)), '-target', '${
+          ZIG_PLATFORM_TARGET_MAP[triple.raw]
+        }'], { stdio: 'inherit', shell: true })\nwriteFileSync('${linkerWrapper.replaceAll(
+          '\\',
+          '/',
+        )}.args.log', process.argv.slice(2).join(' '))\n\nprocess.exit(status || 0)\n`,
+        {
+          mode: '777',
+        },
       )
       const envTarget = triple.raw.replaceAll('-', '_').toUpperCase()
       Object.assign(additionalEnv, {
-        TARGET_CC: linkerWrapper,
+        CC: `zig cc -target ${zigTarget}`,
+        CXX: `zig c++ -target ${zigTarget}`,
+        TARGET_CC: `zig cc -target ${zigTarget}`,
         TARGET_CXX: `zig c++ -target ${zigTarget}`,
       })
-      additionalEnv[`CARGO_TARGET_${envTarget}_LINKER`] = linkerWrapper
+      additionalEnv[`CARGO_TARGET_${envTarget}_LINKER`] = linkerWrapperShell
     }
 
     execSync(cargoCommand, {

cli/src/parse-triple.ts

@@ -21,14 +21,6 @@ const CpuToNodeArch: { [index: string]: NodeJSArch } = {
   armv7: 'arm',
 }
 
-const NodeArchToCpu: { [index: string]: string } = {
-  arm64: 'aarch64',
-  ppc: 'powerpc',
-  ppc64: 'powerpc64',
-  x32: 'i686',
-  x64: 'x86_64',
-}
-
 const SysToNodePlatform: { [index: string]: NodeJS.Platform } = {
   linux: 'linux',
   freebsd: 'freebsd',
@@ -133,7 +125,3 @@ export function getDefaultTargetTriple(rustcfg: string): PlatformDetail {
   }
   return parseTriple(triple)
 }
-
-export function getCpuArch(nodeArch: NodeJSArch): string {
-  return NodeArchToCpu[nodeArch] || nodeArch
-}