From 28257b45c1a45917e4ffe4fcff3f93fb1566db0c Mon Sep 17 00:00:00 2001 From: LongYinan Date: Wed, 10 Jun 2020 18:55:40 +0800 Subject: [PATCH] feat(napi): support musl linux drop future executor due to mutithreads bug. --- .dockerignore | 3 + .github/workflows/docker.yaml | 27 +++++++ .github/workflows/linux-musl.yaml | 45 ++++++++++++ .github/workflows/linux.yaml | 15 ++-- .github/workflows/macos.yaml | 15 ++-- .github/workflows/windows.yaml | 15 ++-- Dockerfile.alpine | 13 ++++ napi/Cargo.toml | 3 - napi/src/executor.rs | 116 ------------------------------ napi/src/lib.rs | 45 ------------ napi/src/promise.rs | 65 ----------------- test_module/Cargo.toml | 1 - test_module/future.js | 26 ------- test_module/src/lib.rs | 26 ------- 14 files changed, 106 insertions(+), 309 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker.yaml create mode 100644 .github/workflows/linux-musl.yaml create mode 100644 Dockerfile.alpine delete mode 100644 napi/src/executor.rs delete mode 100644 napi/src/promise.rs delete mode 100644 test_module/future.js diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..06f102d3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +target +test_module/target +node_modules diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 00000000..a05517ae --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,27 @@ +name: Docker nightly build + +on: + schedule: + - cron: '0 1 * * *' + +jobs: + build_image: + name: Build rust-nodejs-alpine:lts + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + uses: actions/docker/login@master + env: + DOCKER_REGISTRY_URL: docker.pkg.github.com + DOCKER_USERNAME: ${{ github.actor }} + DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: Build docker image + run: | + docker build . -f Dockerfile.alpine --pull --no-cache -t docker.pkg.github.com/napi-rs/napi-rs/rust-nodejs-alpine:lts + + - name: Push docker image + run: docker push docker.pkg.github.com/napi-rs/napi-rs/rust-nodejs-alpine:lts diff --git a/.github/workflows/linux-musl.yaml b/.github/workflows/linux-musl.yaml new file mode 100644 index 00000000..8a7ad147 --- /dev/null +++ b/.github/workflows/linux-musl.yaml @@ -0,0 +1,45 @@ +name: Musl + +on: [push, pull_request] + +jobs: + build: + name: stable - x86_64-unknown-linux-musl - node@${{ matrix.node }} + strategy: + fail-fast: false + matrix: + node: ['10', '12', '13', '14'] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Log in to registry + run: | + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD $DOCKER_REGISTRY_URL + env: + DOCKER_REGISTRY_URL: docker.pkg.github.com + DOCKER_USERNAME: ${{ github.actor }} + DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull docker image + run: | + docker pull docker.pkg.github.com/napi-rs/napi-rs/rust-nodejs-alpine:lts + docker tag docker.pkg.github.com/napi-rs/napi-rs/rust-nodejs-alpine:lts builder + + - name: Run check + run: | + docker run --rm -v $(pwd)/.cargo:/root/.cargo -v $(pwd):/napi-rs -w /napi-rs builder cargo check --all --bins --examples --tests -vvv + + - name: Run tests + run: | + docker run --rm -v $(pwd)/.cargo:/root/.cargo -v $(pwd):/napi-rs -w /napi-rs builder cargo test -p napi-rs --lib -- --nocapture + + - name: Build native module + run: | + docker run --rm -v $(pwd)/.cargo:/root/.cargo -v $(pwd):/napi-rs -w /napi-rs builder sh -c "yarn && cd test_module && yarn build" + env: + RUST_BACKTRACE: 1 + + - name: Fuzzy + run: docker run --rm -v $(pwd):/napi-rs -w /napi-rs/test_module node:${{ matrix.node }}-alpine node fuzzy.js diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 65639baf..c981d426 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -8,11 +8,8 @@ jobs: fail-fast: false matrix: node: ['10', '12', '13', '14'] - version: - - stable - - nightly - name: ${{ matrix.version }} - x86_64-unknown-linux-gnu - node@${{ matrix.node }} + name: stable - x86_64-unknown-linux-gnu - node@${{ matrix.node }} runs-on: ubuntu-latest steps: @@ -23,10 +20,10 @@ jobs: with: node-version: ${{ matrix.node }} - - name: Install ${{ matrix.version }} + - name: Install stable uses: actions-rs/toolchain@v1 with: - toolchain: ${{ matrix.version }}-x86_64-unknown-linux-gnu + toolchain: stable-x86_64-unknown-linux-gnu profile: minimal override: true @@ -38,17 +35,17 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ matrix.version }}-x86_64-unknown-linux-gnu-cargo-registry-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-x86_64-unknown-linux-gnu-cargo-registry-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ matrix.version }}-x86_64-unknown-linux-gnu-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-x86_64-unknown-linux-gnu-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo build uses: actions/cache@v1 with: path: target - key: ${{ matrix.version }}-x86_64-unknown-linux-gnu-cargo-build-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-x86_64-unknown-linux-gnu-cargo-build-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Check build uses: actions-rs/cargo@v1 diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index c6bdb536..67e77f06 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -8,11 +8,8 @@ jobs: fail-fast: false matrix: node: ['10', '12', '13', '14'] - version: - - stable - - nightly - name: ${{ matrix.version }} - x86_64-apple-darwin - node@${{ matrix.node }} + name: stable - x86_64-apple-darwin - node@${{ matrix.node }} runs-on: macos-latest steps: @@ -23,10 +20,10 @@ jobs: with: node-version: ${{ matrix.node }} - - name: Install ${{ matrix.version }} + - name: Install stable uses: actions-rs/toolchain@v1 with: - toolchain: ${{ matrix.version }}-x86_64-apple-darwin + toolchain: stable-x86_64-apple-darwin profile: minimal override: true @@ -38,17 +35,17 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ matrix.version }}-x86_64-apple-darwin-cargo-registry-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-x86_64-apple-darwin-cargo-registry-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ matrix.version }}-x86_64-apple-darwin-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-x86_64-apple-darwin-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo build uses: actions/cache@v1 with: path: target - key: ${{ matrix.version }}-x86_64-apple-darwin-cargo-build-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-x86_64-apple-darwin-cargo-build-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Check build uses: actions-rs/cargo@v1 diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index e75f6686..29e3bb97 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -8,13 +8,10 @@ jobs: fail-fast: false matrix: node: ['10', '12', '13', '14'] - version: - - stable - - nightly target: - x86_64-pc-windows-msvc - name: ${{ matrix.version }} - ${{ matrix.target }} - node@${{ matrix.node }} + name: stable - ${{ matrix.target }} - node@${{ matrix.node }} runs-on: windows-latest steps: @@ -24,10 +21,10 @@ jobs: uses: actions/setup-node@v1 with: node-version: ${{ matrix.node }} - - name: Install ${{ matrix.version }} + - name: Install stable uses: actions-rs/toolchain@v1 with: - toolchain: ${{ matrix.version }}-${{ matrix.target }} + toolchain: stable-${{ matrix.target }} profile: minimal override: true - name: Install llvm @@ -44,17 +41,17 @@ jobs: uses: actions/cache@v1 with: path: ~/.cargo/registry - key: ${{ matrix.version }}-${{ matrix.target }}-cargo-registry-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-${{ matrix.target }}-cargo-registry-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo index uses: actions/cache@v1 with: path: ~/.cargo/git - key: ${{ matrix.version }}-${{ matrix.target }}-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-${{ matrix.target }}-cargo-index-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo build uses: actions/cache@v1 with: path: target - key: ${{ matrix.version }}-${{ matrix.target }}-cargo-build-trimmed-${{ hashFiles('**/Cargo.lock') }} + key: stable-${{ matrix.target }}-cargo-build-trimmed-${{ hashFiles('**/Cargo.lock') }} - name: Check build uses: actions-rs/cargo@v1 diff --git a/Dockerfile.alpine b/Dockerfile.alpine new file mode 100644 index 00000000..650b574b --- /dev/null +++ b/Dockerfile.alpine @@ -0,0 +1,13 @@ +FROM rust:alpine + +ENV RUSTFLAGS="-C target-feature=-crt-static" + +RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && \ + apk update && \ + apk add nodejs yarn clang musl-dev llvm-dev curl && \ + export NODE_VERSION=$(node -e "console.log(process.version)") && \ + curl -fsSLO $(node -e "console.log(process.release.headersUrl)") && \ + tar -xf "node-$NODE_VERSION-headers.tar.gz" && \ + mv "node-$NODE_VERSION/include/node" include && \ + rm -rf "node-$NODE_VERSION" && \ + rm "node-$NODE_VERSION-headers.tar.gz" \ No newline at end of file diff --git a/napi/Cargo.toml b/napi/Cargo.toml index 33c7f939..951f9862 100644 --- a/napi/Cargo.toml +++ b/napi/Cargo.toml @@ -9,9 +9,6 @@ repository = "https://github.com/Brooooooklyn/napi-rs" keywords = ["NodeJS", "FFI", "NAPI", "n-api"] edition = "2018" -[dependencies] -futures = { version = "0.3", features = ["default", "thread-pool"] } - [target.'cfg(windows)'.build-dependencies] flate2 = "1.0" reqwest = { version = "0.10", features = ["native-tls", "blocking"] } diff --git a/napi/src/executor.rs b/napi/src/executor.rs deleted file mode 100644 index b2ce3064..00000000 --- a/napi/src/executor.rs +++ /dev/null @@ -1,116 +0,0 @@ -extern crate alloc; - -use alloc::alloc::{alloc, alloc_zeroed, Layout}; -use futures::future::LocalBoxFuture; -use futures::task::{waker, ArcWake, Context, Poll}; -use std::future::Future; -use std::os::raw::c_void; -use std::pin::Pin; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; - -use crate::{sys, Error, Result, Status}; - -struct Task<'a> { - future: LocalBoxFuture<'a, ()>, - context: Context<'a>, - is_polling: AtomicBool, -} - -struct UvWaker(*mut sys::uv_async_t); - -unsafe impl Send for UvWaker {} -unsafe impl Sync for UvWaker {} - -impl UvWaker { - fn new(event_loop: *mut sys::uv_loop_s) -> Result { - let uv_async_t = unsafe { - let layout = Layout::new::(); - debug_assert!(layout.size() != 0, "uv_async_t alloc size should not be 0"); - if cfg!(windows) { - alloc_zeroed(layout) as *mut sys::uv_async_t - } else { - alloc(layout) as *mut sys::uv_async_t - } - }; - unsafe { - let status = sys::uv_async_init(event_loop, uv_async_t, Some(poll_future)); - if status != 0 { - return Err(Error { - status: Status::Unknown, - reason: Some("Non-zero status returned from uv_async_init".to_owned()), - }); - } - }; - Ok(UvWaker(uv_async_t)) - } - - #[inline] - fn assign_task(&self, mut task: Task) { - if !task.poll_future() { - task.is_polling.store(false, Ordering::Relaxed); - let arc_task = Arc::new(task); - unsafe { - sys::uv_handle_set_data( - self.0 as *mut sys::uv_handle_t, - Arc::into_raw(arc_task) as *mut c_void, - ) - }; - } else { - unsafe { sys::uv_close(self.0 as *mut sys::uv_handle_t, None) }; - }; - } -} - -impl ArcWake for UvWaker { - fn wake_by_ref(arc_self: &Arc) { - let status = unsafe { sys::uv_async_send(arc_self.0) }; - assert!(status == 0, "wake_uv_async_by_ref failed"); - } -} - -#[inline] -pub fn execute(event_loop: *mut sys::uv_loop_s, future: LocalBoxFuture<()>) -> Result<()> { - let uv_waker = UvWaker::new(event_loop)?; - let arc_waker = Arc::new(uv_waker); - let waker_to_poll = Arc::clone(&arc_waker); - let waker = waker(arc_waker); - let context = Context::from_waker(&waker); - let task = Task { - future, - context, - is_polling: AtomicBool::from(false), - }; - waker_to_poll.assign_task(task); - Ok(()) -} - -impl<'a> Task<'a> { - fn poll_future(&mut self) -> bool { - if self.is_polling.load(Ordering::Relaxed) { - return false; - } - self.is_polling.store(true, Ordering::Relaxed); - let mut pinned = Pin::new(&mut self.future); - let fut_mut = pinned.as_mut(); - match fut_mut.poll(&mut self.context) { - Poll::Ready(_) => true, - Poll::Pending => false, - } - } -} - -unsafe extern "C" fn poll_future(handle: *mut sys::uv_async_t) { - let data_ptr = sys::uv_handle_get_data(handle as *mut sys::uv_handle_t) as *mut Task; - let mut task = Arc::from_raw(data_ptr); - if let Some(mut_task) = Arc::get_mut(&mut task) { - if mut_task.poll_future() { - sys::uv_close(handle as *mut sys::uv_handle_t, None); - } else { - mut_task.is_polling.store(false, Ordering::Relaxed); - Arc::into_raw(task); - }; - } else { - Arc::into_raw(task); - } -} diff --git a/napi/src/lib.rs b/napi/src/lib.rs index 67bc73f8..05fa35a1 100644 --- a/napi/src/lib.rs +++ b/napi/src/lib.rs @@ -1,6 +1,5 @@ use async_work::AsyncWork; use core::fmt::Debug; -use futures::prelude::*; use std::any::TypeId; use std::convert::{TryFrom, TryInto}; use std::ffi::CString; @@ -15,8 +14,6 @@ use std::string::String as RustString; mod async_work; mod call_context; -mod executor; -mod promise; pub mod sys; mod task; mod version; @@ -549,48 +546,6 @@ impl Env { Ok(Value::from_raw_value(self, result, Object)) } - #[inline] - pub fn execute< - T: 'static, - V: 'static + ValueType, - F: 'static + Future>, - R: 'static + FnOnce(&mut Env, T) -> Result>, - >( - &self, - deferred: F, - resolver: R, - ) -> Result> { - let mut raw_promise = ptr::null_mut(); - let mut raw_deferred = ptr::null_mut(); - - unsafe { - let status = sys::napi_create_promise(self.0, &mut raw_deferred, &mut raw_promise); - check_status(status)?; - } - - let event_loop = unsafe { sys::uv_default_loop() }; - let raw_env = self.0; - let future_to_execute = - promise::resolve(self.0, deferred, resolver, raw_deferred).map(move |v| match v { - Ok(value) => value, - Err(e) => { - let cloned_error = e.clone(); - unsafe { - sys::napi_throw_error( - raw_env, - ptr::null(), - e.reason.unwrap_or(format!("{:?}", e.status)).as_ptr() as *const _, - ); - }; - eprintln!("{:?}", &cloned_error); - panic!(cloned_error); - } - }); - executor::execute(event_loop, Box::pin(future_to_execute))?; - - Ok(Value::from_raw_value(self, raw_promise, Object)) - } - pub fn spawn(&self, task: T) -> Result> { let mut raw_promise = ptr::null_mut(); let mut raw_deferred = ptr::null_mut(); diff --git a/napi/src/promise.rs b/napi/src/promise.rs deleted file mode 100644 index d99c35a7..00000000 --- a/napi/src/promise.rs +++ /dev/null @@ -1,65 +0,0 @@ -use futures::prelude::*; -use std::os::raw::c_char; -use std::ptr; - -use crate::{check_status, sys, Env, Result, Value, ValueType}; - -#[inline] -pub async fn resolve< - T, - V: ValueType, - R: FnOnce(&mut Env, T) -> Result>, - F: Future>, ->( - env: sys::napi_env, - fut: F, - resolver: R, - raw_deferred: sys::napi_deferred, -) -> Result<()> { - let mut raw_resource = ptr::null_mut(); - let status = unsafe { sys::napi_create_object(env, &mut raw_resource) }; - check_status(status)?; - let mut raw_name = ptr::null_mut(); - let s = "napi_async_context"; - let status = unsafe { - sys::napi_create_string_utf8( - env, - s.as_ptr() as *const c_char, - s.len() as u64, - &mut raw_name, - ) - }; - check_status(status)?; - let mut raw_context = ptr::null_mut(); - unsafe { - let status = sys::napi_async_init(env, raw_resource, raw_name, &mut raw_context); - check_status(status)?; - } - let mut handle_scope = ptr::null_mut(); - match fut.await { - Ok(v) => unsafe { - check_status(sys::napi_open_handle_scope(env, &mut handle_scope))?; - let mut tmp_env = Env::from_raw(env); - let js_value = resolver(&mut tmp_env, v)?; - check_status(sys::napi_resolve_deferred( - env, - raw_deferred, - js_value.raw_value, - ))?; - check_status(sys::napi_close_handle_scope(env, handle_scope))?; - }, - Err(e) => unsafe { - check_status(sys::napi_open_handle_scope(env, &mut handle_scope))?; - check_status(sys::napi_reject_deferred( - env, - raw_deferred, - Env::from_raw(env) - .create_error(e) - .map(|e| e.into_raw()) - .unwrap_or(ptr::null_mut()), - ))?; - check_status(sys::napi_close_handle_scope(env, handle_scope))?; - }, - }; - Ok(()) -} diff --git a/test_module/Cargo.toml b/test_module/Cargo.toml index cc669935..81a7e16f 100644 --- a/test_module/Cargo.toml +++ b/test_module/Cargo.toml @@ -8,7 +8,6 @@ edition = "2018" crate-type = ["cdylib"] [dependencies] -futures = "0.3" napi-rs = { path = "../napi" } napi-rs-derive = { path = "../napi-derive" } diff --git a/test_module/future.js b/test_module/future.js deleted file mode 100644 index f96b98c7..00000000 --- a/test_module/future.js +++ /dev/null @@ -1,26 +0,0 @@ -const testModule = require('./index.node') - -function testSpawn() { - console.log('=== Test spawning a future on libuv event loop') - return testModule.testSpawn() -} - -function testThrow() { - console.log('=== Test throwing from Rust') - try { - testModule.testThrow() - console.log('Expected function to throw an error') - process.exit(1) - } catch (e) { - console.log(e) - } -} -testSpawn() - .then((value) => { - console.info(`${value} from napi`) - testThrow() - }) - .catch((e) => { - console.error(e) - process.exit(1) - }) diff --git a/test_module/src/lib.rs b/test_module/src/lib.rs index de7adc3a..d0d86b1c 100644 --- a/test_module/src/lib.rs +++ b/test_module/src/lib.rs @@ -3,15 +3,12 @@ extern crate napi_rs as napi; #[macro_use] extern crate napi_rs_derive; -extern crate futures; - use napi::{Any, CallContext, Env, Error, Number, Object, Result, Status, Task, Value}; use std::convert::TryInto; register_module!(test_module, init); fn init(env: &Env, exports: &mut Value) -> Result<()> { - exports.set_named_property("testSpawn", env.create_function("testSpawn", test_spawn)?)?; exports.set_named_property("testThrow", env.create_function("testThrow", test_throw)?)?; exports.set_named_property( "testSpawnThread", @@ -20,29 +17,6 @@ fn init(env: &Env, exports: &mut Value) -> Result<()> { Ok(()) } -#[js_function] -fn test_spawn(ctx: CallContext) -> Result> { - use futures::executor::ThreadPool; - use futures::StreamExt; - let env = ctx.env; - let pool = ThreadPool::new().expect("Failed to build pool"); - let (tx, rx) = futures::channel::mpsc::unbounded::(); - let fut_values = async move { - let fut_tx_result = async move { - (0..200).for_each(|v| { - tx.unbounded_send(v).expect("Failed to send"); - }) - }; - pool.spawn_ok(fut_tx_result); - let fut = rx.map(|v| v * 2).collect::>(); - let results = fut.await; - println!("Collected result lenght {}", results.len()); - Ok(results.len() as u32) - }; - - env.execute(fut_values, |&mut env, len| env.create_uint32(len)) -} - struct ComputeFib { n: u32, }