naskya/napi-rs
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(cli): allow sync fs operation between workers/mainThread (#2064)

Browse source 
* feat(cli): allow sync fs operation between workers/mainThread

* allow sync fs operation between workers/mainThread (#2065)

* Fix

* Update fixture

* flaky test

* Fix cross compile target

* Update zig

* macos-cross test was filtered

---------

Co-authored-by: Toyo Li <lifenglin314@outlook.com>
This commit is contained in:
LongYinan 2024-04-23 12:14:06 +08:00 committed by GitHub
parent 9b6361afc3
commit 46cbcf3ff1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 424 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
renovate.json → .github/renovate.json vendored
View file

@ -4,7 +4,6 @@
"**/node_modules/**",
"**/bower_components/**",
"**/vendor/**",
"**/examples/**",
"**/__tests__/**",
"**/test/**",
"**/__fixtures__/**"

5
.github/workflows/test-release.yaml vendored
View file

@ -388,13 +388,11 @@ jobs:
args: '--platform linux/ppc64le'
arch: 'ppc64'
libc: 'gnu'
without-lerna: true
- image: 'node:{:version}-slim'
target: s390x-unknown-linux-gnu
args: '--platform linux/s390x'
arch: 's390x'
libc: 'gnu'
without-lerna: true
- image: 'node:{:version}-alpine'
target: x86_64-unknown-linux-musl
args: ''
@ -444,8 +442,7 @@ jobs:
with:
image: ${{ steps.image-name.outputs.docker-image }}
options: ${{ matrix.settings.args }} -v ${{ github.workspace }}/cores:/cores -v ${{ github.workspace }}:/build -w /build
run: >-
${{ matrix.settings.without-lerna && 'yarn test:without-lerna' || 'yarn test' }}
run: yarn test
- name: List files
run: |
ls -la .

12
.github/workflows/zig.yaml vendored
View file

@ -23,7 +23,7 @@ jobs:
matrix:
target:
[
'x86_64-apple-darwin',
'aarch64-apple-darwin',
'x86_64-unknown-linux-musl',
'aarch64-unknown-linux-musl',
'armv7-unknown-linux-musleabihf',
@ -53,7 +53,7 @@ jobs:
- name: Install ziglang
uses: goto-bus-stop/setup-zig@v2
with:
version: 0.11.0
version: 0.12.0
- name: Install dependencies
run: yarn install --immutable --mode=skip-build
- name: install MacOS SDK
@ -89,8 +89,8 @@ jobs:
settings:
- host: ubuntu-latest
target: x86_64-unknown-linux-musl
- host: macos-latest
target: x86_64-apple-darwin
- host: macos-14
target: aarch64-apple-darwin
- host: ubuntu-latest
target: aarch64-unknown-linux-musl
docker-platform: arm64
@ -128,7 +128,7 @@ jobs:
run: yarn test --verbose
env:
SKIP_UNWIND_TEST: 1
if: matrix.settings.host == 'macos-latest'
if: matrix.settings.host == 'macos-14'
- run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
if: matrix.settings.host == 'ubuntu-latest'
- name: Test
@ -149,7 +149,7 @@ jobs:
options: --platform linux/${{ matrix.settings.docker-platform }} -v ${{ github.workspace }}:/build -w /build
run: |
set -e
yarn test:without-lerna
yarn test
- name: Test
uses: addnab/docker-run-action@v3
if: matrix.settings.target == 'x86_64-unknown-linux-musl'

5
Cargo.toml
View file

@ -19,6 +19,11 @@ exclude = ["./testing"]
[profile.release]
lto = true
[profile.wasi]
inherits = "release"
opt-level = "z"
debug = 'full'
[profile.napi-rs-custom]
inherits = "dev"
codegen-units = 1024

2
alpine-zig.Dockerfile
View file

@ -1,6 +1,6 @@
FROM ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
ARG ZIG_VERSION=0.11.0
ARG ZIG_VERSION=0.12.0
RUN apk add --update --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing xz xz-dev && \
rustup target add x86_64-unknown-linux-gnu && \

4
cli/src/api/artifacts.ts
View file

@ -31,8 +31,8 @@ export async function collectArtifacts(userOptions: ArtifactsOptions) {
const universalSourceBins = new Set(
targets
.filter((platform) => platform.arch === 'universal')
.flatMap(
(p) => UniArchsByPlatform[p.platform]?.map((a) => `${p.platform}-${a}`),
.flatMap((p) =>
UniArchsByPlatform[p.platform]?.map((a) => `${p.platform}-${a}`),
)
.filter(Boolean) as string[],
)

2
cli/src/api/templates/ci-template.ts
View file

@ -136,7 +136,7 @@ jobs:
- uses: goto-bus-stop/setup-zig@v2
if: \${{ contains(matrix.settings.target, 'musl') }}
with:
version: 0.11.0
version: 0.12.0
- name: Setup toolchain
run: \${{ matrix.settings.setup }}

16
cli/src/api/templates/load-wasi-template.ts
View file

@ -22,10 +22,15 @@ const __wasi = new __WASI({
version: 'preview1',
})`
const workerFsHandler = fs
? `\n worker.addEventListener('message', __wasmCreateOnMessageForFsProxy(__fs))\n`
: ''
return `import {
instantiateNapiModuleSync as __emnapiInstantiateNapiModuleSync,
getDefaultContext as __emnapiGetDefaultContext,
WASI as __WASI,
createOnMessage as __wasmCreateOnMessageForFsProxy,
} from '@napi-rs/wasm-runtime'
${fsImport}
import __wasmUrl from './${wasiFilename}.wasm?url'
@ -50,9 +55,11 @@ const {
asyncWorkPoolSize: 4,
wasi: __wasi,
onCreateWorker() {
return new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
const worker = new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
type: 'module',
})
${workerFsHandler}
return worker
},
overwriteImports(importObject) {
importObject.env = {
@ -95,6 +102,7 @@ const { Worker } = require('node:worker_threads')
const {
instantiateNapiModuleSync: __emnapiInstantiateNapiModuleSync,
getDefaultContext: __emnapiGetDefaultContext,
createOnMessage: __wasmCreateOnMessageForFsProxy,
} = require('@napi-rs/wasm-runtime')
const __rootDir = __nodePath.parse(process.cwd()).root
@ -141,10 +149,14 @@ const { instance: __napiInstance, module: __wasiModule, napiModule: __napiModule
})(),
wasi: __wasi,
onCreateWorker() {
return new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
const worker = new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
env: process.env,
execArgv: ['--experimental-wasi-unstable-preview1'],
})
worker.onmessage = ({ data }) => {
__wasmCreateOnMessageForFsProxy(__nodeFs)(data)
}
return worker
},
overwriteImports(importObject) {
importObject.env = {

20
cli/src/api/templates/wasi-worker-template.ts
View file

@ -1,5 +1,6 @@
export const WASI_WORKER_TEMPLATE = `import fs from "node:fs";
import { createRequire } from "node:module";
import { parse } from "node:path";
import { WASI } from "node:wasi";
import { parentPort, Worker } from "node:worker_threads";
@ -27,7 +28,9 @@ Object.assign(globalThis, {
},
});
const emnapiContext = getDefaultContext()
const emnapiContext = getDefaultContext();
const __rootDir = parse(process.cwd()).root;
const handler = new MessageHandler({
onLoad({ wasmModule, wasmMemory }) {
@ -35,7 +38,7 @@ const handler = new MessageHandler({
version: 'preview1',
env: process.env,
preopens: {
'/': '/',
[__rootDir]: __rootDir,
},
});
@ -62,21 +65,20 @@ globalThis.onmessage = function (e) {
export const createWasiBrowserWorkerBinding = (fs: boolean) => {
const fsImport = fs
? `import { Volume, createFsFromVolume } from '@napi-rs/wasm-runtime/fs'
? `import { memfsExported as __memfsExported } from '@napi-rs/wasm-runtime/fs'
const fs = createFsFromVolume(
Volume.fromJSON({
'/': null,
}),
)`
const fs = createFsProxy(__memfsExported)`
: '\nconst fs = null'
return `import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
return `import { instantiateNapiModuleSync, MessageHandler, WASI, createFsProxy } from '@napi-rs/wasm-runtime'
${fsImport}
const handler = new MessageHandler({
onLoad({ wasmModule, wasmMemory }) {
const wasi = new WASI({
fs,
preopens: {
'/': '/',
},
print: function () {
// eslint-disable-next-line no-console
console.log.apply(console, arguments)

12
cli/src/api/universalize.ts
View file

@ -42,8 +42,12 @@ export async function universalizeBinaries(userOptions: UniversalizeOptions) {
)
}
const srcFiles = UniArchsByPlatform[process.platform]?.map(
(arch) => resolve(options.cwd, options.outputDir, `${config.binaryName}.${process.platform}-${arch}.node`),
const srcFiles = UniArchsByPlatform[process.platform]?.map((arch) =>
resolve(
options.cwd,
options.outputDir,
`${config.binaryName}.${process.platform}-${arch}.node`,
),
)
if (!srcFiles || !universalizers[process.platform]) {
@ -55,9 +59,7 @@ export async function universalizeBinaries(userOptions: UniversalizeOptions) {
debug(`Looking up source binaries to combine: `)
debug(' %O', srcFiles)
const srcFileLookup = await Promise.all(
srcFiles.map((f) => fileExists(f)),
)
const srcFileLookup = await Promise.all(srcFiles.map((f) => fileExists(f)))
const notFoundFiles = srcFiles.filter((_, i) => !srcFileLookup[i])

2
debian-zig.Dockerfile
View file

@ -1,6 +1,6 @@
FROM ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
ARG ZIG_VERSION=0.11.0
ARG ZIG_VERSION=0.12.0
RUN wget https://ziglang.org/download/${ZIG_VERSION}/zig-linux-x86_64-${ZIG_VERSION}.tar.xz && \
tar -xvf zig-linux-x86_64-${ZIG_VERSION}.tar.xz && \

2
examples/napi/__tests__/__snapshots__/typegen.spec.ts.md
View file

@ -295,6 +295,8 @@ Generated by [AVA](https://avajs.dev).
␊
export function asyncTaskOptionalReturn(): Promise<number | null>␊
␊
export function asyncTaskReadFile(path: string): Promise<Buffer>␊
␊
export function asyncTaskVoidReturn(): Promise<void>␊
␊
export interface B {␊

BIN
examples/napi/__tests__/__snapshots__/typegen.spec.ts.snap
View file

Binary file not shown.

1
examples/napi/__tests__/__snapshots__/values.spec.ts.md
View file

@ -16,6 +16,7 @@ Generated by [AVA](https://avajs.dev).
'@vitest/browser',
'@vitest/ui',
'ava',
'buffer',
'cross-env',
'electron',
'lodash',

BIN
examples/napi/__tests__/__snapshots__/values.spec.ts.snap
View file

Binary file not shown.

2
examples/napi/__tests__/worker-thread.spec.ts
View file

@ -21,7 +21,7 @@ const concurrency = process.env.WASI_TEST
// @ts-expect-error
process?.report?.getReport()?.header?.glibcVersionRuntime &&
!process.env.ASAN_OPTIONS)
? 50
? 20
: 3
t('should be able to require in worker thread', async (t) => {

16
examples/napi/browser/values.spec.ts
View file

@ -1,17 +1,19 @@
import { Buffer } from 'buffer'
import { describe, it, expect } from 'vitest'
global.Buffer = Buffer
// @ts-expect-error
const {
// @ts-expect-error
__fs,
// @ts-expect-error
__volume,
DEFAULT_COST,
Bird,
GetterSetterWithClosures,
tsfnReturnPromise,
tsfnReturnPromiseTimeout,
readFileAsync,
asyncTaskReadFile,
}: typeof import('../index.cjs') = await import('../example.wasi-browser')
describe('NAPI-RS wasi browser test', function () {
@ -59,9 +61,9 @@ describe('NAPI-RS wasi browser test', function () {
await new Promise((resolve) => setTimeout(resolve, 400))
})
it.skip('readFileAsync', async () => {
__volume.writeFileSync('/test.txt', 'hello world')
const value = await readFileAsync('/test.txt')
expect(value).toBe('hello world')
it('readFileAsync', async () => {
__fs.writeFileSync('/test.txt', 'hello world')
const value = await asyncTaskReadFile('/test.txt')
expect(value.toString('utf8')).toBe('hello world')
})
})

88
examples/napi/example.wasi-browser.js
View file

@ -2,6 +2,7 @@ import {
instantiateNapiModuleSync as __emnapiInstantiateNapiModuleSync,
getDefaultContext as __emnapiGetDefaultContext,
WASI as __WASI,
createOnMessage as __wasmCreateOnMessageForFsProxy,
} from '@napi-rs/wasm-runtime'
import { memfs } from '@napi-rs/wasm-runtime/fs'
import __wasmUrl from './example.wasm32-wasi.wasm?url'
@ -35,9 +36,13 @@ const {
asyncWorkPoolSize: 4,
wasi: __wasi,
onCreateWorker() {
return new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
const worker = new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
type: 'module',
})
worker.addEventListener('message', __wasmCreateOnMessageForFsProxy(__fs))
return worker
},
overwriteImports(importObject) {
importObject.env = {
@ -292,45 +297,47 @@ function __napi_rs_initialize_modules(__napiInstance) {
__napiInstance.exports['__napi_register__async_task_void_return_296']?.()
__napiInstance.exports['__napi_register__AsyncTaskOptionalReturn_impl_297']?.()
__napiInstance.exports['__napi_register__async_task_optional_return_298']?.()
__napiInstance.exports['__napi_register__call_threadsafe_function_299']?.()
__napiInstance.exports['__napi_register__call_long_threadsafe_function_300']?.()
__napiInstance.exports['__napi_register__threadsafe_function_throw_error_301']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_302']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_303']?.()
__napiInstance.exports['__napi_register__threadsafe_function_closure_capture_304']?.()
__napiInstance.exports['__napi_register__tsfn_call_with_callback_305']?.()
__napiInstance.exports['__napi_register__tsfn_async_call_306']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_307']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_308']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_309']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_310']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_timeout_311']?.()
__napiInstance.exports['__napi_register__tsfn_throw_from_js_312']?.()
__napiInstance.exports['__napi_register__get_buffer_313']?.()
__napiInstance.exports['__napi_register__append_buffer_314']?.()
__napiInstance.exports['__napi_register__get_empty_buffer_315']?.()
__napiInstance.exports['__napi_register__convert_u32_array_316']?.()
__napiInstance.exports['__napi_register__create_external_typed_array_317']?.()
__napiInstance.exports['__napi_register__mutate_typed_array_318']?.()
__napiInstance.exports['__napi_register__deref_uint8_array_319']?.()
__napiInstance.exports['__napi_register__buffer_pass_through_320']?.()
__napiInstance.exports['__napi_register__array_buffer_pass_through_321']?.()
__napiInstance.exports['__napi_register__accept_slice_322']?.()
__napiInstance.exports['__napi_register__u8_array_to_array_323']?.()
__napiInstance.exports['__napi_register__i8_array_to_array_324']?.()
__napiInstance.exports['__napi_register__u16_array_to_array_325']?.()
__napiInstance.exports['__napi_register__i16_array_to_array_326']?.()
__napiInstance.exports['__napi_register__u32_array_to_array_327']?.()
__napiInstance.exports['__napi_register__i32_array_to_array_328']?.()
__napiInstance.exports['__napi_register__f32_array_to_array_329']?.()
__napiInstance.exports['__napi_register__f64_array_to_array_330']?.()
__napiInstance.exports['__napi_register__u64_array_to_array_331']?.()
__napiInstance.exports['__napi_register__i64_array_to_array_332']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_333']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_334']?.()
__napiInstance.exports['__napi_register__AsyncBuffer_impl_335']?.()
__napiInstance.exports['__napi_register__async_reduce_buffer_336']?.()
__napiInstance.exports['__napi_register__async_buffer_to_array_337']?.()
__napiInstance.exports['__napi_register__AsyncTaskReadFile_impl_299']?.()
__napiInstance.exports['__napi_register__async_task_read_file_300']?.()
__napiInstance.exports['__napi_register__call_threadsafe_function_301']?.()
__napiInstance.exports['__napi_register__call_long_threadsafe_function_302']?.()
__napiInstance.exports['__napi_register__threadsafe_function_throw_error_303']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_304']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_305']?.()
__napiInstance.exports['__napi_register__threadsafe_function_closure_capture_306']?.()
__napiInstance.exports['__napi_register__tsfn_call_with_callback_307']?.()
__napiInstance.exports['__napi_register__tsfn_async_call_308']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_309']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_310']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_311']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_312']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_timeout_313']?.()
__napiInstance.exports['__napi_register__tsfn_throw_from_js_314']?.()
__napiInstance.exports['__napi_register__get_buffer_315']?.()
__napiInstance.exports['__napi_register__append_buffer_316']?.()
__napiInstance.exports['__napi_register__get_empty_buffer_317']?.()
__napiInstance.exports['__napi_register__convert_u32_array_318']?.()
__napiInstance.exports['__napi_register__create_external_typed_array_319']?.()
__napiInstance.exports['__napi_register__mutate_typed_array_320']?.()
__napiInstance.exports['__napi_register__deref_uint8_array_321']?.()
__napiInstance.exports['__napi_register__buffer_pass_through_322']?.()
__napiInstance.exports['__napi_register__array_buffer_pass_through_323']?.()
__napiInstance.exports['__napi_register__accept_slice_324']?.()
__napiInstance.exports['__napi_register__u8_array_to_array_325']?.()
__napiInstance.exports['__napi_register__i8_array_to_array_326']?.()
__napiInstance.exports['__napi_register__u16_array_to_array_327']?.()
__napiInstance.exports['__napi_register__i16_array_to_array_328']?.()
__napiInstance.exports['__napi_register__u32_array_to_array_329']?.()
__napiInstance.exports['__napi_register__i32_array_to_array_330']?.()
__napiInstance.exports['__napi_register__f32_array_to_array_331']?.()
__napiInstance.exports['__napi_register__f64_array_to_array_332']?.()
__napiInstance.exports['__napi_register__u64_array_to_array_333']?.()
__napiInstance.exports['__napi_register__i64_array_to_array_334']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_335']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_336']?.()
__napiInstance.exports['__napi_register__AsyncBuffer_impl_337']?.()
__napiInstance.exports['__napi_register__async_reduce_buffer_338']?.()
__napiInstance.exports['__napi_register__async_buffer_to_array_339']?.()
}
export const Animal = __napiModule.exports.Animal
export const AnimalWithDefaultConstructor = __napiModule.exports.AnimalWithDefaultConstructor
@ -389,6 +396,7 @@ export const asyncMultiTwo = __napiModule.exports.asyncMultiTwo
export const asyncPlus100 = __napiModule.exports.asyncPlus100
export const asyncReduceBuffer = __napiModule.exports.asyncReduceBuffer
export const asyncTaskOptionalReturn = __napiModule.exports.asyncTaskOptionalReturn
export const asyncTaskReadFile = __napiModule.exports.asyncTaskReadFile
export const asyncTaskVoidReturn = __napiModule.exports.asyncTaskVoidReturn
export const bigintAdd = __napiModule.exports.bigintAdd
export const bigintFromI128 = __napiModule.exports.bigintFromI128

88
examples/napi/example.wasi.cjs
View file

@ -11,6 +11,7 @@ const { Worker } = require('node:worker_threads')
const {
instantiateNapiModuleSync: __emnapiInstantiateNapiModuleSync,
getDefaultContext: __emnapiGetDefaultContext,
createOnMessage: __wasmCreateOnMessageForFsProxy,
} = require('@napi-rs/wasm-runtime')
const __rootDir = __nodePath.parse(process.cwd()).root
@ -57,10 +58,14 @@ const { instance: __napiInstance, module: __wasiModule, napiModule: __napiModule
})(),
wasi: __wasi,
onCreateWorker() {
return new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
const worker = new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
env: process.env,
execArgv: ['--experimental-wasi-unstable-preview1'],
})
worker.onmessage = ({ data }) => {
__wasmCreateOnMessageForFsProxy(__nodeFs)(data)
}
return worker
},
overwriteImports(importObject) {
importObject.env = {
@ -315,45 +320,47 @@ function __napi_rs_initialize_modules(__napiInstance) {
__napiInstance.exports['__napi_register__async_task_void_return_296']?.()
__napiInstance.exports['__napi_register__AsyncTaskOptionalReturn_impl_297']?.()
__napiInstance.exports['__napi_register__async_task_optional_return_298']?.()
__napiInstance.exports['__napi_register__call_threadsafe_function_299']?.()
__napiInstance.exports['__napi_register__call_long_threadsafe_function_300']?.()
__napiInstance.exports['__napi_register__threadsafe_function_throw_error_301']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_302']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_303']?.()
__napiInstance.exports['__napi_register__threadsafe_function_closure_capture_304']?.()
__napiInstance.exports['__napi_register__tsfn_call_with_callback_305']?.()
__napiInstance.exports['__napi_register__tsfn_async_call_306']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_307']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_308']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_309']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_310']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_timeout_311']?.()
__napiInstance.exports['__napi_register__tsfn_throw_from_js_312']?.()
__napiInstance.exports['__napi_register__get_buffer_313']?.()
__napiInstance.exports['__napi_register__append_buffer_314']?.()
__napiInstance.exports['__napi_register__get_empty_buffer_315']?.()
__napiInstance.exports['__napi_register__convert_u32_array_316']?.()
__napiInstance.exports['__napi_register__create_external_typed_array_317']?.()
__napiInstance.exports['__napi_register__mutate_typed_array_318']?.()
__napiInstance.exports['__napi_register__deref_uint8_array_319']?.()
__napiInstance.exports['__napi_register__buffer_pass_through_320']?.()
__napiInstance.exports['__napi_register__array_buffer_pass_through_321']?.()
__napiInstance.exports['__napi_register__accept_slice_322']?.()
__napiInstance.exports['__napi_register__u8_array_to_array_323']?.()
__napiInstance.exports['__napi_register__i8_array_to_array_324']?.()
__napiInstance.exports['__napi_register__u16_array_to_array_325']?.()
__napiInstance.exports['__napi_register__i16_array_to_array_326']?.()
__napiInstance.exports['__napi_register__u32_array_to_array_327']?.()
__napiInstance.exports['__napi_register__i32_array_to_array_328']?.()
__napiInstance.exports['__napi_register__f32_array_to_array_329']?.()
__napiInstance.exports['__napi_register__f64_array_to_array_330']?.()
__napiInstance.exports['__napi_register__u64_array_to_array_331']?.()
__napiInstance.exports['__napi_register__i64_array_to_array_332']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_333']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_334']?.()
__napiInstance.exports['__napi_register__AsyncBuffer_impl_335']?.()
__napiInstance.exports['__napi_register__async_reduce_buffer_336']?.()
__napiInstance.exports['__napi_register__async_buffer_to_array_337']?.()
__napiInstance.exports['__napi_register__AsyncTaskReadFile_impl_299']?.()
__napiInstance.exports['__napi_register__async_task_read_file_300']?.()
__napiInstance.exports['__napi_register__call_threadsafe_function_301']?.()
__napiInstance.exports['__napi_register__call_long_threadsafe_function_302']?.()
__napiInstance.exports['__napi_register__threadsafe_function_throw_error_303']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_304']?.()
__napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_305']?.()
__napiInstance.exports['__napi_register__threadsafe_function_closure_capture_306']?.()
__napiInstance.exports['__napi_register__tsfn_call_with_callback_307']?.()
__napiInstance.exports['__napi_register__tsfn_async_call_308']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_309']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_310']?.()
__napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_311']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_312']?.()
__napiInstance.exports['__napi_register__tsfn_return_promise_timeout_313']?.()
__napiInstance.exports['__napi_register__tsfn_throw_from_js_314']?.()
__napiInstance.exports['__napi_register__get_buffer_315']?.()
__napiInstance.exports['__napi_register__append_buffer_316']?.()
__napiInstance.exports['__napi_register__get_empty_buffer_317']?.()
__napiInstance.exports['__napi_register__convert_u32_array_318']?.()
__napiInstance.exports['__napi_register__create_external_typed_array_319']?.()
__napiInstance.exports['__napi_register__mutate_typed_array_320']?.()
__napiInstance.exports['__napi_register__deref_uint8_array_321']?.()
__napiInstance.exports['__napi_register__buffer_pass_through_322']?.()
__napiInstance.exports['__napi_register__array_buffer_pass_through_323']?.()
__napiInstance.exports['__napi_register__accept_slice_324']?.()
__napiInstance.exports['__napi_register__u8_array_to_array_325']?.()
__napiInstance.exports['__napi_register__i8_array_to_array_326']?.()
__napiInstance.exports['__napi_register__u16_array_to_array_327']?.()
__napiInstance.exports['__napi_register__i16_array_to_array_328']?.()
__napiInstance.exports['__napi_register__u32_array_to_array_329']?.()
__napiInstance.exports['__napi_register__i32_array_to_array_330']?.()
__napiInstance.exports['__napi_register__f32_array_to_array_331']?.()
__napiInstance.exports['__napi_register__f64_array_to_array_332']?.()
__napiInstance.exports['__napi_register__u64_array_to_array_333']?.()
__napiInstance.exports['__napi_register__i64_array_to_array_334']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_335']?.()
__napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_336']?.()
__napiInstance.exports['__napi_register__AsyncBuffer_impl_337']?.()
__napiInstance.exports['__napi_register__async_reduce_buffer_338']?.()
__napiInstance.exports['__napi_register__async_buffer_to_array_339']?.()
}
module.exports.Animal = __napiModule.exports.Animal
module.exports.AnimalWithDefaultConstructor = __napiModule.exports.AnimalWithDefaultConstructor
@ -412,6 +419,7 @@ module.exports.asyncMultiTwo = __napiModule.exports.asyncMultiTwo
module.exports.asyncPlus100 = __napiModule.exports.asyncPlus100
module.exports.asyncReduceBuffer = __napiModule.exports.asyncReduceBuffer
module.exports.asyncTaskOptionalReturn = __napiModule.exports.asyncTaskOptionalReturn
module.exports.asyncTaskReadFile = __napiModule.exports.asyncTaskReadFile
module.exports.asyncTaskVoidReturn = __napiModule.exports.asyncTaskVoidReturn
module.exports.bigintAdd = __napiModule.exports.bigintAdd
module.exports.bigintFromI128 = __napiModule.exports.bigintFromI128

1
examples/napi/index.cjs
View file

@ -418,6 +418,7 @@ module.exports.asyncMultiTwo = nativeBinding.asyncMultiTwo
module.exports.asyncPlus100 = nativeBinding.asyncPlus100
module.exports.asyncReduceBuffer = nativeBinding.asyncReduceBuffer
module.exports.asyncTaskOptionalReturn = nativeBinding.asyncTaskOptionalReturn
module.exports.asyncTaskReadFile = nativeBinding.asyncTaskReadFile
module.exports.asyncTaskVoidReturn = nativeBinding.asyncTaskVoidReturn
module.exports.bigintAdd = nativeBinding.bigintAdd
module.exports.bigintFromI128 = nativeBinding.bigintFromI128

2
examples/napi/index.d.cts
View file

@ -285,6 +285,8 @@ export function asyncReduceBuffer(buf: Buffer): Promise<number>
export function asyncTaskOptionalReturn(): Promise<number | null>
export function asyncTaskReadFile(path: string): Promise<Buffer>
export function asyncTaskVoidReturn(): Promise<void>
export interface B {

3
examples/napi/package.json
View file

@ -29,6 +29,7 @@
"@vitest/browser": "^1.2.2",
"@vitest/ui": "^1.2.2",
"ava": "^6.1.1",
"buffer": "^6.0.3",
"cross-env": "7.0.3",
"electron": "^30.0.0",
"lodash": "^4.17.21",
@ -52,6 +53,6 @@
"timeout": "10m"
},
"dependencies": {
"@emnapi/core": "1.1.0"
"@emnapi/core": "^1.1.1"
}
}

23
examples/napi/src/task.rs
View file

@ -70,3 +70,26 @@ impl Task for AsyncTaskOptionalReturn {
fn async_task_optional_return() -> AsyncTask<AsyncTaskOptionalReturn> {
AsyncTask::new(AsyncTaskOptionalReturn {})
}
pub struct AsyncTaskReadFile {
path: String,
}
#[napi]
impl Task for AsyncTaskReadFile {
type Output = Vec<u8>;
type JsValue = Buffer;
fn compute(&mut self) -> Result<Self::Output> {
std::fs::read(&self.path).map_err(|e| Error::new(Status::GenericFailure, format!("{}", e)))
}
fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
Ok(output.into())
}
}
#[napi]
pub fn async_task_read_file(path: String) -> AsyncTask<AsyncTaskReadFile> {
AsyncTask::new(AsyncTaskReadFile { path })
}

16
examples/napi/vite-entry.js
View file

@ -1,10 +1,16 @@
import { Buffer } from 'buffer'
import {
Animal,
Kind,
asyncMultiTwo,
tsfnReturnPromise,
__fs,
asyncTaskReadFile,
} from './example.wasi-browser'
global.Buffer = Buffer
console.info(new Animal(Kind.Cat, 'Tom'))
asyncMultiTwo(200).then((res) => {
console.info(res)
@ -17,3 +23,13 @@ const value = await tsfnReturnPromise((err, value) => {
})
console.info(value)
__fs.writeFileSync('/test.txt', 'Hello, World!')
asyncTaskReadFile('/test.txt')
.then((res) => {
console.log(`readFileAsync: ${res}`)
})
.catch((err) => {
console.error(err)
})

13
examples/napi/wasi-worker-browser.mjs
View file

@ -1,16 +1,15 @@
import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
import { Volume, createFsFromVolume } from '@napi-rs/wasm-runtime/fs'
import { instantiateNapiModuleSync, MessageHandler, WASI, createFsProxy } from '@napi-rs/wasm-runtime'
import { memfsExported as __memfsExported } from '@napi-rs/wasm-runtime/fs'
const fs = createFsFromVolume(
Volume.fromJSON({
'/': null,
}),
)
const fs = createFsProxy(__memfsExported)
const handler = new MessageHandler({
onLoad({ wasmModule, wasmMemory }) {
const wasi = new WASI({
fs,
preopens: {
'/': '/',
},
print: function () {
// eslint-disable-next-line no-console
console.log.apply(console, arguments)

7
examples/napi/wasi-worker.mjs
View file

@ -1,5 +1,6 @@
import fs from "node:fs";
import { createRequire } from "node:module";
import { parse } from "node:path";
import { WASI } from "node:wasi";
import { parentPort, Worker } from "node:worker_threads";
@ -27,7 +28,9 @@ Object.assign(globalThis, {
},
});
const emnapiContext = getDefaultContext()
const emnapiContext = getDefaultContext();
const __rootDir = parse(process.cwd()).root;
const handler = new MessageHandler({
onLoad({ wasmModule, wasmMemory }) {
@ -35,7 +38,7 @@ const handler = new MessageHandler({
version: 'preview1',
env: process.env,
preopens: {
'/': '/',
[__rootDir]: __rootDir,
},
});

3
package.json
View file

@ -28,8 +28,7 @@
"format:rs": "cargo fmt",
"format:toml": "taplo format",
"lint": "oxlint --import-plugin --ignore-path=./.oxlintignore --deny-warnings -D correctness -A no-export",
"test": "lerna run test --concurrency=1 --ignore @napi-rs/cli",
"test:without-lerna": "yarn workspaces foreach -A --exclude \"{cli,napi-rs}\" run test",
"test": "yarn workspaces foreach -A --exclude \"{cli,napi-rs}\" run test",
"test:bun": "bun test examples/napi/__tests__/values.spec.ts",
"test:cli": "yarn workspace @napi-rs/cli test",
"test:electron": "electron examples/napi/electron.cjs",

192
wasm-runtime/fs-proxy.cjs Normal file
View file

@ -0,0 +1,192 @@
// @ts-check
/**
* @param {unknown} value
*/
const getType = (value) => {
if (value === undefined) return 0
if (value === null) return 1
const t = typeof value
if (t === 'boolean') return 2
if (t === 'number') return 3
if (t === 'string') return 4
if (t === 'object') return 6
if (t === 'bigint') return 9
return -1
}
/**
* @param {import('memfs').IFs} memfs
* @param {any} value
* @param {ReturnType<typeof getType>} type
* @returns {Uint8Array}
*/
const encodeValue = (memfs, value, type) => {
switch (type) {
case 0:
case 1:
return new Uint8Array(0)
case 2: {
const view = new Int32Array(1)
view[0] = value ? 1 : 0
return new Uint8Array(view.buffer)
}
case 3: {
const view = new Float64Array(1)
view[0] = value
return new Uint8Array(view.buffer)
}
case 4: {
const view = new TextEncoder().encode(value)
return view
}
case 6: {
const [entry] = Object.entries(memfs).filter(([_, v]) => v === value.constructor)[0] ?? []
if (entry) {
Object.defineProperty(value, '__constructor__', {
configurable: true,
writable: true,
enumerable: true,
value: entry
})
}
const json = JSON.stringify(value, (_, value) => {
if (typeof value === 'bigint') {
return `BigInt(${String(value)})`
}
return value
})
const view = new TextEncoder().encode(json)
return view
}
case 9: {
const view = new BigInt64Array(1)
view[0] = value
return new Uint8Array(view.buffer)
}
case -1:
default:
throw new Error('unsupported data')
}
}
/**
* @param {import('memfs').IFs} fs
* @returns {(e: { data: { __fs__: { sab: Int32Array, type: keyof import('memfs').IFs, payload: any[] } } }) => void}
*/
module.exports.createOnMessage = (fs) => function onMessage(e) {
if (e.data.__fs__) {
/**
* 0..4 status(int32_t): 21(waiting) 0(success) 1(error)
* 5..8 type(napi_valuetype): 0(undefined) 1(null) 2(boolean) 3(number) 4(string) 6(jsonstring) 9(bigint) -1(unsupported)
* 9..16 payload_size(uint32_t) <= 1024
* 16..16 + payload_size payload_content
*/
const { sab, type, payload } = e.data.__fs__
const fn = fs[type]
const args = payload ? payload.map((value) => {
if (value instanceof Uint8Array) {
// buffer polyfill bug
// @ts-expect-error
value._isBuffer = true
}
return value
}) : payload
try {
const ret = fn.apply(fs, args)
const t = getType(ret)
const v = encodeValue(fs, ret, t)
Atomics.store(sab, 0, 0)
Atomics.store(sab, 1, t)
Atomics.store(sab, 2, v.length)
new Uint8Array(sab.buffer).set(v, 16)
} catch (/** @type {any} */ err) {
Atomics.store(sab, 0, 1)
Atomics.store(sab, 1, 6)
const payloadContent = new TextEncoder().encode(JSON.stringify({
...err,
message: err.message,
stack: err.stack
}))
Atomics.store(sab, 2, payloadContent.length)
new Uint8Array(sab.buffer).set(payloadContent, 16)
} finally {
Atomics.notify(sab, 0)
}
}
}
/**
* @param {import('memfs').IFs} memfs
*/
module.exports.createFsProxy = (memfs) => new Proxy({}, {
get (_target, p, _receiver) {
/**
* @param {any[]} args
*/
return function (...args) {
const sab = new SharedArrayBuffer(16 + 1024)
const i32arr = new Int32Array(sab)
Atomics.store(i32arr, 0, 21)
// @ts-expect-error
postMessage({
__fs__: {
sab: i32arr,
type: p,
payload: args
}
})
Atomics.wait(i32arr, 0, 21)
const status = Atomics.load(i32arr, 0)
const type = Atomics.load(i32arr, 1)
const size = Atomics.load(i32arr, 2)
const content = new Uint8Array(sab, 16, size)
if (status === 1) {
const errobj = JSON.parse(new TextDecoder().decode(content.slice()))
const err = new Error(errobj.message)
Object.defineProperty(err, 'stack', {
configurable: true,
enumerable: false,
writable: true,
value: errobj.stack
})
for (const [k, v] of Object.entries(errobj)) {
if (k === 'message' || k === 'stack') continue
// @ts-expect-error
err[k] = v
}
throw err
}
if (type === 0) return undefined
if (type === 1) return null
if (type === 2) return Boolean(content[0])
if (type === 3) return new Float64Array(sab, 16, 1)[0]
if (type === 4) return new TextDecoder().decode(content.slice())
if (type === 6) {
const obj = JSON.parse(new TextDecoder().decode(content.slice()), (_key, value) => {
if (typeof value === 'string') {
const matched = value.match(/^BigInt\((-?\d+)\)$/)
if (matched && matched[1]) {
return BigInt(matched[1])
}
}
return value
})
if (obj.__constructor__) {
const ctor = obj.__constructor__
delete obj.__constructor__
// @ts-expect-error
Object.setPrototypeOf(obj, memfs[ctor].prototype)
}
return obj
}
if (type === 9) return new BigInt64Array(sab, 16, 1)[0]
throw new Error('unsupported data')
}
}
})

2
wasm-runtime/fs.js
View file

@ -2,4 +2,4 @@ import * as memfsExported from 'memfs'
const { createFsFromVolume, Volume, fs, memfs } = memfsExported
export { createFsFromVolume, Volume, fs, memfs }
export { createFsFromVolume, Volume, fs, memfs, memfsExported }

2
wasm-runtime/package.json
View file

@ -41,7 +41,7 @@
"dependencies": {
"@emnapi/core": "^1.1.0",
"@emnapi/runtime": "^1.1.0",
"@tybys/wasm-util": "^0.8.2"
"@tybys/wasm-util": "^0.8.3"
},
"scripts": {
"build": "rollup -c rollup.config.js"

1
wasm-runtime/rollup.config.js
View file

@ -56,6 +56,7 @@ export default defineConfig([
__webpack_public_path__: undefined,
preventAssignment: false,
}),
commonjs(),
nodeResolve({
preferBuiltins: false,
mainFields: ['browser', 'module', 'main'],

4
wasm-runtime/runtime.cjs
View file

@ -2,10 +2,14 @@ const { MessageHandler, instantiateNapiModuleSync, instantiateNapiModule } = req
const { getDefaultContext } = require('@emnapi/runtime')
const { WASI } = require('@tybys/wasm-util')
const { createFsProxy, createOnMessage } = require('./fs-proxy.cjs')
module.exports = {
MessageHandler,
instantiateNapiModule,
instantiateNapiModuleSync,
getDefaultContext,
WASI,
createFsProxy,
createOnMessage,
}

1
wasm-runtime/runtime.js
View file

@ -5,3 +5,4 @@ export {
} from '@emnapi/core'
export { getDefaultContext } from '@emnapi/runtime'
export { WASI } from '@tybys/wasm-util'
export { createOnMessage, createFsProxy } from './fs-proxy.cjs'

24
yarn.lock
View file

@ -120,16 +120,7 @@ __metadata:
languageName: node
linkType: hard
"@emnapi/core@npm:1.1.0":
version: 1.1.0
resolution: "@emnapi/core@npm:1.1.0"
dependencies:
tslib: "npm:^2.4.0"
checksum: 10c0/4ce4d214c863ba63eb93532c018b5d8fcf3c39a0a78e29aca76046f06584c0696dabaeb8922a31d876ed20d6b9a34615bbbb0cc1585707ea70972d4e431d546d
languageName: node
linkType: hard
"@emnapi/core@npm:^1.1.0":
"@emnapi/core@npm:^1.1.0, @emnapi/core@npm:^1.1.1":
version: 1.1.1
resolution: "@emnapi/core@npm:1.1.1"
dependencies:
@ -322,7 +313,7 @@ __metadata:
version: 0.0.0-use.local
resolution: "@examples/napi@workspace:examples/napi"
dependencies:
"@emnapi/core": "npm:1.1.0"
"@emnapi/core": "npm:^1.1.1"
"@napi-rs/cli": "workspace:*"
"@napi-rs/triples": "workspace:*"
"@napi-rs/wasm-runtime": "workspace:*"
@ -330,6 +321,7 @@ __metadata:
"@vitest/browser": "npm:^1.2.2"
"@vitest/ui": "npm:^1.2.2"
ava: "npm:^6.1.1"
buffer: "npm:^6.0.3"
cross-env: "npm:7.0.3"
electron: "npm:^30.0.0"
lodash: "npm:^4.17.21"
@ -904,7 +896,7 @@ __metadata:
"@rollup/plugin-json": "npm:^6.1.0"
"@rollup/plugin-node-resolve": "npm:^15.2.3"
"@rollup/plugin-replace": "npm:^5.0.5"
"@tybys/wasm-util": "npm:^0.8.2"
"@tybys/wasm-util": "npm:^0.8.3"
buffer: "npm:^6.0.3"
memfs: "npm:^4.8.2"
node-inspect-extracted: "npm:^3.0.0"
@ -2207,12 +2199,12 @@ __metadata:
languageName: node
linkType: hard
"@tybys/wasm-util@npm:^0.8.1, @tybys/wasm-util@npm:^0.8.2":
version: 0.8.2
resolution: "@tybys/wasm-util@npm:0.8.2"
"@tybys/wasm-util@npm:^0.8.1, @tybys/wasm-util@npm:^0.8.3":
version: 0.8.3
resolution: "@tybys/wasm-util@npm:0.8.3"
dependencies:
tslib: "npm:^2.4.0"
checksum: 10c0/3d2c1afa8623625311655b19c28ba6efa30070b86c9e7ad7d7e68295b2bc855f34a383ef39e4be3ee159db70ce28d28343df7885072e831416f16962229ce758
checksum: 10c0/62ec238710453713a207a2cbcfa74ae5bebc5cd4d718dc34370a7fd5263e8213fe7e24436ec6544105331006f6d0806115956bed030672268dad4a3c123e0f2d
languageName: node
linkType: hard