feat(cli): allow sync fs operation between workers/mainThread (#2064)

* feat(cli): allow sync fs operation between workers/mainThread

* allow sync fs operation between workers/mainThread (#2065)

* Fix

* Update fixture

* flaky test

* Fix cross compile target

* Update zig

* macos-cross test was filtered

---------

Co-authored-by: Toyo Li <lifenglin314@outlook.com>

.github/renovate.json

@@ -4,7 +4,6 @@
     "**/node_modules/**",
     "**/bower_components/**",
     "**/vendor/**",
-    "**/examples/**",
     "**/__tests__/**",
     "**/test/**",
     "**/__fixtures__/**"

.github/workflows/test-release.yaml

@@ -388,13 +388,11 @@ jobs:
             args: '--platform linux/ppc64le'
             arch: 'ppc64'
             libc: 'gnu'
-            without-lerna: true
           - image: 'node:{:version}-slim'
             target: s390x-unknown-linux-gnu
             args: '--platform linux/s390x'
             arch: 's390x'
             libc: 'gnu'
-            without-lerna: true
           - image: 'node:{:version}-alpine'
             target: x86_64-unknown-linux-musl
             args: ''
@@ -444,8 +442,7 @@ jobs:
         with:
           image: ${{ steps.image-name.outputs.docker-image }}
           options: ${{ matrix.settings.args }} -v ${{ github.workspace }}/cores:/cores -v ${{ github.workspace }}:/build -w /build
-          run: >-
+          run: yarn test
-            ${{ matrix.settings.without-lerna && 'yarn test:without-lerna' || 'yarn test' }}
       - name: List files
         run: |
           ls -la .

.github/workflows/zig.yaml

@@ -23,7 +23,7 @@ jobs:
       matrix:
         target:
           [
-            'x86_64-apple-darwin',
+            'aarch64-apple-darwin',
             'x86_64-unknown-linux-musl',
             'aarch64-unknown-linux-musl',
             'armv7-unknown-linux-musleabihf',
@@ -53,7 +53,7 @@ jobs:
       - name: Install ziglang
         uses: goto-bus-stop/setup-zig@v2
         with:
-          version: 0.11.0
+          version: 0.12.0
       - name: Install dependencies
         run: yarn install --immutable --mode=skip-build
       - name: install MacOS SDK
@@ -89,8 +89,8 @@ jobs:
         settings:
           - host: ubuntu-latest
             target: x86_64-unknown-linux-musl
-          - host: macos-latest
+          - host: macos-14
-            target: x86_64-apple-darwin
+            target: aarch64-apple-darwin
           - host: ubuntu-latest
             target: aarch64-unknown-linux-musl
             docker-platform: arm64
@@ -128,7 +128,7 @@ jobs:
         run: yarn test --verbose
         env:
           SKIP_UNWIND_TEST: 1
-        if: matrix.settings.host == 'macos-latest'
+        if: matrix.settings.host == 'macos-14'
       - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
         if: matrix.settings.host == 'ubuntu-latest'
       - name: Test
@@ -149,7 +149,7 @@ jobs:
           options: --platform linux/${{ matrix.settings.docker-platform }} -v ${{ github.workspace }}:/build -w /build
           run: |
             set -e
-            yarn test:without-lerna
+            yarn test
       - name: Test
         uses: addnab/docker-run-action@v3
         if: matrix.settings.target == 'x86_64-unknown-linux-musl'

Cargo.toml

@@ -19,6 +19,11 @@ exclude = ["./testing"]
 [profile.release]
 lto = true
 
+[profile.wasi]
+inherits = "release"
+opt-level = "z"
+debug = 'full'
+
 [profile.napi-rs-custom]
 inherits = "dev"
 codegen-units = 1024

alpine-zig.Dockerfile

@@ -1,6 +1,6 @@
 FROM ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
 
-ARG ZIG_VERSION=0.11.0
+ARG ZIG_VERSION=0.12.0
 
 RUN apk add --update --no-cache --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing xz xz-dev && \
   rustup target add x86_64-unknown-linux-gnu && \

cli/src/api/artifacts.ts

@@ -31,8 +31,8 @@ export async function collectArtifacts(userOptions: ArtifactsOptions) {
   const universalSourceBins = new Set(
     targets
       .filter((platform) => platform.arch === 'universal')
-      .flatMap(
+      .flatMap((p) =>
-        (p) => UniArchsByPlatform[p.platform]?.map((a) => `${p.platform}-${a}`),
+        UniArchsByPlatform[p.platform]?.map((a) => `${p.platform}-${a}`),
       )
       .filter(Boolean) as string[],
   )

cli/src/api/templates/ci-template.ts

@@ -136,7 +136,7 @@ jobs:
       - uses: goto-bus-stop/setup-zig@v2
         if: \${{ contains(matrix.settings.target, 'musl') }}
         with:
-          version: 0.11.0
+          version: 0.12.0
 
       - name: Setup toolchain
         run: \${{ matrix.settings.setup }}

cli/src/api/templates/load-wasi-template.ts

@@ -22,10 +22,15 @@ const __wasi = new __WASI({
   version: 'preview1',
 })`
 
+  const workerFsHandler = fs
+    ? `\n    worker.addEventListener('message', __wasmCreateOnMessageForFsProxy(__fs))\n`
+    : ''
+
   return `import {
   instantiateNapiModuleSync as __emnapiInstantiateNapiModuleSync,
   getDefaultContext as __emnapiGetDefaultContext,
   WASI as __WASI,
+  createOnMessage as __wasmCreateOnMessageForFsProxy,
 } from '@napi-rs/wasm-runtime'
 ${fsImport}
 import __wasmUrl from './${wasiFilename}.wasm?url'
@@ -50,9 +55,11 @@ const {
   asyncWorkPoolSize: 4,
   wasi: __wasi,
   onCreateWorker() {
-    return new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
+    const worker = new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
       type: 'module',
     })
+    ${workerFsHandler}
+    return worker
   },
   overwriteImports(importObject) {
     importObject.env = {
@@ -87,7 +94,7 @@ export const createWasiBinding = (
 
 /* auto-generated by NAPI-RS */
 
-const __nodeFs= require('node:fs')
+const __nodeFs = require('node:fs')
 const __nodePath = require('node:path')
 const { WASI: __nodeWASI } = require('node:wasi')
 const { Worker } = require('node:worker_threads')
@@ -95,6 +102,7 @@ const { Worker } = require('node:worker_threads')
 const {
   instantiateNapiModuleSync: __emnapiInstantiateNapiModuleSync,
   getDefaultContext: __emnapiGetDefaultContext,
+  createOnMessage: __wasmCreateOnMessageForFsProxy,
 } = require('@napi-rs/wasm-runtime')
 
 const __rootDir = __nodePath.parse(process.cwd()).root
@@ -141,10 +149,14 @@ const { instance: __napiInstance, module: __wasiModule, napiModule: __napiModule
   })(),
   wasi: __wasi,
   onCreateWorker() {
-    return new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
+    const worker = new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
       env: process.env,
       execArgv: ['--experimental-wasi-unstable-preview1'],
     })
+    worker.onmessage = ({ data }) => {
+      __wasmCreateOnMessageForFsProxy(__nodeFs)(data)
+    }
+    return worker
   },
   overwriteImports(importObject) {
     importObject.env = {

cli/src/api/templates/wasi-worker-template.ts

@@ -1,5 +1,6 @@
 export const WASI_WORKER_TEMPLATE = `import fs from "node:fs";
 import { createRequire } from "node:module";
+import { parse } from "node:path";
 import { WASI } from "node:wasi";
 import { parentPort, Worker } from "node:worker_threads";
 
@@ -27,7 +28,9 @@ Object.assign(globalThis, {
   },
 });
 
-const emnapiContext = getDefaultContext()
+const emnapiContext = getDefaultContext();
+
+const __rootDir = parse(process.cwd()).root;
 
 const handler = new MessageHandler({
   onLoad({ wasmModule, wasmMemory }) {
@@ -35,7 +38,7 @@ const handler = new MessageHandler({
       version: 'preview1',
       env: process.env,
       preopens: {
-        '/': '/',
+        [__rootDir]: __rootDir,
       },
     });
 
@@ -62,21 +65,20 @@ globalThis.onmessage = function (e) {
 
 export const createWasiBrowserWorkerBinding = (fs: boolean) => {
   const fsImport = fs
-    ? `import { Volume, createFsFromVolume } from '@napi-rs/wasm-runtime/fs'
+    ? `import { memfsExported as __memfsExported } from '@napi-rs/wasm-runtime/fs'
 
-const fs = createFsFromVolume(
+const fs = createFsProxy(__memfsExported)`
-  Volume.fromJSON({
-    '/': null,
-  }),
-)`
     : '\nconst fs = null'
-  return `import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
+  return `import { instantiateNapiModuleSync, MessageHandler, WASI, createFsProxy } from '@napi-rs/wasm-runtime'
 ${fsImport}
 
 const handler = new MessageHandler({
   onLoad({ wasmModule, wasmMemory }) {
     const wasi = new WASI({
       fs,
+      preopens: {
+        '/': '/',
+      },
       print: function () {
         // eslint-disable-next-line no-console
         console.log.apply(console, arguments)

cli/src/api/universalize.ts

@@ -42,8 +42,12 @@ export async function universalizeBinaries(userOptions: UniversalizeOptions) {
     )
   }
 
-  const srcFiles = UniArchsByPlatform[process.platform]?.map(
+  const srcFiles = UniArchsByPlatform[process.platform]?.map((arch) =>
-    (arch) => resolve(options.cwd, options.outputDir, `${config.binaryName}.${process.platform}-${arch}.node`),
+    resolve(
+      options.cwd,
+      options.outputDir,
+      `${config.binaryName}.${process.platform}-${arch}.node`,
+    ),
   )
 
   if (!srcFiles || !universalizers[process.platform]) {
@@ -55,9 +59,7 @@ export async function universalizeBinaries(userOptions: UniversalizeOptions) {
   debug(`Looking up source binaries to combine: `)
   debug('  %O', srcFiles)
 
-  const srcFileLookup = await Promise.all(
+  const srcFileLookup = await Promise.all(srcFiles.map((f) => fileExists(f)))
-    srcFiles.map((f) => fileExists(f)),
-  )
 
   const notFoundFiles = srcFiles.filter((_, i) => !srcFileLookup[i])
 

debian-zig.Dockerfile

@@ -1,6 +1,6 @@
 FROM ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
 
-ARG ZIG_VERSION=0.11.0
+ARG ZIG_VERSION=0.12.0
 
 RUN wget https://ziglang.org/download/${ZIG_VERSION}/zig-linux-x86_64-${ZIG_VERSION}.tar.xz && \
   tar -xvf zig-linux-x86_64-${ZIG_VERSION}.tar.xz && \

examples/napi/__tests__/__snapshots__/typegen.spec.ts.md

@@ -295,6 +295,8 @@ Generated by [AVA](https://avajs.dev).
     ␊
     export function asyncTaskOptionalReturn(): Promise<number | null>␊
     ␊
+    export function asyncTaskReadFile(path: string): Promise<Buffer>␊
+    ␊
     export function asyncTaskVoidReturn(): Promise<void>␊
     ␊
     export interface B {␊

examples/napi/__tests__/__snapshots__/values.spec.ts.md

@@ -16,6 +16,7 @@ Generated by [AVA](https://avajs.dev).
       '@vitest/browser',
       '@vitest/ui',
       'ava',
+      'buffer',
       'cross-env',
       'electron',
       'lodash',

examples/napi/__tests__/worker-thread.spec.ts

@@ -21,7 +21,7 @@ const concurrency = process.env.WASI_TEST
         // @ts-expect-error
         process?.report?.getReport()?.header?.glibcVersionRuntime &&
         !process.env.ASAN_OPTIONS)
-    ? 50
+    ? 20
     : 3
 
 t('should be able to require in worker thread', async (t) => {

examples/napi/browser/values.spec.ts

@@ -1,17 +1,19 @@
+import { Buffer } from 'buffer'
+
 import { describe, it, expect } from 'vitest'
 
+global.Buffer = Buffer
+
 // @ts-expect-error
 const {
   // @ts-expect-error
   __fs,
-  // @ts-expect-error
-  __volume,
   DEFAULT_COST,
   Bird,
   GetterSetterWithClosures,
   tsfnReturnPromise,
   tsfnReturnPromiseTimeout,
-  readFileAsync,
+  asyncTaskReadFile,
 }: typeof import('../index.cjs') = await import('../example.wasi-browser')
 
 describe('NAPI-RS wasi browser test', function () {
@@ -59,9 +61,9 @@ describe('NAPI-RS wasi browser test', function () {
     await new Promise((resolve) => setTimeout(resolve, 400))
   })
 
-  it.skip('readFileAsync', async () => {
+  it('readFileAsync', async () => {
-    __volume.writeFileSync('/test.txt', 'hello world')
+    __fs.writeFileSync('/test.txt', 'hello world')
-    const value = await readFileAsync('/test.txt')
+    const value = await asyncTaskReadFile('/test.txt')
-    expect(value).toBe('hello world')
+    expect(value.toString('utf8')).toBe('hello world')
   })
 })

examples/napi/example.wasi-browser.js

@@ -2,6 +2,7 @@ import {
   instantiateNapiModuleSync as __emnapiInstantiateNapiModuleSync,
   getDefaultContext as __emnapiGetDefaultContext,
   WASI as __WASI,
+  createOnMessage as __wasmCreateOnMessageForFsProxy,
 } from '@napi-rs/wasm-runtime'
 import { memfs } from '@napi-rs/wasm-runtime/fs'
 import __wasmUrl from './example.wasm32-wasi.wasm?url'
@@ -35,9 +36,13 @@ const {
   asyncWorkPoolSize: 4,
   wasi: __wasi,
   onCreateWorker() {
-    return new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
+    const worker = new Worker(new URL('./wasi-worker-browser.mjs', import.meta.url), {
       type: 'module',
     })
+    
+    worker.addEventListener('message', __wasmCreateOnMessageForFsProxy(__fs))
+
+    return worker
   },
   overwriteImports(importObject) {
     importObject.env = {
@@ -292,45 +297,47 @@ function __napi_rs_initialize_modules(__napiInstance) {
   __napiInstance.exports['__napi_register__async_task_void_return_296']?.()
   __napiInstance.exports['__napi_register__AsyncTaskOptionalReturn_impl_297']?.()
   __napiInstance.exports['__napi_register__async_task_optional_return_298']?.()
-  __napiInstance.exports['__napi_register__call_threadsafe_function_299']?.()
+  __napiInstance.exports['__napi_register__AsyncTaskReadFile_impl_299']?.()
-  __napiInstance.exports['__napi_register__call_long_threadsafe_function_300']?.()
+  __napiInstance.exports['__napi_register__async_task_read_file_300']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_throw_error_301']?.()
+  __napiInstance.exports['__napi_register__call_threadsafe_function_301']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_302']?.()
+  __napiInstance.exports['__napi_register__call_long_threadsafe_function_302']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_303']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_throw_error_303']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_closure_capture_304']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_304']?.()
-  __napiInstance.exports['__napi_register__tsfn_call_with_callback_305']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_305']?.()
-  __napiInstance.exports['__napi_register__tsfn_async_call_306']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_closure_capture_306']?.()
-  __napiInstance.exports['__napi_register__accept_threadsafe_function_307']?.()
+  __napiInstance.exports['__napi_register__tsfn_call_with_callback_307']?.()
-  __napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_308']?.()
+  __napiInstance.exports['__napi_register__tsfn_async_call_308']?.()
-  __napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_309']?.()
+  __napiInstance.exports['__napi_register__accept_threadsafe_function_309']?.()
-  __napiInstance.exports['__napi_register__tsfn_return_promise_310']?.()
+  __napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_310']?.()
-  __napiInstance.exports['__napi_register__tsfn_return_promise_timeout_311']?.()
+  __napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_311']?.()
-  __napiInstance.exports['__napi_register__tsfn_throw_from_js_312']?.()
+  __napiInstance.exports['__napi_register__tsfn_return_promise_312']?.()
-  __napiInstance.exports['__napi_register__get_buffer_313']?.()
+  __napiInstance.exports['__napi_register__tsfn_return_promise_timeout_313']?.()
-  __napiInstance.exports['__napi_register__append_buffer_314']?.()
+  __napiInstance.exports['__napi_register__tsfn_throw_from_js_314']?.()
-  __napiInstance.exports['__napi_register__get_empty_buffer_315']?.()
+  __napiInstance.exports['__napi_register__get_buffer_315']?.()
-  __napiInstance.exports['__napi_register__convert_u32_array_316']?.()
+  __napiInstance.exports['__napi_register__append_buffer_316']?.()
-  __napiInstance.exports['__napi_register__create_external_typed_array_317']?.()
+  __napiInstance.exports['__napi_register__get_empty_buffer_317']?.()
-  __napiInstance.exports['__napi_register__mutate_typed_array_318']?.()
+  __napiInstance.exports['__napi_register__convert_u32_array_318']?.()
-  __napiInstance.exports['__napi_register__deref_uint8_array_319']?.()
+  __napiInstance.exports['__napi_register__create_external_typed_array_319']?.()
-  __napiInstance.exports['__napi_register__buffer_pass_through_320']?.()
+  __napiInstance.exports['__napi_register__mutate_typed_array_320']?.()
-  __napiInstance.exports['__napi_register__array_buffer_pass_through_321']?.()
+  __napiInstance.exports['__napi_register__deref_uint8_array_321']?.()
-  __napiInstance.exports['__napi_register__accept_slice_322']?.()
+  __napiInstance.exports['__napi_register__buffer_pass_through_322']?.()
-  __napiInstance.exports['__napi_register__u8_array_to_array_323']?.()
+  __napiInstance.exports['__napi_register__array_buffer_pass_through_323']?.()
-  __napiInstance.exports['__napi_register__i8_array_to_array_324']?.()
+  __napiInstance.exports['__napi_register__accept_slice_324']?.()
-  __napiInstance.exports['__napi_register__u16_array_to_array_325']?.()
+  __napiInstance.exports['__napi_register__u8_array_to_array_325']?.()
-  __napiInstance.exports['__napi_register__i16_array_to_array_326']?.()
+  __napiInstance.exports['__napi_register__i8_array_to_array_326']?.()
-  __napiInstance.exports['__napi_register__u32_array_to_array_327']?.()
+  __napiInstance.exports['__napi_register__u16_array_to_array_327']?.()
-  __napiInstance.exports['__napi_register__i32_array_to_array_328']?.()
+  __napiInstance.exports['__napi_register__i16_array_to_array_328']?.()
-  __napiInstance.exports['__napi_register__f32_array_to_array_329']?.()
+  __napiInstance.exports['__napi_register__u32_array_to_array_329']?.()
-  __napiInstance.exports['__napi_register__f64_array_to_array_330']?.()
+  __napiInstance.exports['__napi_register__i32_array_to_array_330']?.()
-  __napiInstance.exports['__napi_register__u64_array_to_array_331']?.()
+  __napiInstance.exports['__napi_register__f32_array_to_array_331']?.()
-  __napiInstance.exports['__napi_register__i64_array_to_array_332']?.()
+  __napiInstance.exports['__napi_register__f64_array_to_array_332']?.()
-  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_333']?.()
+  __napiInstance.exports['__napi_register__u64_array_to_array_333']?.()
-  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_334']?.()
+  __napiInstance.exports['__napi_register__i64_array_to_array_334']?.()
-  __napiInstance.exports['__napi_register__AsyncBuffer_impl_335']?.()
+  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_335']?.()
-  __napiInstance.exports['__napi_register__async_reduce_buffer_336']?.()
+  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_336']?.()
-  __napiInstance.exports['__napi_register__async_buffer_to_array_337']?.()
+  __napiInstance.exports['__napi_register__AsyncBuffer_impl_337']?.()
+  __napiInstance.exports['__napi_register__async_reduce_buffer_338']?.()
+  __napiInstance.exports['__napi_register__async_buffer_to_array_339']?.()
 }
 export const Animal = __napiModule.exports.Animal
 export const AnimalWithDefaultConstructor = __napiModule.exports.AnimalWithDefaultConstructor
@@ -389,6 +396,7 @@ export const asyncMultiTwo = __napiModule.exports.asyncMultiTwo
 export const asyncPlus100 = __napiModule.exports.asyncPlus100
 export const asyncReduceBuffer = __napiModule.exports.asyncReduceBuffer
 export const asyncTaskOptionalReturn = __napiModule.exports.asyncTaskOptionalReturn
+export const asyncTaskReadFile = __napiModule.exports.asyncTaskReadFile
 export const asyncTaskVoidReturn = __napiModule.exports.asyncTaskVoidReturn
 export const bigintAdd = __napiModule.exports.bigintAdd
 export const bigintFromI128 = __napiModule.exports.bigintFromI128

examples/napi/example.wasi.cjs

@@ -3,7 +3,7 @@
 
 /* auto-generated by NAPI-RS */
 
-const __nodeFs= require('node:fs')
+const __nodeFs = require('node:fs')
 const __nodePath = require('node:path')
 const { WASI: __nodeWASI } = require('node:wasi')
 const { Worker } = require('node:worker_threads')
@@ -11,6 +11,7 @@ const { Worker } = require('node:worker_threads')
 const {
   instantiateNapiModuleSync: __emnapiInstantiateNapiModuleSync,
   getDefaultContext: __emnapiGetDefaultContext,
+  createOnMessage: __wasmCreateOnMessageForFsProxy,
 } = require('@napi-rs/wasm-runtime')
 
 const __rootDir = __nodePath.parse(process.cwd()).root
@@ -57,10 +58,14 @@ const { instance: __napiInstance, module: __wasiModule, napiModule: __napiModule
   })(),
   wasi: __wasi,
   onCreateWorker() {
-    return new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
+    const worker = new Worker(__nodePath.join(__dirname, 'wasi-worker.mjs'), {
       env: process.env,
       execArgv: ['--experimental-wasi-unstable-preview1'],
     })
+    worker.onmessage = ({ data }) => {
+      __wasmCreateOnMessageForFsProxy(__nodeFs)(data)
+    }
+    return worker
   },
   overwriteImports(importObject) {
     importObject.env = {
@@ -315,45 +320,47 @@ function __napi_rs_initialize_modules(__napiInstance) {
   __napiInstance.exports['__napi_register__async_task_void_return_296']?.()
   __napiInstance.exports['__napi_register__AsyncTaskOptionalReturn_impl_297']?.()
   __napiInstance.exports['__napi_register__async_task_optional_return_298']?.()
-  __napiInstance.exports['__napi_register__call_threadsafe_function_299']?.()
+  __napiInstance.exports['__napi_register__AsyncTaskReadFile_impl_299']?.()
-  __napiInstance.exports['__napi_register__call_long_threadsafe_function_300']?.()
+  __napiInstance.exports['__napi_register__async_task_read_file_300']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_throw_error_301']?.()
+  __napiInstance.exports['__napi_register__call_threadsafe_function_301']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_302']?.()
+  __napiInstance.exports['__napi_register__call_long_threadsafe_function_302']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_303']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_throw_error_303']?.()
-  __napiInstance.exports['__napi_register__threadsafe_function_closure_capture_304']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_304']?.()
-  __napiInstance.exports['__napi_register__tsfn_call_with_callback_305']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_fatal_mode_error_305']?.()
-  __napiInstance.exports['__napi_register__tsfn_async_call_306']?.()
+  __napiInstance.exports['__napi_register__threadsafe_function_closure_capture_306']?.()
-  __napiInstance.exports['__napi_register__accept_threadsafe_function_307']?.()
+  __napiInstance.exports['__napi_register__tsfn_call_with_callback_307']?.()
-  __napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_308']?.()
+  __napiInstance.exports['__napi_register__tsfn_async_call_308']?.()
-  __napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_309']?.()
+  __napiInstance.exports['__napi_register__accept_threadsafe_function_309']?.()
-  __napiInstance.exports['__napi_register__tsfn_return_promise_310']?.()
+  __napiInstance.exports['__napi_register__accept_threadsafe_function_fatal_310']?.()
-  __napiInstance.exports['__napi_register__tsfn_return_promise_timeout_311']?.()
+  __napiInstance.exports['__napi_register__accept_threadsafe_function_tuple_args_311']?.()
-  __napiInstance.exports['__napi_register__tsfn_throw_from_js_312']?.()
+  __napiInstance.exports['__napi_register__tsfn_return_promise_312']?.()
-  __napiInstance.exports['__napi_register__get_buffer_313']?.()
+  __napiInstance.exports['__napi_register__tsfn_return_promise_timeout_313']?.()
-  __napiInstance.exports['__napi_register__append_buffer_314']?.()
+  __napiInstance.exports['__napi_register__tsfn_throw_from_js_314']?.()
-  __napiInstance.exports['__napi_register__get_empty_buffer_315']?.()
+  __napiInstance.exports['__napi_register__get_buffer_315']?.()
-  __napiInstance.exports['__napi_register__convert_u32_array_316']?.()
+  __napiInstance.exports['__napi_register__append_buffer_316']?.()
-  __napiInstance.exports['__napi_register__create_external_typed_array_317']?.()
+  __napiInstance.exports['__napi_register__get_empty_buffer_317']?.()
-  __napiInstance.exports['__napi_register__mutate_typed_array_318']?.()
+  __napiInstance.exports['__napi_register__convert_u32_array_318']?.()
-  __napiInstance.exports['__napi_register__deref_uint8_array_319']?.()
+  __napiInstance.exports['__napi_register__create_external_typed_array_319']?.()
-  __napiInstance.exports['__napi_register__buffer_pass_through_320']?.()
+  __napiInstance.exports['__napi_register__mutate_typed_array_320']?.()
-  __napiInstance.exports['__napi_register__array_buffer_pass_through_321']?.()
+  __napiInstance.exports['__napi_register__deref_uint8_array_321']?.()
-  __napiInstance.exports['__napi_register__accept_slice_322']?.()
+  __napiInstance.exports['__napi_register__buffer_pass_through_322']?.()
-  __napiInstance.exports['__napi_register__u8_array_to_array_323']?.()
+  __napiInstance.exports['__napi_register__array_buffer_pass_through_323']?.()
-  __napiInstance.exports['__napi_register__i8_array_to_array_324']?.()
+  __napiInstance.exports['__napi_register__accept_slice_324']?.()
-  __napiInstance.exports['__napi_register__u16_array_to_array_325']?.()
+  __napiInstance.exports['__napi_register__u8_array_to_array_325']?.()
-  __napiInstance.exports['__napi_register__i16_array_to_array_326']?.()
+  __napiInstance.exports['__napi_register__i8_array_to_array_326']?.()
-  __napiInstance.exports['__napi_register__u32_array_to_array_327']?.()
+  __napiInstance.exports['__napi_register__u16_array_to_array_327']?.()
-  __napiInstance.exports['__napi_register__i32_array_to_array_328']?.()
+  __napiInstance.exports['__napi_register__i16_array_to_array_328']?.()
-  __napiInstance.exports['__napi_register__f32_array_to_array_329']?.()
+  __napiInstance.exports['__napi_register__u32_array_to_array_329']?.()
-  __napiInstance.exports['__napi_register__f64_array_to_array_330']?.()
+  __napiInstance.exports['__napi_register__i32_array_to_array_330']?.()
-  __napiInstance.exports['__napi_register__u64_array_to_array_331']?.()
+  __napiInstance.exports['__napi_register__f32_array_to_array_331']?.()
-  __napiInstance.exports['__napi_register__i64_array_to_array_332']?.()
+  __napiInstance.exports['__napi_register__f64_array_to_array_332']?.()
-  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_333']?.()
+  __napiInstance.exports['__napi_register__u64_array_to_array_333']?.()
-  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_334']?.()
+  __napiInstance.exports['__napi_register__i64_array_to_array_334']?.()
-  __napiInstance.exports['__napi_register__AsyncBuffer_impl_335']?.()
+  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_335']?.()
-  __napiInstance.exports['__napi_register__async_reduce_buffer_336']?.()
+  __napiInstance.exports['__napi_register__accept_uint8_clamped_slice_and_buffer_slice_336']?.()
-  __napiInstance.exports['__napi_register__async_buffer_to_array_337']?.()
+  __napiInstance.exports['__napi_register__AsyncBuffer_impl_337']?.()
+  __napiInstance.exports['__napi_register__async_reduce_buffer_338']?.()
+  __napiInstance.exports['__napi_register__async_buffer_to_array_339']?.()
 }
 module.exports.Animal = __napiModule.exports.Animal
 module.exports.AnimalWithDefaultConstructor = __napiModule.exports.AnimalWithDefaultConstructor
@@ -412,6 +419,7 @@ module.exports.asyncMultiTwo = __napiModule.exports.asyncMultiTwo
 module.exports.asyncPlus100 = __napiModule.exports.asyncPlus100
 module.exports.asyncReduceBuffer = __napiModule.exports.asyncReduceBuffer
 module.exports.asyncTaskOptionalReturn = __napiModule.exports.asyncTaskOptionalReturn
+module.exports.asyncTaskReadFile = __napiModule.exports.asyncTaskReadFile
 module.exports.asyncTaskVoidReturn = __napiModule.exports.asyncTaskVoidReturn
 module.exports.bigintAdd = __napiModule.exports.bigintAdd
 module.exports.bigintFromI128 = __napiModule.exports.bigintFromI128

examples/napi/index.cjs

@@ -418,6 +418,7 @@ module.exports.asyncMultiTwo = nativeBinding.asyncMultiTwo
 module.exports.asyncPlus100 = nativeBinding.asyncPlus100
 module.exports.asyncReduceBuffer = nativeBinding.asyncReduceBuffer
 module.exports.asyncTaskOptionalReturn = nativeBinding.asyncTaskOptionalReturn
+module.exports.asyncTaskReadFile = nativeBinding.asyncTaskReadFile
 module.exports.asyncTaskVoidReturn = nativeBinding.asyncTaskVoidReturn
 module.exports.bigintAdd = nativeBinding.bigintAdd
 module.exports.bigintFromI128 = nativeBinding.bigintFromI128

examples/napi/index.d.cts

@@ -285,6 +285,8 @@ export function asyncReduceBuffer(buf: Buffer): Promise<number>
 
 export function asyncTaskOptionalReturn(): Promise<number | null>
 
+export function asyncTaskReadFile(path: string): Promise<Buffer>
+
 export function asyncTaskVoidReturn(): Promise<void>
 
 export interface B {

examples/napi/package.json

@@ -29,6 +29,7 @@
     "@vitest/browser": "^1.2.2",
     "@vitest/ui": "^1.2.2",
     "ava": "^6.1.1",
+    "buffer": "^6.0.3",
     "cross-env": "7.0.3",
     "electron": "^30.0.0",
     "lodash": "^4.17.21",
@@ -52,6 +53,6 @@
     "timeout": "10m"
   },
   "dependencies": {
-    "@emnapi/core": "1.1.0"
+    "@emnapi/core": "^1.1.1"
   }
 }

examples/napi/src/task.rs

@@ -70,3 +70,26 @@ impl Task for AsyncTaskOptionalReturn {
 fn async_task_optional_return() -> AsyncTask<AsyncTaskOptionalReturn> {
   AsyncTask::new(AsyncTaskOptionalReturn {})
 }
+
+pub struct AsyncTaskReadFile {
+  path: String,
+}
+
+#[napi]
+impl Task for AsyncTaskReadFile {
+  type Output = Vec<u8>;
+  type JsValue = Buffer;
+
+  fn compute(&mut self) -> Result<Self::Output> {
+    std::fs::read(&self.path).map_err(|e| Error::new(Status::GenericFailure, format!("{}", e)))
+  }
+
+  fn resolve(&mut self, _: Env, output: Self::Output) -> Result<Self::JsValue> {
+    Ok(output.into())
+  }
+}
+
+#[napi]
+pub fn async_task_read_file(path: String) -> AsyncTask<AsyncTaskReadFile> {
+  AsyncTask::new(AsyncTaskReadFile { path })
+}

examples/napi/vite-entry.js

@@ -1,10 +1,16 @@
+import { Buffer } from 'buffer'
+
 import {
   Animal,
   Kind,
   asyncMultiTwo,
   tsfnReturnPromise,
+  __fs,
+  asyncTaskReadFile,
 } from './example.wasi-browser'
 
+global.Buffer = Buffer
+
 console.info(new Animal(Kind.Cat, 'Tom'))
 asyncMultiTwo(200).then((res) => {
   console.info(res)
@@ -17,3 +23,13 @@ const value = await tsfnReturnPromise((err, value) => {
 })
 
 console.info(value)
+
+__fs.writeFileSync('/test.txt', 'Hello, World!')
+
+asyncTaskReadFile('/test.txt')
+  .then((res) => {
+    console.log(`readFileAsync: ${res}`)
+  })
+  .catch((err) => {
+    console.error(err)
+  })

examples/napi/wasi-worker-browser.mjs

@@ -1,16 +1,15 @@
-import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
+import { instantiateNapiModuleSync, MessageHandler, WASI, createFsProxy } from '@napi-rs/wasm-runtime'
-import { Volume, createFsFromVolume } from '@napi-rs/wasm-runtime/fs'
+import { memfsExported as __memfsExported } from '@napi-rs/wasm-runtime/fs'
 
-const fs = createFsFromVolume(
+const fs = createFsProxy(__memfsExported)
-  Volume.fromJSON({
-    '/': null,
-  }),
-)
 
 const handler = new MessageHandler({
   onLoad({ wasmModule, wasmMemory }) {
     const wasi = new WASI({
       fs,
+      preopens: {
+        '/': '/',
+      },
       print: function () {
         // eslint-disable-next-line no-console
         console.log.apply(console, arguments)

examples/napi/wasi-worker.mjs

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import { createRequire } from "node:module";
+import { parse } from "node:path";
 import { WASI } from "node:wasi";
 import { parentPort, Worker } from "node:worker_threads";
 
@@ -27,7 +28,9 @@ Object.assign(globalThis, {
   },
 });
 
-const emnapiContext = getDefaultContext()
+const emnapiContext = getDefaultContext();
+
+const __rootDir = parse(process.cwd()).root;
 
 const handler = new MessageHandler({
   onLoad({ wasmModule, wasmMemory }) {
@@ -35,7 +38,7 @@ const handler = new MessageHandler({
       version: 'preview1',
       env: process.env,
       preopens: {
-        '/': '/',
+        [__rootDir]: __rootDir,
       },
     });
 

package.json

@@ -28,8 +28,7 @@
     "format:rs": "cargo fmt",
     "format:toml": "taplo format",
     "lint": "oxlint --import-plugin --ignore-path=./.oxlintignore --deny-warnings -D correctness -A no-export",
-    "test": "lerna run test --concurrency=1 --ignore @napi-rs/cli",
+    "test": "yarn workspaces foreach -A --exclude \"{cli,napi-rs}\" run test",
-    "test:without-lerna": "yarn workspaces foreach -A --exclude \"{cli,napi-rs}\" run test",
     "test:bun": "bun test examples/napi/__tests__/values.spec.ts",
     "test:cli": "yarn workspace @napi-rs/cli test",
     "test:electron": "electron examples/napi/electron.cjs",

wasm-runtime/fs-proxy.cjs

@@ -0,0 +1,192 @@
+// @ts-check
+
+/**
+ * @param {unknown} value
+ */
+const getType = (value) => {
+  if (value === undefined) return 0
+  if (value === null) return 1
+  const t = typeof value
+  if (t === 'boolean') return 2
+  if (t === 'number') return 3
+  if (t === 'string') return 4
+  if (t === 'object') return 6
+  if (t === 'bigint') return 9
+  return -1
+}
+
+/**
+ * @param {import('memfs').IFs} memfs
+ * @param {any} value
+ * @param {ReturnType<typeof getType>} type
+ * @returns {Uint8Array}
+ */
+const encodeValue = (memfs, value, type) => {
+  switch (type) {
+    case 0:
+    case 1:
+      return new Uint8Array(0)
+    case 2: {
+      const view = new Int32Array(1)
+      view[0] = value ? 1 : 0
+      return new Uint8Array(view.buffer)
+    }
+    case 3: {
+      const view = new Float64Array(1)
+      view[0] = value
+      return new Uint8Array(view.buffer)
+    }
+    case 4: {
+      const view = new TextEncoder().encode(value)
+      return view
+    }
+    case 6: {
+      const [entry] = Object.entries(memfs).filter(([_, v]) => v === value.constructor)[0] ?? []
+      if (entry) {
+        Object.defineProperty(value, '__constructor__', {
+          configurable: true,
+          writable: true,
+          enumerable: true,
+          value: entry
+        })
+      }
+
+      const json = JSON.stringify(value, (_, value) => {
+        if (typeof value === 'bigint') {
+          return `BigInt(${String(value)})`
+        }
+        return value
+      })
+      const view = new TextEncoder().encode(json)
+      return view
+    }
+    case 9: {
+      const view = new BigInt64Array(1)
+      view[0] = value
+      return new Uint8Array(view.buffer)
+    }
+    case -1:
+    default:
+      throw new Error('unsupported data')
+  }
+}
+
+/**
+ * @param {import('memfs').IFs} fs
+ * @returns {(e: { data: { __fs__: { sab: Int32Array, type: keyof import('memfs').IFs, payload: any[] } } }) => void}
+ */
+module.exports.createOnMessage = (fs) => function onMessage(e) {
+  if (e.data.__fs__) {
+    /**
+     * 0..4                    status(int32_t):        21(waiting) 0(success) 1(error)
+     * 5..8                    type(napi_valuetype):   0(undefined) 1(null) 2(boolean) 3(number) 4(string) 6(jsonstring) 9(bigint) -1(unsupported)
+     * 9..16                   payload_size(uint32_t)  <= 1024
+     * 16..16 + payload_size   payload_content
+     */
+    const { sab, type, payload } = e.data.__fs__
+    const fn = fs[type]
+    const args = payload ? payload.map((value) => {
+      if (value instanceof Uint8Array) {
+        // buffer polyfill bug
+        // @ts-expect-error
+        value._isBuffer = true
+      }
+      return value
+    }) : payload
+    try {
+      const ret = fn.apply(fs, args)
+      const t = getType(ret)
+      const v = encodeValue(fs, ret, t)
+      Atomics.store(sab, 0, 0)
+      Atomics.store(sab, 1, t)
+      Atomics.store(sab, 2, v.length)
+      new Uint8Array(sab.buffer).set(v, 16)
+
+    } catch (/** @type {any} */ err) {
+      Atomics.store(sab, 0, 1)
+      Atomics.store(sab, 1, 6)
+      const payloadContent = new TextEncoder().encode(JSON.stringify({
+        ...err,
+        message: err.message,
+        stack: err.stack
+      }))
+      Atomics.store(sab, 2, payloadContent.length)
+      new Uint8Array(sab.buffer).set(payloadContent, 16)
+    } finally {
+      Atomics.notify(sab, 0)
+    }
+  }
+}
+
+/**
+ * @param {import('memfs').IFs} memfs
+ */
+module.exports.createFsProxy = (memfs) => new Proxy({}, {
+  get (_target, p, _receiver) {
+    /**
+     * @param {any[]} args
+     */
+    return function (...args) {
+      const sab = new SharedArrayBuffer(16 + 1024)
+      const i32arr = new Int32Array(sab)
+      Atomics.store(i32arr, 0, 21)
+
+      // @ts-expect-error
+      postMessage({
+        __fs__: {
+          sab: i32arr,
+          type: p,
+          payload: args
+        }
+      })
+
+      Atomics.wait(i32arr, 0, 21)
+
+      const status = Atomics.load(i32arr, 0)
+      const type = Atomics.load(i32arr, 1)
+      const size = Atomics.load(i32arr, 2)
+      const content = new Uint8Array(sab, 16, size)
+      if (status === 1) {
+        const errobj = JSON.parse(new TextDecoder().decode(content.slice()))
+        const err = new Error(errobj.message)
+        Object.defineProperty(err, 'stack', {
+          configurable: true,
+          enumerable: false,
+          writable: true,
+          value: errobj.stack
+        })
+        for (const [k, v] of Object.entries(errobj)) {
+          if (k === 'message' || k === 'stack') continue
+          // @ts-expect-error
+          err[k] = v
+        }
+        throw err
+      }
+      if (type === 0) return undefined
+      if (type === 1) return null
+      if (type === 2) return Boolean(content[0])
+      if (type === 3) return new Float64Array(sab, 16, 1)[0]
+      if (type === 4) return new TextDecoder().decode(content.slice())
+      if (type === 6) {
+        const obj = JSON.parse(new TextDecoder().decode(content.slice()), (_key, value) => {
+          if (typeof value === 'string') {
+            const matched = value.match(/^BigInt\((-?\d+)\)$/)
+            if (matched && matched[1]) {
+              return BigInt(matched[1])
+            }
+          }
+          return value
+        })
+        if (obj.__constructor__) {
+          const ctor = obj.__constructor__
+          delete obj.__constructor__
+          // @ts-expect-error
+          Object.setPrototypeOf(obj, memfs[ctor].prototype)
+        }
+        return obj
+      }
+      if (type === 9) return new BigInt64Array(sab, 16, 1)[0]
+      throw new Error('unsupported data')
+    }
+  }
+})

wasm-runtime/fs.js

@@ -2,4 +2,4 @@ import * as memfsExported from 'memfs'
 
 const { createFsFromVolume, Volume, fs, memfs } = memfsExported
 
-export { createFsFromVolume, Volume, fs, memfs }
+export { createFsFromVolume, Volume, fs, memfs, memfsExported }

wasm-runtime/package.json

@@ -41,7 +41,7 @@
   "dependencies": {
     "@emnapi/core": "^1.1.0",
     "@emnapi/runtime": "^1.1.0",
-    "@tybys/wasm-util": "^0.8.2"
+    "@tybys/wasm-util": "^0.8.3"
   },
   "scripts": {
     "build": "rollup -c rollup.config.js"

wasm-runtime/rollup.config.js

@@ -56,6 +56,7 @@ export default defineConfig([
         __webpack_public_path__: undefined,
         preventAssignment: false,
       }),
+      commonjs(),
       nodeResolve({
         preferBuiltins: false,
         mainFields: ['browser', 'module', 'main'],

wasm-runtime/runtime.cjs

@@ -2,10 +2,14 @@ const { MessageHandler, instantiateNapiModuleSync, instantiateNapiModule } = req
 const { getDefaultContext } = require('@emnapi/runtime')
 const { WASI } = require('@tybys/wasm-util')
 
+const { createFsProxy, createOnMessage } = require('./fs-proxy.cjs')
+
 module.exports = {
   MessageHandler,
   instantiateNapiModule,
   instantiateNapiModuleSync,
   getDefaultContext,
   WASI,
+  createFsProxy,
+  createOnMessage,
 }

wasm-runtime/runtime.js

@@ -5,3 +5,4 @@ export {
 } from '@emnapi/core'
 export { getDefaultContext } from '@emnapi/runtime'
 export { WASI } from '@tybys/wasm-util'
+export { createOnMessage, createFsProxy } from './fs-proxy.cjs'

yarn.lock

@@ -120,16 +120,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@emnapi/core@npm:1.1.0":
+"@emnapi/core@npm:^1.1.0, @emnapi/core@npm:^1.1.1":
-  version: 1.1.0
-  resolution: "@emnapi/core@npm:1.1.0"
-  dependencies:
-    tslib: "npm:^2.4.0"
-  checksum: 10c0/4ce4d214c863ba63eb93532c018b5d8fcf3c39a0a78e29aca76046f06584c0696dabaeb8922a31d876ed20d6b9a34615bbbb0cc1585707ea70972d4e431d546d
-  languageName: node
-  linkType: hard
-
-"@emnapi/core@npm:^1.1.0":
   version: 1.1.1
   resolution: "@emnapi/core@npm:1.1.1"
   dependencies:
@@ -322,7 +313,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@examples/napi@workspace:examples/napi"
   dependencies:
-    "@emnapi/core": "npm:1.1.0"
+    "@emnapi/core": "npm:^1.1.1"
     "@napi-rs/cli": "workspace:*"
     "@napi-rs/triples": "workspace:*"
     "@napi-rs/wasm-runtime": "workspace:*"
@@ -330,6 +321,7 @@ __metadata:
     "@vitest/browser": "npm:^1.2.2"
     "@vitest/ui": "npm:^1.2.2"
     ava: "npm:^6.1.1"
+    buffer: "npm:^6.0.3"
     cross-env: "npm:7.0.3"
     electron: "npm:^30.0.0"
     lodash: "npm:^4.17.21"
@@ -904,7 +896,7 @@ __metadata:
     "@rollup/plugin-json": "npm:^6.1.0"
     "@rollup/plugin-node-resolve": "npm:^15.2.3"
     "@rollup/plugin-replace": "npm:^5.0.5"
-    "@tybys/wasm-util": "npm:^0.8.2"
+    "@tybys/wasm-util": "npm:^0.8.3"
     buffer: "npm:^6.0.3"
     memfs: "npm:^4.8.2"
     node-inspect-extracted: "npm:^3.0.0"
@@ -2207,12 +2199,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@tybys/wasm-util@npm:^0.8.1, @tybys/wasm-util@npm:^0.8.2":
+"@tybys/wasm-util@npm:^0.8.1, @tybys/wasm-util@npm:^0.8.3":
-  version: 0.8.2
+  version: 0.8.3
-  resolution: "@tybys/wasm-util@npm:0.8.2"
+  resolution: "@tybys/wasm-util@npm:0.8.3"
   dependencies:
     tslib: "npm:^2.4.0"
-  checksum: 10c0/3d2c1afa8623625311655b19c28ba6efa30070b86c9e7ad7d7e68295b2bc855f34a383ef39e4be3ee159db70ce28d28343df7885072e831416f16962229ce758
+  checksum: 10c0/62ec238710453713a207a2cbcfa74ae5bebc5cd4d718dc34370a7fd5263e8213fe7e24436ec6544105331006f6d0806115956bed030672268dad4a3c123e0f2d
   languageName: node
   linkType: hard