feat: accept the .wasm bytes or a compiled module in loadModule()

LOADING_WASM.md

@@ -19,4 +19,14 @@ Bundlers don’t emit `.wasm` files by default — they stay inside `node_module
 
 * **Next.js + Webpack:** Add a `.wasm` asset rule or use the copy method.
 * **Turbopack:** Only the copy method works for now.
-* **Dev mode:** Optionally use a watcher script to auto-copy on rebuilds.
+* **Dev mode:** Optionally use a watcher script to auto-copy on rebuilds.
+
+#### 📦 Hosts that can't fetch the file
+
+If the runtime can't fetch or read the `.wasm` file at all (e.g. a custom protocol without fetch support), load the bytes yourself and pass them in:
+
+```ts
+import { loadModule } from 'libpg-query';
+
+await loadModule({ wasm: bytes }); // ArrayBuffer, typed array, or a compiled WebAssembly.Module
+```

templates/full/index.ts

@@ -157,6 +157,9 @@ let replaceError: unknown;
 const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024;
 let largeUnfreed = 0;
 
+// Set once the host hands over the .wasm itself; the wrapper then never reads the file.
+let wasmProvided = false;
+
 // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch.
 async function compileWasm(url: string): Promise<WebAssembly.Module> {
   const proc = (globalThis as any).process;
@@ -172,11 +175,20 @@ async function compileWasm(url: string): Promise<WebAssembly.Module> {
 function instantiate(): Promise<void> {
   if (loading) return loading;
   const promise = (async () => {
+    if (wasmProvided) {
+      // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first.
+      await undefined;
+      startProvidedInstance();
+      return;
+    }
     let wasmUrl: string | undefined;
     const module: WasmModule = await PgQueryModule({
       locateFile: (path: string, dir: string) => (wasmUrl = dir + path),
     });
+    // loadModule({ wasm }) may have finished while the file was loading; keep its instance.
+    if (wasmProvided) return;
     compiled ??= await compileWasm(wasmUrl);
+    if (wasmProvided) return;
     wasmModule = module;
     replaceError = undefined;
   })();
@@ -263,7 +275,36 @@ function ensureLoaded() {
   throw new Error("WASM module not initialized. Call `loadModule()` first.");
 }
 
-export async function loadModule(): Promise<void> {
+type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never;
+
+export interface LoadModuleOptions {
+  /**
+   * The contents of `libpg-query.wasm`, or the module compiled from them, for
+   * hosts where the wrapper cannot read the file itself (a custom protocol
+   * without fetch support, a bundle that inlines assets). It is compiled once;
+   * every instance, including replacements after a failed call, is created from it.
+   */
+  wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule;
+}
+
+function startProvidedInstance(): void {
+  replaceInstance();
+  if (!wasmModule) throw replaceError;
+}
+
+// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not
+// mistaken for a callback-style hook by test runners.
+export async function loadModule(...args: [options?: LoadModuleOptions]): Promise<void> {
+  const wasm = args[0]?.wasm;
+  if (wasm !== undefined) {
+    const module = wasm instanceof WebAssembly.Module
+      ? wasm
+      : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView);
+    compiled = module;
+    wasmProvided = true;
+    startProvidedInstance();
+    return;
+  }
   while (!wasmModule) await instantiate();
 }
 

templates/index.ts

@@ -142,6 +142,9 @@ let replaceError: unknown;
 const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024;
 let largeUnfreed = 0;
 
+// Set once the host hands over the .wasm itself; the wrapper then never reads the file.
+let wasmProvided = false;
+
 // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch.
 async function compileWasm(url: string): Promise<WebAssembly.Module> {
   const proc = (globalThis as any).process;
@@ -157,11 +160,20 @@ async function compileWasm(url: string): Promise<WebAssembly.Module> {
 function instantiate(): Promise<void> {
   if (loading) return loading;
   const promise = (async () => {
+    if (wasmProvided) {
+      // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first.
+      await undefined;
+      startProvidedInstance();
+      return;
+    }
     let wasmUrl: string | undefined;
     const module: any = await PgQueryModule({
       locateFile: (path: string, dir: string) => (wasmUrl = dir + path),
     });
+    // loadModule({ wasm }) may have finished while the file was loading; keep its instance.
+    if (wasmProvided) return;
     compiled ??= await compileWasm(wasmUrl);
+    if (wasmProvided) return;
     wasmModule = module;
     replaceError = undefined;
   })();
@@ -248,7 +260,36 @@ function ensureLoaded() {
   throw new Error("WASM module not initialized. Call `loadModule()` first.");
 }
 
-export async function loadModule() {
+type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never;
+
+export interface LoadModuleOptions {
+  /**
+   * The contents of `libpg-query.wasm`, or the module compiled from them, for
+   * hosts where the wrapper cannot read the file itself (a custom protocol
+   * without fetch support, a bundle that inlines assets). It is compiled once;
+   * every instance, including replacements after a failed call, is created from it.
+   */
+  wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule;
+}
+
+function startProvidedInstance(): void {
+  replaceInstance();
+  if (!wasmModule) throw replaceError;
+}
+
+// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not
+// mistaken for a callback-style hook by test runners.
+export async function loadModule(...args: [options?: LoadModuleOptions]): Promise<void> {
+  const wasm = args[0]?.wasm;
+  if (wasm !== undefined) {
+    const module = wasm instanceof WebAssembly.Module
+      ? wasm
+      : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView);
+    compiled = module;
+    wasmProvided = true;
+    startProvidedInstance();
+    return;
+  }
   while (!wasmModule) await instantiate();
 }
 

versions/13/README.md

@@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users');
 
 Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first.
 
+### `loadModule({ wasm }): Promise<void>`
+
+For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it.
+
+```typescript
+import { loadModule, parseSync } from 'libpg-query';
+import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader
+
+await loadModule({ wasm });
+const result = parseSync('SELECT * FROM users');
+```
+
 ### Type Definitions
 
 ```typescript

versions/13/package.json

@@ -30,7 +30,7 @@
     "wasm:rebuild": "pnpm wasm:make rebuild",
     "wasm:clean": "pnpm wasm:make clean",
     "wasm:clean-cache": "pnpm wasm:make clean-cache",
-    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js"
+    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/load-module.test.js"
   },
   "author": "Constructive <developers@constructive.io>",
   "license": "MIT",

versions/13/src/index.ts

@@ -149,6 +149,9 @@ let replaceError: unknown;
 const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024;
 let largeUnfreed = 0;
 
+// Set once the host hands over the .wasm itself; the wrapper then never reads the file.
+let wasmProvided = false;
+
 // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch.
 async function compileWasm(url: string): Promise<WebAssembly.Module> {
   const proc = (globalThis as any).process;
@@ -164,11 +167,20 @@ async function compileWasm(url: string): Promise<WebAssembly.Module> {
 function instantiate(): Promise<void> {
   if (loading) return loading;
   const promise = (async () => {
+    if (wasmProvided) {
+      // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first.
+      await undefined;
+      startProvidedInstance();
+      return;
+    }
     let wasmUrl: string | undefined;
     const module: any = await PgQueryModule({
       locateFile: (path: string, dir: string) => (wasmUrl = dir + path),
     });
+    // loadModule({ wasm }) may have finished while the file was loading; keep its instance.
+    if (wasmProvided) return;
     compiled ??= await compileWasm(wasmUrl);
+    if (wasmProvided) return;
     wasmModule = module;
     replaceError = undefined;
   })();
@@ -255,7 +267,36 @@ function ensureLoaded() {
   throw new Error("WASM module not initialized. Call `loadModule()` first.");
 }
 
-export async function loadModule() {
+type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never;
+
+export interface LoadModuleOptions {
+  /**
+   * The contents of `libpg-query.wasm`, or the module compiled from them, for
+   * hosts where the wrapper cannot read the file itself (a custom protocol
+   * without fetch support, a bundle that inlines assets). It is compiled once;
+   * every instance, including replacements after a failed call, is created from it.
+   */
+  wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule;
+}
+
+function startProvidedInstance(): void {
+  replaceInstance();
+  if (!wasmModule) throw replaceError;
+}
+
+// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not
+// mistaken for a callback-style hook by test runners.
+export async function loadModule(...args: [options?: LoadModuleOptions]): Promise<void> {
+  const wasm = args[0]?.wasm;
+  if (wasm !== undefined) {
+    const module = wasm instanceof WebAssembly.Module
+      ? wasm
+      : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView);
+    compiled = module;
+    wasmProvided = true;
+    startProvidedInstance();
+    return;
+  }
   while (!wasmModule) await instantiate();
 }
 

versions/13/test/load-module.test.js

@@ -0,0 +1,94 @@
+const { describe, it } = require('node:test');
+const assert = require('node:assert/strict');
+const { execFileSync } = require('node:child_process');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const WASM = path.join(__dirname, '../wasm/libpg-query.wasm');
+
+// Runs `body` in a fresh process whose copy of the wrapper has no
+// libpg-query.wasm next to it, like a host that cannot read its own files.
+function withoutWasmFile(body) {
+  const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-'));
+  try {
+    for (const file of ['index.cjs', 'libpg-query.js']) {
+      fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file));
+    }
+    const script = `
+      const fs = require('node:fs');
+      const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))});
+      const bytes = fs.readFileSync(${JSON.stringify(WASM)});
+      const DEEP = 'SELECT 1' + '+1'.repeat(100000);
+      (async () => { ${body} })().then(
+        (result) => console.log(JSON.stringify(result)),
+        (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); }
+      );
+    `;
+    const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+      timeout: 120000,
+    });
+    return JSON.parse(out.trim().split('\n').pop());
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => {
+  it('fails to load without it', () => {
+    const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`);
+    assert.ok(result.error, 'expected loadModule() to reject without the .wasm file');
+  });
+
+  it('loads from the bytes and recovers after a failed call without the file', () => {
+    const result = withoutWasmFile(`
+      await lib.loadModule({ wasm: bytes });
+      const first = lib.parseSync('SELECT 1').stmts.length;
+      let threw = false;
+      try { lib.parseSync(DEEP); } catch { threw = true; }
+      const afterSync = lib.parseSync('SELECT 2').stmts.length;
+      let threwAsync = false;
+      try { await lib.parse(DEEP); } catch { threwAsync = true; }
+      const afterAsync = (await lib.parse('SELECT 3')).stmts.length;
+      return { first, threw, afterSync, threwAsync, afterAsync };
+    `);
+    assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 });
+  });
+
+  it('accepts a compiled WebAssembly.Module and a typed array view', () => {
+    const result = withoutWasmFile(`
+      await lib.loadModule({ wasm: new WebAssembly.Module(bytes) });
+      const fromModule = lib.parseSync('SELECT 1').stmts.length;
+      await lib.loadModule({ wasm: new Uint8Array(bytes) });
+      return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length };
+    `);
+    assert.deepEqual(result, { fromModule: 1, fromView: 2 });
+  });
+
+  it('starts a new instance from the bytes after repeated out-of-memory failures', () => {
+    const result = withoutWasmFile(`
+      await lib.loadModule({ wasm: bytes });
+      const huge = 'SELECT 1;\\n'.repeat(5000000);
+      const messages = [];
+      for (let i = 0; i < 3; i++) {
+        try { lib.parseSync(huge); } catch (e) { messages.push(e.message); }
+      }
+      return { messages, after: (await lib.parse('SELECT 1')).stmts.length };
+    `);
+    assert.equal(result.messages.length, 3);
+    assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/);
+    assert.equal(result.after, 1);
+  });
+
+  it('rejects input that is not WebAssembly', () => {
+    const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`);
+    assert.ok(result.error);
+  });
+});
+
+describe('loadModule() arity', () => {
+  it('stays 0 so test runners can pass it directly as a hook', () => {
+    assert.equal(require('../wasm/index.cjs').loadModule.length, 0);
+  });
+});

versions/14/README.md

@@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users');
 
 Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first.
 
+### `loadModule({ wasm }): Promise<void>`
+
+For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it.
+
+```typescript
+import { loadModule, parseSync } from 'libpg-query';
+import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader
+
+await loadModule({ wasm });
+const result = parseSync('SELECT * FROM users');
+```
+
 ### Type Definitions
 
 ```typescript

versions/14/package.json

@@ -30,7 +30,7 @@
     "wasm:rebuild": "pnpm wasm:make rebuild",
     "wasm:clean": "pnpm wasm:make clean",
     "wasm:clean-cache": "pnpm wasm:make clean-cache",
-    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js"
+    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/load-module.test.js"
   },
   "author": "Constructive <developers@constructive.io>",
   "license": "MIT",