Initial Commit

packages/foundation/flow/streams/src/utils/buffer/buffer-to-uint8/buffer-to-uint8.md

@@ -0,0 +1,71 @@
+# bufferToUint8Array
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+`bufferToUint8Array` normalises the two byte-container shapes that show up in
+mixed Node/browser code — `Buffer` and `ArrayBuffer` — into the single
+canonical `Uint8Array` form used throughout `@suaveplan/streams`. The streams
+package treats `Uint8Array` as the universal byte chunk; this helper lets call
+sites accept either input without sprinkling `instanceof` checks across every
+adapter.
+
+The conversion is zero-copy. For `Buffer` inputs we construct a view that
+respects the source's `byteOffset` / `byteLength`, which matters because Node
+typically slices new `Buffer`s out of a shared pool. For `ArrayBuffer` inputs
+we wrap the entire buffer.
+
+## Features
+
+- Accepts both Node `Buffer` and standard `ArrayBuffer` inputs.
+- Zero-copy: never allocates a new backing buffer.
+- Honours `Buffer.subarray` offsets correctly (avoids a common foot-gun).
+- Pure synchronous function with no dependencies.
+
+## Usage: Basic Example
+
+```ts
+import { bufferToUint8Array } from "@suaveplan/streams/utils/buffer/buffer-to-uint8/buffer-to-uint8";
+
+const u8 = bufferToUint8Array(Buffer.from("hello"));
+console.log(u8.byteLength); // 5
+```
+
+## Usage: Advanced Example
+
+```ts
+import { bufferToUint8Array } from "@suaveplan/streams/utils/buffer/buffer-to-uint8/buffer-to-uint8";
+
+// Coming from `node:fs/promises#readFile` (returns Buffer in Node)
+async function loadAsU8(path: string): Promise<Uint8Array> {
+    const { readFile } = await import("node:fs/promises");
+    return bufferToUint8Array(await readFile(path));
+}
+
+// Coming from a browser `Response.arrayBuffer()`
+async function fetchAsU8(url: string): Promise<Uint8Array> {
+    const ab = await (await fetch(url)).arrayBuffer();
+    return bufferToUint8Array(ab);
+}
+```
+
+## API Reference
+
+### `bufferToUint8Array(buffer: Buffer | ArrayBuffer): Uint8Array`
+
+- **`buffer`** — either a Node `Buffer` (a `Uint8Array` subclass with possibly
+  non-zero `byteOffset`) or a standard DOM/ES `ArrayBuffer`.
+- **Returns** — a `Uint8Array` view over the same memory as the input.
+- **Throws** — never; type narrowing is performed via `instanceof ArrayBuffer`.
+
+## Implementation Notes
+
+- The `instanceof ArrayBuffer` check intentionally precedes the `Buffer` branch
+  because `Buffer instanceof ArrayBuffer` is `false` in Node (Buffer extends
+  `Uint8Array`, not `ArrayBuffer`).
+- The returned view shares memory with the source. If you need an independent
+  copy, follow up with `.slice()` or pair this helper with
+  `createUint8ArraySlice`.
+- Universal: this module has no Node-only or DOM-only imports.

packages/foundation/flow/streams/src/utils/buffer/buffer-to-uint8/buffer-to-uint8.test.ts

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { bufferToUint8Array } from "./buffer-to-uint8.js";
+
+describe("bufferToUint8Array", () => {
+    it("converts an ArrayBuffer to a Uint8Array", { timeout: 5000 }, () => {
+        const ab = new ArrayBuffer(4);
+        new Uint8Array(ab).set([10, 20, 30, 40]);
+        const out = bufferToUint8Array(ab);
+        expect(out).toBeInstanceOf(Uint8Array);
+        expect(Array.from(out)).toEqual([10, 20, 30, 40]);
+    });
+
+    it("converts a Node Buffer to a Uint8Array", { timeout: 5000 }, () => {
+        const buf = Buffer.from([1, 2, 3]);
+        const out = bufferToUint8Array(buf);
+        expect(Array.from(out)).toEqual([1, 2, 3]);
+    });
+
+    it("respects Buffer byteOffset / byteLength on a sub-region", {
+        timeout: 5000,
+    }, () => {
+        const big = Buffer.from([1, 2, 3, 4, 5, 6, 7, 8]);
+        const slice = big.subarray(2, 6); // [3,4,5,6]
+        const out = bufferToUint8Array(slice);
+        expect(Array.from(out)).toEqual([3, 4, 5, 6]);
+    });
+
+    it("returns a zero-length array for empty input", { timeout: 5000 }, () => {
+        expect(bufferToUint8Array(new ArrayBuffer(0)).byteLength).toBe(0);
+        expect(bufferToUint8Array(Buffer.alloc(0)).byteLength).toBe(0);
+    });
+
+    it("Buffer view shares memory with the source", { timeout: 5000 }, () => {
+        const buf = Buffer.from([5, 6, 7]);
+        const out = bufferToUint8Array(buf);
+        out[0] = 99;
+        expect(buf[0]).toBe(99);
+    });
+});

packages/foundation/flow/streams/src/utils/buffer/buffer-to-uint8/buffer-to-uint8.ts

@@ -0,0 +1,14 @@
+/**
+ * Converts a Node `Buffer` or a standard `ArrayBuffer` into a `Uint8Array`.
+ *
+ * For `Buffer` inputs the result is a zero-copy view over the same backing
+ * memory (respecting `byteOffset` / `byteLength`). For `ArrayBuffer` inputs the
+ * result wraps the entire buffer.
+ */
+export function bufferToUint8Array(buffer: Buffer | ArrayBuffer): Uint8Array {
+    if (buffer instanceof ArrayBuffer) {
+        return new Uint8Array(buffer);
+    }
+    // Node's Buffer is a Uint8Array subclass with a possibly non-zero byteOffset.
+    return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+}

packages/foundation/flow/streams/src/utils/buffer/buffer-to-uint8/index.ts

@@ -0,0 +1 @@
+export { bufferToUint8Array } from "./buffer-to-uint8.js";

packages/foundation/flow/streams/src/utils/buffer/get-data-from-file/get-data-from-file.md

@@ -0,0 +1,75 @@
+# getDataFromFile
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+`getDataFromFile` reads a file from the local filesystem and returns its contents
+as a plain `Uint8Array`. It is a thin Node-only convenience for the streams
+package: the streams package is environment-agnostic, but many byte-oriented
+helpers (chunking, framing, hashing) need a way to load test fixtures and
+real-world inputs from disk without each call site re-implementing the dynamic
+import + error-mapping dance.
+
+The function is browser-safe in the sense that importing it does not pull in
+`node:fs` eagerly. The Node module is loaded lazily inside the function body and
+calling it in a non-Node environment surfaces a structured
+`StreamFileBrowserNotSupportedError` rather than an opaque module-resolution
+crash.
+
+## Features
+
+- Lazy `node:fs/promises` import — zero overhead on the browser bundle path.
+- Runtime environment detection via `globalThis.process?.versions?.node`.
+- Returns a fresh `Uint8Array` decoupled from Node's pooled `Buffer` allocator.
+- Structured errors with the original `cause` and the offending `filePath`.
+- Stable `STREAM.FILE_BROWSER_NOT_SUPPORTED` and `STREAM.FILE_READ_FAILED` codes.
+
+## Usage: Basic Example
+
+```ts
+import { getDataFromFile } from "@suaveplan/streams/utils/buffer/get-data-from-file/get-data-from-file";
+
+const bytes = await getDataFromFile("/etc/hostname");
+console.log(bytes.byteLength);
+```
+
+## Usage: Advanced Example
+
+```ts
+import { getDataFromFile } from "@suaveplan/streams/utils/buffer/get-data-from-file/get-data-from-file";
+import { StreamFileReadFailedError } from "@suaveplan/streams/errors/stream-error";
+
+try {
+    const data = await getDataFromFile("/no/such/file");
+    process.stdout.write(data);
+} catch (err) {
+    if (err instanceof StreamFileReadFailedError) {
+        console.error("read failed", err.context, err.cause);
+    } else {
+        throw err;
+    }
+}
+```
+
+## API Reference
+
+### `getDataFromFile(filePath: string): Promise<Uint8Array>`
+
+- **`filePath`** — absolute or relative path resolved against `process.cwd()`.
+- **Returns** — `Promise<Uint8Array>` containing the file's exact bytes.
+- **Throws** — `StreamFileBrowserNotSupportedError` outside Node;
+  `StreamFileReadFailedError` (with `cause` and `context.filePath`) on any
+  underlying I/O failure (ENOENT, EACCES, EISDIR, …).
+
+## Implementation Notes
+
+- The dynamic `import("node:fs/promises")` keeps the universal entry-point free
+  of Node-only deps; bundlers that target the browser tree-shake this branch
+  away, and the runtime guard prevents accidental invocation.
+- The returned array is sliced off the underlying `ArrayBuffer` so consumers
+  never receive a `Buffer` subclass — useful when feeding the result into APIs
+  that strictly type-check `Uint8Array`.
+- All thrown errors extend `SuaveplanError` and round-trip through the standard
+  redaction / wire-format adapters supplied by `@suaveplan/error`.

packages/foundation/flow/streams/src/utils/buffer/get-data-from-file/get-data-from-file.test.ts

@@ -0,0 +1,71 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+    afterAll,
+    beforeAll,
+    describe,
+    expect,
+    it,
+} from "@suaveplan/testing/runner";
+import {
+    StreamFileBrowserNotSupportedError,
+    StreamFileReadFailedError,
+} from "../../../errors/stream-error.js";
+import { getDataFromFile } from "./get-data-from-file.js";
+
+describe("getDataFromFile", () => {
+    let dir: string;
+    let filePath: string;
+
+    beforeAll(async () => {
+        dir = await mkdtemp(join(tmpdir(), "stream-getdata-"));
+        filePath = join(dir, "sample.bin");
+        await writeFile(filePath, Buffer.from([1, 2, 3, 4, 5]));
+    });
+
+    afterAll(async () => {
+        await rm(dir, { recursive: true, force: true });
+    });
+
+    it("reads file contents as Uint8Array", { timeout: 5000 }, async () => {
+        const data = await getDataFromFile(filePath);
+        expect(data).toBeInstanceOf(Uint8Array);
+        expect(Array.from(data)).toEqual([1, 2, 3, 4, 5]);
+    });
+
+    it("returns an independent Uint8Array (not a Buffer alias)", {
+        timeout: 5000,
+    }, async () => {
+        const data = await getDataFromFile(filePath);
+        // Modifying returned array must not throw and must be a plain Uint8Array
+        expect(data.constructor.name).toBe("Uint8Array");
+        data[0] = 99;
+        const fresh = await getDataFromFile(filePath);
+        expect(fresh[0]).toBe(1);
+    });
+
+    it("throws StreamFileReadFailedError on missing file", {
+        timeout: 5000,
+    }, async () => {
+        await expect(
+            getDataFromFile(join(dir, "does-not-exist.bin"))
+        ).rejects.toBeInstanceOf(StreamFileReadFailedError);
+    });
+
+    it("throws StreamFileBrowserNotSupportedError when not in Node", {
+        timeout: 5000,
+    }, async () => {
+        const g = globalThis as { process?: unknown };
+        const original = g.process;
+        // @ts-expect-error - simulate non-node
+        g.process = undefined;
+        try {
+            await expect(getDataFromFile(filePath)).rejects.toBeInstanceOf(
+                StreamFileBrowserNotSupportedError
+            );
+        } finally {
+            g.process = original;
+        }
+    });
+});

packages/foundation/flow/streams/src/utils/buffer/get-data-from-file/get-data-from-file.ts

@@ -0,0 +1,33 @@
+import {
+    StreamFileBrowserNotSupportedError,
+    StreamFileReadFailedError,
+} from "../../../errors/stream-error.js";
+
+/**
+ * Reads a file from disk into a `Uint8Array`. Node-only.
+ *
+ * Throws {@link StreamFileBrowserNotSupportedError} when invoked outside Node.js,
+ * and {@link StreamFileReadFailedError} when the underlying read fails.
+ */
+export async function getDataFromFile(filePath: string): Promise<Uint8Array> {
+    const isNode = Boolean(
+        (globalThis as { process?: { versions?: { node?: string } } }).process
+            ?.versions?.node
+    );
+    if (!isNode) {
+        throw new StreamFileBrowserNotSupportedError();
+    }
+    try {
+        const fs = await import("node:fs/promises");
+        const buf = await fs.readFile(filePath);
+        // Buffer is a Uint8Array view — copy into a fresh Uint8Array so callers
+        // own a plain Uint8Array rather than a Buffer subclass tied to a pool.
+        return new Uint8Array(
+            buf.buffer,
+            buf.byteOffset,
+            buf.byteLength
+        ).slice();
+    } catch (err) {
+        throw new StreamFileReadFailedError({ cause: err, filePath });
+    }
+}

packages/foundation/flow/streams/src/utils/buffer/get-data-from-file/index.ts

@@ -0,0 +1 @@
+export { getDataFromFile } from "./get-data-from-file.js";

packages/foundation/flow/streams/src/utils/buffer/uint8-slice/index.ts

@@ -0,0 +1 @@
+export { createUint8ArraySlice } from "./uint8-slice.js";

packages/foundation/flow/streams/src/utils/buffer/uint8-slice/uint8-slice.md

@@ -0,0 +1,68 @@
+# createUint8ArraySlice
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+`createUint8ArraySlice` returns an independent copy of a sub-range of a
+`Uint8Array`. The streams package favours explicit copy-vs-view semantics over
+the conventional `subarray` (view) / `slice` (copy) ambiguity that has bitten
+many consumers writing both browser and Node code. This helper makes the
+intent obvious at the call site: when you call `createUint8ArraySlice`, you
+get back memory you fully own.
+
+It is functionally a thin wrapper around `Uint8Array.prototype.slice`, but
+named to communicate the copy semantics, and centralised so that future
+optimisations (e.g. fast-path detection of zero-length, large-buffer pooling)
+have a single place to live.
+
+## Features
+
+- Always copies — never returns a view aliasing the source buffer.
+- Clamps `end` past `length` per the standard `slice` algorithm.
+- Returns an empty `Uint8Array` when `start === end`.
+- Pure, synchronous, dependency-free.
+
+## Usage: Basic Example
+
+```ts
+import { createUint8ArraySlice } from "@suaveplan/streams/utils/buffer/uint8-slice/uint8-slice";
+
+const src = new Uint8Array([1, 2, 3, 4, 5]);
+const middle = createUint8ArraySlice(src, 1, 4); // Uint8Array(3) [2, 3, 4]
+```
+
+## Usage: Advanced Example
+
+```ts
+import { createUint8ArraySlice } from "@suaveplan/streams/utils/buffer/uint8-slice/uint8-slice";
+
+// Frame parser: each frame owns its bytes so the surrounding ring buffer can
+// be recycled without stalling downstream consumers.
+function parseFrames(buf: Uint8Array, frameSize: number): Uint8Array[] {
+    const out: Uint8Array[] = [];
+    for (let i = 0; i + frameSize <= buf.byteLength; i += frameSize) {
+        out.push(createUint8ArraySlice(buf, i, i + frameSize));
+    }
+    return out;
+}
+```
+
+## API Reference
+
+### `createUint8ArraySlice(array, start, end): Uint8Array`
+
+- **`array: Uint8Array`** — source buffer.
+- **`start: number`** — inclusive start index.
+- **`end: number`** — exclusive end index; clamped to `array.length`.
+- **Returns** — a fresh `Uint8Array` of length `max(0, end - start)`.
+- **Throws** — never.
+
+## Implementation Notes
+
+- Implemented as `array.slice(start, end)`. The function exists because the
+  copy-vs-view distinction matters for streams correctness (subarray would
+  hold the entire upstream buffer alive) and a named helper documents intent
+  better than an inline `.slice` call would.
+- Universal: zero environment dependencies.

packages/foundation/flow/streams/src/utils/buffer/uint8-slice/uint8-slice.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { createUint8ArraySlice } from "./uint8-slice.js";
+
+describe("createUint8ArraySlice", () => {
+    it("returns a slice of the given range", { timeout: 5000 }, () => {
+        const src = new Uint8Array([10, 20, 30, 40, 50]);
+        const out = createUint8ArraySlice(src, 1, 4);
+        expect(Array.from(out)).toEqual([20, 30, 40]);
+    });
+
+    it("produces an independent copy (no shared memory)", {
+        timeout: 5000,
+    }, () => {
+        const src = new Uint8Array([1, 2, 3, 4]);
+        const out = createUint8ArraySlice(src, 0, 4);
+        out[0] = 99;
+        expect(src[0]).toBe(1);
+    });
+
+    it("returns empty when start === end", { timeout: 5000 }, () => {
+        const src = new Uint8Array([1, 2, 3]);
+        expect(createUint8ArraySlice(src, 1, 1).byteLength).toBe(0);
+    });
+
+    it("clamps end past length", { timeout: 5000 }, () => {
+        const src = new Uint8Array([1, 2, 3]);
+        expect(Array.from(createUint8ArraySlice(src, 0, 99))).toEqual([
+            1, 2, 3,
+        ]);
+    });
+
+    it("supports zero-length input", { timeout: 5000 }, () => {
+        const src = new Uint8Array(0);
+        expect(createUint8ArraySlice(src, 0, 0).byteLength).toBe(0);
+    });
+});

packages/foundation/flow/streams/src/utils/buffer/uint8-slice/uint8-slice.ts

@@ -0,0 +1,13 @@
+/**
+ * Returns an independent COPY of the slice `[start, end)` of `array`.
+ *
+ * Unlike `subarray`, the returned `Uint8Array` does not share memory with the
+ * source; mutating either side leaves the other untouched.
+ */
+export function createUint8ArraySlice(
+    array: Uint8Array,
+    start: number,
+    end: number
+): Uint8Array {
+    return array.slice(start, end);
+}

packages/foundation/flow/streams/src/utils/stream/create-data-stream/create-data-stream.md

@@ -0,0 +1,81 @@
+# createDataStream
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+`createDataStream` is the canonical bridge from "I have a typed in-memory
+value" to "I have a `ReadableStream<Uint8Array>`". It is the read-side counter
+to `processDataStream`: each accepts a transformer that bridges between the
+domain type `T` and the byte representation, and they compose cleanly when a
+pipeline needs to enter and leave the byte world without callers spelling out
+the encode/decode boilerplate.
+
+The transformer runs once when the function is called. The function returns a
+`Promise<ReadableStream<Uint8Array>>` so that asynchronous serialisation
+(JSON.stringify of a complex value, msgpack encoding, gzip pre-compression)
+fits naturally into the call signature without forcing every consumer to wrap
+the producer in a `start()` callback.
+
+## Features
+
+- Sync and async transformer functions both supported.
+- Pluggable chunk size via the underlying `uint8ArrayToStream`.
+- Type-safe generic input — the compiler infers `T` from the value supplied.
+- Propagates transformer errors as the rejection of the returned promise.
+
+## Usage: Basic Example
+
+```ts
+import { createDataStream } from "@suaveplan/streams/utils/stream/create-data-stream/create-data-stream";
+
+const stream = await createDataStream(
+    { hello: "world" },
+    (obj) => new TextEncoder().encode(JSON.stringify(obj))
+);
+```
+
+## Usage: Advanced Example
+
+```ts
+import { createDataStream } from "@suaveplan/streams/utils/stream/create-data-stream/create-data-stream";
+import { encode } from "@msgpack/msgpack";
+
+interface Telemetry {
+    ts: number;
+    metrics: Record<string, number>;
+}
+
+async function telemetryStream(t: Telemetry) {
+    return createDataStream<Telemetry>(t, (v) => encode(v), {
+        chunkSize: 4096,
+    });
+}
+
+const stream = await telemetryStream({
+    ts: Date.now(),
+    metrics: { rps: 42, p99: 18.7 },
+});
+```
+
+## API Reference
+
+### `createDataStream<T>(data, transformer, options?)`
+
+- **`data: T`** — value to serialise.
+- **`transformer: (data: T) => Uint8Array | Promise<Uint8Array>`** — encoder.
+- **`options?: { chunkSize?: number }`** — emission chunk size, forwarded to
+  `uint8ArrayToStream`.
+- **Returns** — `Promise<ReadableStream<Uint8Array>>`.
+- **Throws** — anything the transformer throws (rejected promise).
+
+## Implementation Notes
+
+- The transformer runs exactly once before the stream is constructed, so the
+  byte count is known up-front; this lets downstream sinks pre-allocate when
+  they want to.
+- The function does not accept an `AbortSignal` — cancellation belongs to the
+  drain side (`streamToUint8Array`, `processDataStream`) where it can actually
+  influence work in flight.
+- Universal: zero environment dependencies.

packages/foundation/flow/streams/src/utils/stream/create-data-stream/create-data-stream.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { streamToUint8Array } from "../round-trip/round-trip.js";
+import { createDataStream } from "./create-data-stream.js";
+
+describe("createDataStream", () => {
+    it("applies a sync transformer and emits the bytes", {
+        timeout: 5000,
+    }, async () => {
+        const stream = await createDataStream("hi", (s) =>
+            new TextEncoder().encode(s)
+        );
+        const out = await streamToUint8Array(stream);
+        expect(new TextDecoder().decode(out)).toBe("hi");
+    });
+
+    it("awaits an async transformer", { timeout: 5000 }, async () => {
+        const stream = await createDataStream(
+            { n: 4 },
+            async (v) => new Uint8Array([v.n, v.n * 2])
+        );
+        const out = await streamToUint8Array(stream);
+        expect(Array.from(out)).toEqual([4, 8]);
+    });
+
+    it("respects chunkSize", { timeout: 5000 }, async () => {
+        const stream = await createDataStream(
+            null,
+            () => new Uint8Array([1, 2, 3, 4, 5, 6]),
+            { chunkSize: 2 }
+        );
+        const reader = stream.getReader();
+        const sizes: number[] = [];
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done) break;
+            sizes.push((value as Uint8Array).byteLength);
+        }
+        expect(sizes).toEqual([2, 2, 2]);
+    });
+
+    it("propagates transformer errors", { timeout: 5000 }, async () => {
+        await expect(
+            createDataStream<number>(1, () => {
+                throw new Error("nope");
+            })
+        ).rejects.toThrow("nope");
+    });
+
+    it("handles empty bytes", { timeout: 5000 }, async () => {
+        const stream = await createDataStream(0, () => new Uint8Array(0));
+        const out = await streamToUint8Array(stream);
+        expect(out.byteLength).toBe(0);
+    });
+});

packages/foundation/flow/streams/src/utils/stream/create-data-stream/create-data-stream.ts

@@ -0,0 +1,21 @@
+import {
+    type Uint8ArrayToStreamOptions,
+    uint8ArrayToStream,
+} from "../round-trip/round-trip.js";
+
+/**
+ * Builds a `ReadableStream<Uint8Array>` from arbitrary input `data` by running
+ * `transformer` once to materialise the bytes, then emitting them via
+ * {@link uint8ArrayToStream}.
+ *
+ * The transformer may be sync or async — the returned stream resolves only
+ * once the bytes are ready to emit.
+ */
+export async function createDataStream<T>(
+    data: T,
+    transformer: (data: T) => Uint8Array | Promise<Uint8Array>,
+    options?: Uint8ArrayToStreamOptions
+): Promise<ReadableStream<Uint8Array>> {
+    const bytes = await transformer(data);
+    return uint8ArrayToStream(bytes, options);
+}

packages/foundation/flow/streams/src/utils/stream/create-data-stream/index.ts

@@ -0,0 +1 @@
+export { createDataStream } from "./create-data-stream.js";

packages/foundation/flow/streams/src/utils/stream/process-data-stream/index.ts

@@ -0,0 +1 @@
+export { processDataStream } from "./process-data-stream.js";

packages/foundation/flow/streams/src/utils/stream/process-data-stream/process-data-stream.md

@@ -0,0 +1,71 @@
+# processDataStream
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+`processDataStream` is the drain-side complement to `createDataStream`. It
+collects every byte emitted by a `ReadableStream<Uint8Array>` into a single
+`Uint8Array` and hands the result to a processor function that maps the bytes
+into a domain value `T`. It exists so consumers can write straight-line code —
+"give me the parsed result" — without explicitly looping over the reader.
+
+The function is intentionally small but centralising it pays off: it standardises
+how upstream errors propagate, how an empty stream is represented (as an empty
+`Uint8Array`, not `undefined`), and gives observability/instrumentation a
+single chokepoint to wrap when needed.
+
+## Features
+
+- Accepts sync and async processor functions.
+- Returns the processor's resolved value with full type inference.
+- Concatenates multiple chunks transparently before processing.
+- Propagates upstream stream errors and processor throws as a rejected promise.
+
+## Usage: Basic Example
+
+```ts
+import { processDataStream } from "@suaveplan/streams/utils/stream/process-data-stream/process-data-stream";
+
+const text = await processDataStream(stream, (b) =>
+    new TextDecoder().decode(b)
+);
+```
+
+## Usage: Advanced Example
+
+```ts
+import { processDataStream } from "@suaveplan/streams/utils/stream/process-data-stream/process-data-stream";
+import { decode } from "@msgpack/msgpack";
+
+interface Record {
+    id: string;
+    payload: unknown;
+}
+
+async function readMsgpack<T = Record>(stream: ReadableStream<Uint8Array>) {
+    return processDataStream(stream, async (bytes) => {
+        // hand off to a worker thread for large payloads
+        return decode(bytes) as T;
+    });
+}
+```
+
+## API Reference
+
+### `processDataStream<T>(stream, processor): Promise<T>`
+
+- **`stream: ReadableStream<Uint8Array>`** — source of bytes to drain.
+- **`processor: (data: Uint8Array) => T | Promise<T>`** — function that maps
+  the concatenated bytes into a domain value.
+- **Returns** — `Promise<T>` resolving with `processor`'s return value.
+- **Throws** — rejects with whatever the underlying stream or processor throws.
+
+## Implementation Notes
+
+- Built on top of `streamToUint8Array`; the same single-allocation behaviour
+  applies (chunks pre-summed before allocation).
+- The processor is awaited even when its return type is synchronous to keep a
+  single promise contract for callers.
+- Universal — no Node or DOM globals beyond the WHATWG Streams API.

packages/foundation/flow/streams/src/utils/stream/process-data-stream/process-data-stream.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { uint8ArrayToStream } from "../round-trip/round-trip.js";
+import { processDataStream } from "./process-data-stream.js";
+
+describe("processDataStream", () => {
+    it("drains the stream and passes bytes to the processor", {
+        timeout: 5000,
+    }, async () => {
+        const stream = uint8ArrayToStream(new TextEncoder().encode("hello"));
+        const out = await processDataStream(stream, (b) =>
+            new TextDecoder().decode(b)
+        );
+        expect(out).toBe("hello");
+    });
+
+    it("supports async processors", { timeout: 5000 }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array([1, 2, 3, 4]));
+        const sum = await processDataStream(stream, async (b) => {
+            await Promise.resolve();
+            return b.reduce((a, n) => a + n, 0);
+        });
+        expect(sum).toBe(10);
+    });
+
+    it("propagates processor errors", { timeout: 5000 }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array([1]));
+        await expect(
+            processDataStream(stream, () => {
+                throw new Error("boom");
+            })
+        ).rejects.toThrow("boom");
+    });
+
+    it("handles empty stream", { timeout: 5000 }, async () => {
+        const empty = new ReadableStream<Uint8Array>({
+            start(c) {
+                c.close();
+            },
+        });
+        const len = await processDataStream(empty, (b) => b.byteLength);
+        expect(len).toBe(0);
+    });
+
+    it("concatenates multiple chunks before invoking processor", {
+        timeout: 5000,
+    }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array([1, 2, 3, 4, 5, 6]), {
+            chunkSize: 2,
+        });
+        const out = await processDataStream(stream, (b) => Array.from(b));
+        expect(out).toEqual([1, 2, 3, 4, 5, 6]);
+    });
+});

packages/foundation/flow/streams/src/utils/stream/process-data-stream/process-data-stream.ts

@@ -0,0 +1,13 @@
+import { streamToUint8Array } from "../round-trip/round-trip.js";
+
+/**
+ * Drains `stream` into a `Uint8Array` and feeds it to `processor`. Returns
+ * the processor's resolved value.
+ */
+export async function processDataStream<T>(
+    stream: ReadableStream<Uint8Array>,
+    processor: (data: Uint8Array) => T | Promise<T>
+): Promise<T> {
+    const bytes = await streamToUint8Array(stream);
+    return await processor(bytes);
+}

packages/foundation/flow/streams/src/utils/stream/round-trip/index.ts

@@ -0,0 +1,5 @@
+export {
+    streamToUint8Array,
+    type Uint8ArrayToStreamOptions,
+    uint8ArrayToStream,
+} from "./round-trip.js";

packages/foundation/flow/streams/src/utils/stream/round-trip/round-trip.md

@@ -0,0 +1,91 @@
+# round-trip (streamToUint8Array / uint8ArrayToStream)
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+The round-trip module provides the two halves of byte ↔ stream conversion that
+underpin most file/network adapters in `@suaveplan/streams`. `uint8ArrayToStream`
+turns an in-memory buffer into a backpressure-respecting `ReadableStream`, and
+`streamToUint8Array` drains an arbitrary readable stream back into a single
+`Uint8Array`. They are deliberately co-located because most consumers reach for
+both in the same call path (e.g. test fixtures that load a buffer, push it
+through a transform pipeline, then re-collect the output for assertion).
+
+The drain side is `AbortSignal`-aware and rejects with `StreamAbortedError`
+when cancellation is requested, cancelling the underlying reader so upstream
+producers can release resources promptly.
+
+## Features
+
+- `uint8ArrayToStream`: configurable chunk size; defaults to a single emission.
+- Per-chunk copy emission so consumers can safely retain references.
+- `streamToUint8Array`: total-length pre-sum, single allocation for output.
+- Honours `AbortSignal` both before and during read; cancels the reader.
+- Pure WHATWG Streams API — no Node-only or DOM-only deps.
+
+## Usage: Basic Example
+
+```ts
+import {
+    streamToUint8Array,
+    uint8ArrayToStream,
+} from "@suaveplan/streams/utils/stream/round-trip/round-trip";
+
+const original = new Uint8Array([1, 2, 3, 4]);
+const drained = await streamToUint8Array(uint8ArrayToStream(original));
+// drained equals original
+```
+
+## Usage: Advanced Example
+
+```ts
+import {
+    streamToUint8Array,
+    uint8ArrayToStream,
+} from "@suaveplan/streams/utils/stream/round-trip/round-trip";
+
+async function pipeWithTimeout(
+    data: Uint8Array,
+    transform: TransformStream<Uint8Array, Uint8Array>,
+    ms: number
+): Promise<Uint8Array> {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(new Error("timeout")), ms);
+    try {
+        const piped = uint8ArrayToStream(data, { chunkSize: 4096 }).pipeThrough(
+            transform
+        );
+        return await streamToUint8Array(piped, ac.signal);
+    } finally {
+        clearTimeout(timer);
+    }
+}
+```
+
+## API Reference
+
+### `uint8ArrayToStream(data, options?)`
+
+- **`data: Uint8Array`** — bytes to emit.
+- **`options.chunkSize?: number`** — bytes per chunk; defaults to entire array.
+- **Returns** — `ReadableStream<Uint8Array>` that closes after the last chunk.
+
+### `streamToUint8Array(stream, signal?)`
+
+- **`stream: ReadableStream<Uint8Array>`** — source stream.
+- **`signal?: AbortSignal`** — cancellation token; cancelling rejects the call
+  with `StreamAbortedError` and cancels the reader.
+- **Returns** — `Promise<Uint8Array>` of the concatenated bytes.
+- **Throws** — `StreamAbortedError` on cancellation.
+
+## Implementation Notes
+
+- The drain pre-collects chunks and sums lengths so the final `Uint8Array` is
+  allocated exactly once — avoids the quadratic-copy trap of repeatedly
+  appending into a growing buffer.
+- Each emission of `uint8ArrayToStream` is a copy via `data.slice`, so the
+  source `Uint8Array` may be mutated independently after the stream is built.
+- The abort listener is removed in `finally` to keep the signal listener count
+  bounded across long-lived signals.

packages/foundation/flow/streams/src/utils/stream/round-trip/round-trip.test.ts

@@ -0,0 +1,89 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { StreamAbortedError } from "../../../errors/stream-error.js";
+import { streamToUint8Array, uint8ArrayToStream } from "./round-trip.js";
+
+describe("uint8ArrayToStream", () => {
+    it("emits the full array as a single chunk by default", {
+        timeout: 5000,
+    }, async () => {
+        const data = new Uint8Array([1, 2, 3, 4, 5]);
+        const stream = uint8ArrayToStream(data);
+        const reader = stream.getReader();
+        const first = await reader.read();
+        expect(first.done).toBe(false);
+        expect(Array.from(first.value as Uint8Array)).toEqual([1, 2, 3, 4, 5]);
+        const second = await reader.read();
+        expect(second.done).toBe(true);
+    });
+
+    it("emits in chunks of chunkSize", { timeout: 5000 }, async () => {
+        const data = new Uint8Array([1, 2, 3, 4, 5]);
+        const stream = uint8ArrayToStream(data, { chunkSize: 2 });
+        const reader = stream.getReader();
+        const collected: number[] = [];
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done) break;
+            collected.push((value as Uint8Array).byteLength);
+        }
+        expect(collected).toEqual([2, 2, 1]);
+    });
+
+    it("handles empty input by closing immediately", {
+        timeout: 5000,
+    }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array(0));
+        const { done } = await stream.getReader().read();
+        expect(done).toBe(true);
+    });
+});
+
+describe("streamToUint8Array", () => {
+    it("round-trips a Uint8Array through uint8ArrayToStream", {
+        timeout: 5000,
+    }, async () => {
+        const data = new Uint8Array([9, 8, 7, 6, 5, 4, 3, 2, 1]);
+        const stream = uint8ArrayToStream(data, { chunkSize: 3 });
+        const out = await streamToUint8Array(stream);
+        expect(Array.from(out)).toEqual(Array.from(data));
+    });
+
+    it("returns empty Uint8Array for an empty stream", {
+        timeout: 5000,
+    }, async () => {
+        const stream = new ReadableStream<Uint8Array>({
+            start(c) {
+                c.close();
+            },
+        });
+        const out = await streamToUint8Array(stream);
+        expect(out.byteLength).toBe(0);
+    });
+
+    it("rejects with StreamAbortedError when signal is already aborted", {
+        timeout: 5000,
+    }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array([1, 2, 3]));
+        const ac = new AbortController();
+        ac.abort(new Error("nope"));
+        await expect(
+            streamToUint8Array(stream, ac.signal)
+        ).rejects.toBeInstanceOf(StreamAbortedError);
+    });
+
+    it("rejects with StreamAbortedError when signal fires mid-read", {
+        timeout: 5000,
+    }, async () => {
+        const ac = new AbortController();
+        const stream = new ReadableStream<Uint8Array>({
+            start(c) {
+                c.enqueue(new Uint8Array([1, 2, 3]));
+                // never closes — abort path drives termination
+            },
+        });
+        queueMicrotask(() => ac.abort(new Error("stop")));
+        await expect(
+            streamToUint8Array(stream, ac.signal)
+        ).rejects.toBeInstanceOf(StreamAbortedError);
+    });
+});

packages/foundation/flow/streams/src/utils/stream/round-trip/round-trip.ts

@@ -0,0 +1,82 @@
+import { StreamAbortedError } from "../../../errors/stream-error.js";
+
+/**
+ * Drains `stream` into a single `Uint8Array`. Honours `signal` for cancellation
+ * (cancels the underlying reader and rejects with {@link StreamAbortedError}).
+ */
+export async function streamToUint8Array(
+    stream: ReadableStream<Uint8Array>,
+    signal?: AbortSignal
+): Promise<Uint8Array> {
+    if (signal?.aborted) {
+        throw new StreamAbortedError({ cause: signal.reason });
+    }
+    const reader = stream.getReader();
+    const chunks: Uint8Array[] = [];
+    let total = 0;
+
+    const onAbort = (): void => {
+        void reader.cancel(signal?.reason);
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+
+    try {
+        while (true) {
+            if (signal?.aborted) {
+                throw new StreamAbortedError({ cause: signal.reason });
+            }
+            const { value, done } = await reader.read();
+            if (done) break;
+            chunks.push(value);
+            total += value.byteLength;
+        }
+    } finally {
+        signal?.removeEventListener("abort", onAbort);
+        reader.releaseLock();
+    }
+
+    const out = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        out.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return out;
+}
+
+export interface Uint8ArrayToStreamOptions {
+    /** Maximum bytes per emitted chunk. Defaults to the entire array (single chunk). */
+    chunkSize?: number;
+}
+
+/**
+ * Builds a `ReadableStream<Uint8Array>` that emits `data` in chunks of at most
+ * `chunkSize` bytes. When `chunkSize` is omitted the entire array is emitted
+ * as a single chunk.
+ */
+export function uint8ArrayToStream(
+    data: Uint8Array,
+    options: Uint8ArrayToStreamOptions = {}
+): ReadableStream<Uint8Array> {
+    const chunkSize =
+        options.chunkSize && options.chunkSize > 0
+            ? options.chunkSize
+            : data.byteLength || 1;
+
+    let offset = 0;
+    return new ReadableStream<Uint8Array>({
+        pull(controller) {
+            if (offset >= data.byteLength) {
+                controller.close();
+                return;
+            }
+            const end = Math.min(offset + chunkSize, data.byteLength);
+            // Emit a copy so consumers can safely retain references.
+            controller.enqueue(data.slice(offset, end));
+            offset = end;
+            if (offset >= data.byteLength) {
+                controller.close();
+            }
+        },
+    });
+}

packages/foundation/flow/streams/src/utils/stream/validate-integrity/index.ts

@@ -0,0 +1 @@
+export { validateStreamIntegrity } from "./validate-integrity.js";

packages/foundation/flow/streams/src/utils/stream/validate-integrity/validate-integrity.md

@@ -0,0 +1,77 @@
+# validateStreamIntegrity
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+## Purpose
+
+`validateStreamIntegrity` walks a `ReadableStream<Uint8Array>` chunk by chunk
+and applies a caller-supplied predicate to each chunk. It returns `true` if
+every chunk satisfies the predicate and the stream closes cleanly, or `false`
+the first time the predicate rejects a chunk. On a rejection it cancels the
+underlying reader so upstream producers can release resources rather than
+continuing to fill a buffer that will be discarded.
+
+The function is the streams-package equivalent of "fold a boolean over the
+chunks". It is small but worth centralising because the short-circuit + reader
+cancel + lock-release dance is easy to get wrong, especially when combined
+with WHATWG Streams' lock semantics around early teardown.
+
+## Features
+
+- Short-circuits on first failure — does not consume the rest of the stream.
+- Cancels the underlying reader on failure to free upstream resources.
+- Always releases the reader lock via `finally`, even on validator throws.
+- Returns `true` for an empty stream (vacuous truth).
+- Synchronous predicate keeps the per-chunk cost minimal.
+
+## Usage: Basic Example
+
+```ts
+import { validateStreamIntegrity } from "@suaveplan/streams/utils/stream/validate-integrity/validate-integrity";
+
+const allEven = await validateStreamIntegrity(stream, (chunk) =>
+    chunk.every((b) => b % 2 === 0)
+);
+```
+
+## Usage: Advanced Example
+
+```ts
+import { validateStreamIntegrity } from "@suaveplan/streams/utils/stream/validate-integrity/validate-integrity";
+
+// Each frame: 4-byte big-endian length || N payload bytes || 1-byte checksum.
+// Verify checksum without buffering the whole stream.
+function checksummed(chunk: Uint8Array): boolean {
+    if (chunk.byteLength < 5) return false;
+    const dv = new DataView(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+    const len = dv.getUint32(0);
+    if (chunk.byteLength !== 4 + len + 1) return false;
+    let sum = 0;
+    for (let i = 0; i < 4 + len; i++) sum = (sum + chunk[i]!) & 0xff;
+    return sum === chunk[chunk.byteLength - 1];
+}
+
+const ok = await validateStreamIntegrity(stream, checksummed);
+```
+
+## API Reference
+
+### `validateStreamIntegrity(stream, validate): Promise<boolean>`
+
+- **`stream: ReadableStream<Uint8Array>`** — source stream.
+- **`validate: (chunk: Uint8Array) => boolean`** — predicate run per chunk.
+- **Returns** — `Promise<boolean>` resolving to `true` if all chunks pass,
+  `false` on first failure.
+- **Throws** — rejects with whatever the underlying stream throws.
+
+## Implementation Notes
+
+- The reader is acquired via `getReader()` and released in `finally`. On the
+  failure path we additionally `await reader.cancel()` to ensure the upstream
+  source observes the cancellation before we resolve `false`.
+- The predicate is invoked exactly once per non-empty chunk; chunks of size 0
+  still trigger a call (a zero-byte chunk is observable in the WHATWG model
+  and a strict validator may want to reject it).
+- A predicate `throw` propagates as a rejection rather than a `false`; callers
+  who wish otherwise can wrap their predicate in a try/catch.

packages/foundation/flow/streams/src/utils/stream/validate-integrity/validate-integrity.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { uint8ArrayToStream } from "../round-trip/round-trip.js";
+import { validateStreamIntegrity } from "./validate-integrity.js";
+
+describe("validateStreamIntegrity", () => {
+    it("returns true when every chunk passes validation", {
+        timeout: 5000,
+    }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array([2, 4, 6, 8]), {
+            chunkSize: 1,
+        });
+        const ok = await validateStreamIntegrity(
+            stream,
+            (chunk) => chunk[0]! % 2 === 0
+        );
+        expect(ok).toBe(true);
+    });
+
+    it("returns false on the first invalid chunk", {
+        timeout: 5000,
+    }, async () => {
+        const stream = uint8ArrayToStream(new Uint8Array([2, 4, 5, 6]), {
+            chunkSize: 1,
+        });
+        const ok = await validateStreamIntegrity(
+            stream,
+            (chunk) => chunk[0]! % 2 === 0
+        );
+        expect(ok).toBe(false);
+    });
+
+    it("short-circuits — does not read beyond the failing chunk", {
+        timeout: 5000,
+    }, async () => {
+        let pulls = 0;
+        const stream = new ReadableStream<Uint8Array>({
+            pull(c) {
+                pulls++;
+                if (pulls === 1) c.enqueue(new Uint8Array([1]));
+                else if (pulls === 2)
+                    c.enqueue(new Uint8Array([99])); // fails
+                else c.enqueue(new Uint8Array([3]));
+            },
+        });
+        const ok = await validateStreamIntegrity(
+            stream,
+            (chunk) => chunk[0]! < 10
+        );
+        expect(ok).toBe(false);
+        expect(pulls).toBeLessThanOrEqual(3);
+    });
+
+    it("returns true on empty stream", { timeout: 5000 }, async () => {
+        const empty = new ReadableStream<Uint8Array>({
+            start(c) {
+                c.close();
+            },
+        });
+        const ok = await validateStreamIntegrity(empty, () => false);
+        expect(ok).toBe(true);
+    });
+
+    it("supports complex validators", { timeout: 5000 }, async () => {
+        // Treat each chunk as a checksummed frame: last byte is sum of others mod 256
+        const frame1 = new Uint8Array([1, 2, 3, 6]);
+        const frame2 = new Uint8Array([10, 20, 30, 60]);
+        const stream = new ReadableStream<Uint8Array>({
+            start(c) {
+                c.enqueue(frame1);
+                c.enqueue(frame2);
+                c.close();
+            },
+        });
+        const ok = await validateStreamIntegrity(stream, (chunk) => {
+            const sum = chunk.slice(0, -1).reduce((a, n) => a + n, 0);
+            return (sum & 0xff) === chunk[chunk.byteLength - 1];
+        });
+        expect(ok).toBe(true);
+    });
+});

packages/foundation/flow/streams/src/utils/stream/validate-integrity/validate-integrity.ts

@@ -0,0 +1,23 @@
+/**
+ * Validates each chunk of `stream` against `validate`. Returns `false` on the
+ * first invalid chunk (short-circuits and cancels the reader); returns `true`
+ * if the stream closes with every chunk passing validation.
+ */
+export async function validateStreamIntegrity(
+    stream: ReadableStream<Uint8Array>,
+    validate: (chunk: Uint8Array) => boolean
+): Promise<boolean> {
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { value, done } = await reader.read();
+            if (done) return true;
+            if (value && !validate(value)) {
+                await reader.cancel();
+                return false;
+            }
+        }
+    } finally {
+        reader.releaseLock();
+    }
+}