Initial Commit

This commit is contained in:
2026-05-27 23:47:53 +01:00
parent fbc70f96e6
commit 9b28db724f
18103 changed files with 1600554 additions and 0 deletions

View File

@@ -0,0 +1,89 @@
# @suaveplan/bitpacker-streaming - Bitpacker streaming plugin
## Summary
Streaming APIs for bitpacker including chunked packing/unpacking and true TransformStream support with framing.
<!-- begin:auto-docs -->
## Documentation tree
- [defaults](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [register-schemas](./src/register-schemas/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
- [register-schemas](./src/register-schemas/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - transform.schema — Zod schemas for the option bundl…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - transform.schema — Zod schemas for the option bundl…
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
<!-- end:auto-docs --> - [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_ - [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ - [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ - [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
## Install
```bash
bun add @suaveplan/bitpacker-streaming
```
## Usage
```typescript
import { BitPacker } from "@suaveplan/bitpacker";
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
import { packToChunks, unpackFromChunks, createFramedPackStream, createFramedUnpackStream } from "@suaveplan/bitpacker-streaming";
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
// Chunked packing
const chunks = packToChunks(42, schema, { chunkSize: 2 });
const value = unpackFromChunks<number>(chunks, schema);
// Transform streams with framing
const packStream = createFramedPackStream(bp, schema);
const unpackStream = createFramedUnpackStream(bp, schema);
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read(); // 42
```
## Modules
- `src/index.ts` — `packToChunks`, `unpackFromChunks`, `unpackFromChunksAsync`
- `src/transform/transform.ts` — `TransformStream` factories with length-prefix, delimiter, fixed-size, and no-framing options

View File

@@ -0,0 +1,60 @@
{
"name": "@suaveplan/bitpacker-streaming",
"version": "0.1.0",
"type": "module",
"sideEffects": false,
"description": "Bitpacker streaming plugin",
"license": "MIT",
"author": "Suaveplan Team",
"main": "./dist/index.js",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
"./package.json": "./package.json",
".": {
"import": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
},
"require": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"./register-schemas": {
"import": {
"types": "./dist/register-schemas/index.d.ts",
"default": "./dist/register-schemas/index.js"
},
"require": {
"types": "./dist/register-schemas/index.d.ts",
"default": "./dist/register-schemas/index.js"
}
}
},
"scripts": {
"build": "vite build",
"test": "vitest run --passWithNoTests",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage --passWithNoTests",
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/"
},
"dependencies": {
"@opentelemetry/api": "catalog:observability",
"@suaveplan/bitpacker": "workspace:^",
"@suaveplan/bitpacker-strategies": "workspace:^",
"@suaveplan/schema-registry": "workspace:^",
"@suaveplan/types": "workspace:^",
"tslib": "catalog:build",
"zod": "catalog:forms"
},
"devDependencies": {
"@suaveplan/testing": "workspace:^",
"@types/node": "catalog:",
"typescript": "catalog:",
"vite": "catalog:build",
"vite-plugin-dts": "catalog:build",
"vitest": "catalog:testing"
}
}

View File

@@ -0,0 +1,55 @@
# defaultClock
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
`defaultClock` is the single point in `@suaveplan/bitpacker-streaming` where the wall clock is read. Every other module that needs a time reading either accepts a `Clock` from the caller or imports this default. Centralising the `Date.now()` call this way makes the package deterministic-by-default in tests (callers inject `FakeClock`) while the production hot path still reads the real wall clock.
This pattern matches the convention used by `@suaveplan/bitpacker-network` and `@suaveplan/bitpacker-compressed`. Suaveplan enforces a "no module-scope `Date.now()`/`Math.random()`" rule outside allow-listed defaults — this file is one of those allow-listed defaults.
## Features
- Single named export `defaultClock: Clock` typed via `@suaveplan/types`.
- Pure function — no side effects beyond reading the wall clock.
- Idiomatic Suaveplan convention for clock injection.
## Usage: Basic Example
```ts
import { defaultClock } from "@suaveplan/bitpacker-streaming";
const now = defaultClock();
```
## Usage: Advanced Example
```ts
import type { Clock } from "@suaveplan/types";
import { StreamingBitPacker } from "@suaveplan/bitpacker-streaming";
// In tests, inject a deterministic clock instead of the default.
const fakeClock: Clock = (() => {
let t = 0;
return () => {
t += 100;
return t;
};
})();
const packer = new StreamingBitPacker({ x: 255 });
await packer.packStream({ x: 1 }, { clock: fakeClock });
```
## API Reference
### `defaultClock: Clock`
Returns the current wall-clock time in milliseconds (`Date.now()`).
## Implementation Notes
The `Clock` type is `() => number`. suaveplan prefers function types over class instances for monotonic primitives because they are trivially serialisable, comparable, and substitutable in tests without dependency injection frameworks.
The reason this lives in `defaults/` rather than as a static class member is the suaveplan lint rule that bans module-scope `Date.now()` calls in production code. By scoping the call to a single named, lint-allow-listed module, we keep the rest of the package free of static analysis exceptions.

View File

@@ -0,0 +1,17 @@
/**
* Package-default `Clock` for streaming throughput timing.
*
* Centralising the single `Date.now()` call here means the rest of the
* package never reads the wall clock directly — every site that needs a
* time reading either accepts an injectable `Clock` from the caller or
* imports this default. This is the same convention used by
* `@suaveplan/bitpacker-network` and `@suaveplan/bitpacker-compressed`.
*/
import type { Clock } from "@suaveplan/types";
/**
* The package-wide default `Clock`. Returns `Date.now()`. Tests should
* inject a `FakeClock` from `@suaveplan/testing` instead of using this.
*/
export const defaultClock: Clock = () => Date.now();

View File

@@ -0,0 +1,8 @@
## index
**Path:** `bitpacker-streaming/src/index.ts`
**Export:** See source for details.
### What it does
Part of the @suaveplan/ package.

View File

@@ -0,0 +1,99 @@
import { BitPacker } from "@suaveplan/bitpacker";
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
import type { BitPackerSchema } from "@suaveplan/types";
export interface StreamingOptions {
readonly chunkSize?: number;
readonly signal?: AbortSignal;
}
export function packToChunks(
data: unknown,
schema: BitPackerSchema,
options: StreamingOptions = {}
): Uint8Array[] {
const { chunkSize = 1024 } = options;
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const bytes = bp.pack(data, schema);
const chunks: Uint8Array[] = [];
for (let offset = 0; offset < bytes.length; offset += chunkSize) {
chunks.push(bytes.slice(offset, offset + chunkSize));
}
return chunks;
}
export function unpackFromChunks<T = unknown>(
chunks: Iterable<Uint8Array>,
schema: BitPackerSchema
): T {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const arrays: Uint8Array[] = [];
let totalLength = 0;
for (const chunk of chunks) {
if (chunk.length === 0) continue;
arrays.push(chunk);
totalLength += chunk.length;
}
const combined = new Uint8Array(totalLength);
let offset = 0;
for (const arr of arrays) {
combined.set(arr, offset);
offset += arr.length;
}
return bp.unpack(combined, schema) as T;
}
export async function unpackFromChunksAsync<T = unknown>(
chunks: AsyncIterable<Uint8Array>,
schema: BitPackerSchema,
options: StreamingOptions = {}
): Promise<T> {
const { signal } = options;
const arrays: Uint8Array[] = [];
let totalLength = 0;
for await (const chunk of chunks) {
if (signal?.aborted) throw new Error("Stream aborted");
if (chunk.length === 0) continue;
arrays.push(chunk);
totalLength += chunk.length;
}
const combined = new Uint8Array(totalLength);
let offset = 0;
for (const arr of arrays) {
combined.set(arr, offset);
offset += arr.length;
}
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
return bp.unpack(combined, schema) as T;
}
export { defaultClock } from "./defaults/default-clock.js";
export {
type ChunkProcessor,
type ReadableStreamFromBytesOptions,
StreamingBitPacker,
type StreamingBitPackerOptions,
type StreamingPackOptions,
type StreamingUnpackOptions,
type WritablePackOptions,
type WritableUnpackOptions,
} from "./streaming-bit-packer/streaming-bit-packer.js";
export {
addFrame,
type BitPackerTransformer,
createFramedPackStream,
createFramedUnpackStream,
createPackTransformStream,
createUnpackTransformStream,
type FrameParseResult,
type FramingConfig,
type PackTransformOptions,
type PipelineStage,
parseFrames,
type StreamTransformer,
type TransformProgress,
type UnpackTransformOptions,
} from "./transform/transform.js";

View File

@@ -0,0 +1,84 @@
# Bitpacker Streaming OTel Instruments
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
OTel meter, tracer, counter, and histogram declarations for `@suaveplan/bitpacker-streaming`. Ships canonical instrumentation primitives for the streaming pack/unpack pipeline plus the framing layer (`length-prefix`, `delimiter`, `fixed-size`, `none`). Centralizing the metric/span vocabulary here lets every entry point — `StreamingBitPacker.packStream`, `createPackTransformStream`, `createFramedUnpackStream` — emit consistent observability data.
The metric and span names follow the suaveplan convention with snake-case for the multi-word package: tracer/meter named `@suaveplan/bitpacker-streaming`; spans named `bitpacker_streaming.<operation>`; counters/histograms named `bitpacker_streaming.<scope>.<metric>`.
## Features
- `getBitPackerStreamingTracer()` — returns the package's tracer
- `streamingOpsTotal` — counter labelled by `op` and `success`
- `streamingOpDurationMs` — histogram of pack/unpack duration
- `streamingChunkBytes` — histogram of bytes-per-chunk on the pipeline
- `streamingFramesTotal` — counter labelled by `framing` strategy
- `BITPACKER_STREAMING_SPAN_NAMES` — four canonical span names
## Usage: Basic Example
```ts
import {
BITPACKER_STREAMING_SPAN_NAMES,
streamingOpsTotal,
getBitPackerStreamingTracer,
} from "@suaveplan/bitpacker-streaming";
const tracer = getBitPackerStreamingTracer();
tracer.startActiveSpan(BITPACKER_STREAMING_SPAN_NAMES.pack, (span) => {
try {
// pack work
streamingOpsTotal.add(1, { op: "pack", success: "true" });
} finally {
span.end();
}
});
```
## Usage: Advanced Example
```ts
import {
streamingChunkBytes,
streamingFramesTotal,
} from "@suaveplan/bitpacker-streaming";
function recordChunk(chunk: Uint8Array, framing: "length-prefix" | "delimiter") {
streamingChunkBytes.record(chunk.byteLength, { op: "pack" });
streamingFramesTotal.add(1, { framing });
}
```
## API Reference
### `getBitPackerStreamingTracer(): Tracer`
Returns the OTel `Tracer` instance for the package.
### `streamingOpsTotal: Counter`
Counter labelled by `op: "pack" | "unpack"` and `success: "true" | "false"`.
### `streamingOpDurationMs: Histogram`
Histogram of elapsed milliseconds per pack/unpack call, labelled by `op`.
### `streamingChunkBytes: Histogram`
Histogram of bytes per chunk on the pipeline, labelled by `op`.
### `streamingFramesTotal: Counter`
Counter labelled by `framing: "length-prefix" | "delimiter" | "fixed-size" | "none"`.
### `BITPACKER_STREAMING_SPAN_NAMES`
Const map: `{ pack, unpack, framedPack, framedUnpack }` → canonical span names.
## Implementation Notes
- **Per-chunk events, not per-chunk spans.** A streaming pack producing 10k chunks would produce 10k spans if each were its own; instead, chunks are recorded as `streamingChunkBytes` histogram observations and as nested span events on the parent pack/unpack span.
- **Framing is a counter label.** `streamingFramesTotal{framing}` lets dashboards group frame throughput by framing strategy without spawning separate counters per type.

View File

@@ -0,0 +1,46 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import {
BITPACKER_STREAMING_SPAN_NAMES,
getBitPackerStreamingTracer,
streamingChunkBytes,
streamingFramesTotal,
streamingOpDurationMs,
streamingOpsTotal,
} from "./instruments.js";
describe("OTel instruments — declarations", () => {
it("exposes the canonical span names", () => {
expect(BITPACKER_STREAMING_SPAN_NAMES.pack).toBe(
"bitpacker_streaming.pack"
);
expect(BITPACKER_STREAMING_SPAN_NAMES.unpack).toBe(
"bitpacker_streaming.unpack"
);
expect(BITPACKER_STREAMING_SPAN_NAMES.framedPack).toBe(
"bitpacker_streaming.framed_pack"
);
expect(BITPACKER_STREAMING_SPAN_NAMES.framedUnpack).toBe(
"bitpacker_streaming.framed_unpack"
);
});
it("counters and histograms are constructed without error", () => {
expect(streamingOpsTotal).toBeDefined();
expect(streamingOpDurationMs).toBeDefined();
expect(streamingChunkBytes).toBeDefined();
expect(streamingFramesTotal).toBeDefined();
});
it("tracer is reachable", () => {
expect(getBitPackerStreamingTracer()).toBeDefined();
});
it("counters accept the documented labels", () => {
expect(() =>
streamingOpsTotal.add(1, { op: "pack", success: "true" })
).not.toThrow();
expect(() =>
streamingFramesTotal.add(1, { framing: "length-prefix" })
).not.toThrow();
});
});

View File

@@ -0,0 +1,61 @@
import { metrics, type Tracer, trace } from "@opentelemetry/api";
const meter = metrics.getMeter("@suaveplan/bitpacker-streaming");
export function getBitPackerStreamingTracer(): Tracer {
return trace.getTracer("@suaveplan/bitpacker-streaming");
}
/**
* Counter incremented once per top-level streaming pack/unpack call. Labels:
* `op` (`pack` | `unpack`) and `success` (`true` | `false`).
*/
export const streamingOpsTotal = meter.createCounter(
"bitpacker_streaming.ops.total",
{
description: "Total streaming pack/unpack operations",
unit: "{ops}",
}
);
/**
* Histogram of streaming-op duration in milliseconds, labelled by `op`.
*/
export const streamingOpDurationMs = meter.createHistogram(
"bitpacker_streaming.op.duration_ms",
{
description: "Duration of streaming pack/unpack in milliseconds",
unit: "ms",
}
);
/**
* Histogram of bytes-per-chunk emitted on the streaming pipeline, labelled
* by `op`.
*/
export const streamingChunkBytes = meter.createHistogram(
"bitpacker_streaming.chunk.bytes",
{
description: "Bytes per chunk on the streaming pipeline",
unit: "By",
}
);
/**
* Counter incremented per frame parsed by the framing pipeline, labelled by
* `framing` (`length-prefix` | `delimiter` | `fixed-size` | `none`).
*/
export const streamingFramesTotal = meter.createCounter(
"bitpacker_streaming.frames.total",
{
description: "Total frames parsed by the streaming framing pipeline",
unit: "{frames}",
}
);
export const BITPACKER_STREAMING_SPAN_NAMES = {
pack: "bitpacker_streaming.pack",
unpack: "bitpacker_streaming.unpack",
framedPack: "bitpacker_streaming.framed_pack",
framedUnpack: "bitpacker_streaming.framed_unpack",
} as const;

View File

@@ -0,0 +1,5 @@
export {
BITPACKER_STREAMING_SCHEMA_VERSION,
BITPACKER_STREAMING_SUBJECT,
registerBitPackerStreamingSchemas,
} from "./register-schemas.js";

View File

@@ -0,0 +1,66 @@
# Register Schemas — `@suaveplan/bitpacker-streaming`
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `registerBitPackerStreamingSchemas(registry)` which converts the package's nine public boundary schemas into JSON Schema form and registers them under `bitpacker:@suaveplan/bitpacker-streaming` at version 1. Re-registration on the same registry is a no-op.
The nine schemas covered are: the six streaming-bit-packer option bundles (`streamingBitPackerOptions`, `streamingPackOptions`, `streamingUnpackOptions`, `readableStreamFromBytesOptions`, `writablePackOptions`, `writableUnpackOptions`) plus the three transform-stream schemas (`framingConfig`, `packTransformOptions`, `unpackTransformOptions`).
## Features
- `BITPACKER_STREAMING_SUBJECT` constant — `"bitpacker:@suaveplan/bitpacker-streaming"`
- `BITPACKER_STREAMING_SCHEMA_VERSION` constant — `1`
- `registerBitPackerStreamingSchemas(registry)` — idempotent registration
- Nine-key payload covering every option bundle the package accepts
## Usage: Basic Example
```ts
import { createRegistry } from "@suaveplan/schema-registry";
import {
BITPACKER_STREAMING_SUBJECT,
registerBitPackerStreamingSchemas,
} from "@suaveplan/bitpacker-streaming/register-schemas";
const registry = createRegistry();
registerBitPackerStreamingSchemas(registry);
```
## Usage: Advanced Example
```ts
import { createRegistry, SchemaRegistry } from "@suaveplan/schema-registry";
import { registerBitPackerSchemas } from "@suaveplan/bitpacker/register-schemas";
import { registerBitPackerStreamingSchemas } from "@suaveplan/bitpacker-streaming/register-schemas";
const registry = createRegistry();
registerBitPackerSchemas(registry);
registerBitPackerStreamingSchemas(registry);
const payload = registry.getLatest("bitpacker:@suaveplan/bitpacker-streaming")
.schema as Record<string, unknown>;
// payload.framingConfig, payload.streamingPackOptions, …
```
## API Reference
### `BITPACKER_STREAMING_SUBJECT: "bitpacker:@suaveplan/bitpacker-streaming"`
Const string.
### `BITPACKER_STREAMING_SCHEMA_VERSION: 1`
Const number.
### `registerBitPackerStreamingSchemas(registry: SchemaRegistry): void`
Idempotent registration. Returns without writing if the subject is already present.
## Implementation Notes
- **Nine-key payload.** Each key corresponds to one of the package's exported schemas.
- **`unrepresentable: "any"`.** Required because the schemas use `z.custom` for `AbortSignal`, callables, and `Uint8Array` checks.

View File

@@ -0,0 +1,44 @@
import { createRegistry } from "@suaveplan/schema-registry";
import { describe, expect, it } from "@suaveplan/testing/runner";
import {
BITPACKER_STREAMING_SCHEMA_VERSION,
BITPACKER_STREAMING_SUBJECT,
registerBitPackerStreamingSchemas,
} from "./register-schemas.js";
describe("registerBitPackerStreamingSchemas", () => {
it("registers under the namespaced subject at version 1", () => {
const registry = createRegistry();
registerBitPackerStreamingSchemas(registry);
expect(registry.getLatest(BITPACKER_STREAMING_SUBJECT).version).toBe(
BITPACKER_STREAMING_SCHEMA_VERSION
);
});
it("includes every public schema in the payload", () => {
const registry = createRegistry();
registerBitPackerStreamingSchemas(registry);
const payload = registry.getLatest(BITPACKER_STREAMING_SUBJECT)
.schema as Record<string, unknown>;
for (const key of [
"streamingBitPackerOptions",
"streamingPackOptions",
"streamingUnpackOptions",
"readableStreamFromBytesOptions",
"writablePackOptions",
"writableUnpackOptions",
"framingConfig",
"packTransformOptions",
"unpackTransformOptions",
]) {
expect(payload).toHaveProperty(key);
}
});
it("is idempotent across repeated calls", () => {
const registry = createRegistry();
registerBitPackerStreamingSchemas(registry);
registerBitPackerStreamingSchemas(registry);
expect(registry.getLatest(BITPACKER_STREAMING_SUBJECT).version).toBe(1);
});
});

View File

@@ -0,0 +1,82 @@
import type { SchemaRegistry } from "@suaveplan/schema-registry";
import z from "zod";
import {
readableStreamFromBytesOptionsSchema,
streamingBitPackerOptionsSchema,
streamingPackOptionsSchema,
streamingUnpackOptionsSchema,
writablePackOptionsSchema,
writableUnpackOptionsSchema,
} from "../streaming-bit-packer/streaming-bit-packer.schema.js";
import {
framingConfigSchema,
packTransformOptionsSchema,
unpackTransformOptionsSchema,
} from "../transform/transform.schema.js";
export const BITPACKER_STREAMING_SUBJECT =
"bitpacker:@suaveplan/bitpacker-streaming" as const;
export const BITPACKER_STREAMING_SCHEMA_VERSION = 1;
interface Payload {
readonly streamingBitPackerOptions: unknown;
readonly streamingPackOptions: unknown;
readonly streamingUnpackOptions: unknown;
readonly readableStreamFromBytesOptions: unknown;
readonly writablePackOptions: unknown;
readonly writableUnpackOptions: unknown;
readonly framingConfig: unknown;
readonly packTransformOptions: unknown;
readonly unpackTransformOptions: unknown;
}
function buildPayload(): Payload {
const opts = { unrepresentable: "any" } as const;
return {
streamingBitPackerOptions: z.toJSONSchema(
streamingBitPackerOptionsSchema,
opts
),
streamingPackOptions: z.toJSONSchema(streamingPackOptionsSchema, opts),
streamingUnpackOptions: z.toJSONSchema(
streamingUnpackOptionsSchema,
opts
),
readableStreamFromBytesOptions: z.toJSONSchema(
readableStreamFromBytesOptionsSchema,
opts
),
writablePackOptions: z.toJSONSchema(writablePackOptionsSchema, opts),
writableUnpackOptions: z.toJSONSchema(
writableUnpackOptionsSchema,
opts
),
framingConfig: z.toJSONSchema(framingConfigSchema, opts),
packTransformOptions: z.toJSONSchema(packTransformOptionsSchema, opts),
unpackTransformOptions: z.toJSONSchema(
unpackTransformOptionsSchema,
opts
),
};
}
/**
* Idempotently register `@suaveplan/bitpacker-streaming`'s public Zod
* schemas with the given `SchemaRegistry`.
*/
export function registerBitPackerStreamingSchemas(
registry: SchemaRegistry
): void {
if (registry.listSubjects().includes(BITPACKER_STREAMING_SUBJECT)) {
return;
}
registry.register(
BITPACKER_STREAMING_SUBJECT,
buildPayload(),
"json-schema",
{
package: "@suaveplan/bitpacker-streaming",
namespace: "bitpacker",
}
);
}

View File

@@ -0,0 +1,129 @@
# StreamingBitPacker
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs --> - streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- streaming-bit-packer.schema — Zod schemas for the o…
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
## Purpose
`StreamingBitPacker<T>` is the class-shaped façade that lets callers process pack/unpack operations as chunked pipelines instead of single-shot calls. It owns an underlying `BitPacker<T>` and surfaces five entry points that mirror the upstream `bitpacker/src/streaming/streaming.ts` API: `packStream`, `unpackStream`, `createReadableStream`, `createPackingStream`, and `createUnpackingStream`. Wire-format compatibility is preserved — bytes produced by `packStream` round-trip through `unpackStream` exactly as bytes produced by `BitPacker.pack` round-trip through `BitPacker.unpack`.
The streaming surface is for callers that need progress callbacks, per-chunk processing (compression, encryption), or backpressure semantics — anything where the single-shot `pack/unpack` is too coarse. For everything else, prefer the simpler chunk helpers (`packToChunks` / `unpackFromChunks`) that already live in this package's `index.ts`.
## Features
- Five entry points covering the chunked-pipeline matrix: synchronous-with-known-input (`packStream`/`unpackStream`), readable-side (`createReadableStream`), and writable-side with in-flight pack/unpack (`createPackingStream`/`createUnpackingStream`).
- Optional `chunkProcessor` runs between bit-packing and emission, so the same façade composes cleanly with downstream compression/encryption stages without callers needing to wire the WHATWG streams API by hand.
- Progress reported via the canonical `TransformProgress` shape from `@suaveplan/types` — observability tools speak one vocabulary across every Suaveplan pipeline.
- Injectable `Clock` for throughput timing; tests run with a `FakeClock` to avoid real-timer flakiness.
- `BitpackerError` codes (`STREAM_ABORTED`, `STREAM_CHUNK_PROCESSOR_FAILED`, `STREAM_INCOMPLETE_DATA`) for every failure mode, replacing upstream's plain `Error` throws.
- Default packing strategies registered automatically (opt-out via `registerDefaults: false`).
- AbortSignal supported on all `*Stream` methods — caller can cancel mid-pipeline.
## Usage: Basic Example
```ts
import { StreamingBitPacker } from "@suaveplan/bitpacker-streaming";
const packer = new StreamingBitPacker({ x: 255, y: 255 });
const packed = await packer.packStream({ x: 42, y: 99 });
const decoded = await packer.unpackStream<{ x: number; y: number }>(packed);
// → { x: 42, y: 99 }
```
## Usage: Advanced Example
```ts
import { StreamingBitPacker } from "@suaveplan/bitpacker-streaming";
import type { TransformProgress } from "@suaveplan/types";
const packer = new StreamingBitPacker({
position: { x: "f32", y: "f32", z: "f32" },
velocity: { x: "f32", y: "f32", z: "f32" },
});
const progressLog: TransformProgress[] = [];
const onProgress = (p: TransformProgress): void => {
progressLog.push(p);
};
// chunkProcessor runs between bit-pack and emission — compose with any
// downstream codec without touching the WHATWG streams API directly.
const xor = (chunk: Uint8Array): Uint8Array => {
const out = new Uint8Array(chunk.length);
for (let i = 0; i < chunk.length; i++) out[i] = (chunk[i] as number) ^ 0xaa;
return out;
};
const packed = await packer.packStream(
{
position: { x: 1, y: 2, z: 3 },
velocity: { x: 0.1, y: 0, z: 0 },
},
{
chunkSize: 4,
chunkProcessor: xor,
onProgress,
}
);
// Reverse path with the same processor undoes the XOR before unpack.
const decoded = await packer.unpackStream(packed, {
chunkSize: 4,
chunkProcessor: xor,
});
```
## API Reference
### `new StreamingBitPacker<T>(schema, options?)`
`schema` — a `BitPackerSchema` describing the data shape.
`options.registerDefaults?` — `boolean`. Default `true`. When `false`, the underlying `BitPacker` ships with no strategies; the caller must register their own before any pack/unpack call.
### `packStream(data, options?) → Promise<Uint8Array>`
Bit-packs `data`, walks the bytes in `chunkSize`-byte chunks (default `1024`), runs each chunk through `options.chunkProcessor` if supplied, and returns the concatenated bytes. `onProgress` fires once per chunk. Throws `BitpackerError(STREAM_ABORTED)` if `options.signal` aborts before or mid-loop.
### `unpackStream<R>(data, options?) → Promise<R>`
Reverse of `packStream`. Walks `data` in chunks (running the inverse processor if supplied), concatenates, and bit-unpacks.
### `createReadableStream(data, options?) → ReadableStream<Uint8Array>`
Returns a WHATWG `ReadableStream<Uint8Array>` that emits `data` in chunks. Useful for piping into another transform (`pipeThrough(compressionStream)`). Emits all chunks during `start()` and immediately closes — back-pressure is delegated to the consumer.
### `createPackingStream(options?) → { stream, result }`
`stream` is a `WritableStream<unknown>` that packs each `write(item)` and pushes the bytes into a buffer. `result` resolves to the concatenated packed bytes when the stream closes. If `chunkProcessor` is supplied, each packed chunk is run through it before being buffered. `result` rejects with `BitpackerError(STREAM_ABORTED)` on `abort()`.
### `createUnpackingStream<R>(options?) → { stream, result }`
`stream` is a `WritableStream<Uint8Array>` that pushes each `write(bytes)` (optionally through `chunkProcessor`) into a buffer. On `close()`, the buffer is concatenated and bit-unpacked. `result` rejects with `BitpackerError(STREAM_INCOMPLETE_DATA)` if the stream closes empty, or `STREAM_ABORTED` on `abort()`.
### Inherited
`bitPacker` — public readonly. The underlying `BitPacker<T>` instance. Useful for callers wanting to register a custom strategy or schema.
## Implementation Notes
The port drops upstream's `EventEmitter` inheritance because `BitPacker` does not extend an emitter and the `'progress'` event fires the same payload `onProgress` would have received — the callback is the simpler shape. This also avoids the duplicate-listener footgun upstream callers occasionally hit.
`validateChunks` from upstream is gone. `BitPacker` does not expose `validate()` or `getValidationErrors()` — those upstream helpers were thin wrappers over schema-introspection logic that suaveplan pushes onto `@suaveplan/validation` callers. If pre-pack validation is needed, run a Zod schema at the call site before invoking `packStream`.
The `runChunkPipeline` helper is shared between `packStream` and `unpackStream` because the per-chunk loop logic — chunk slice, optional processor, progress emit — is identical in both directions. Diverging the two paths would duplicate ~30 lines of logic for no semantic gain.
`createPackingStream` and `createUnpackingStream` deliberately do not auto-emit on `start` (unlike `createReadableStream`) — they are _writable_ sides and emission timing is fully driven by caller writes. The `result` promise pattern lets callers `await` the closure without needing a separate completion signal.

View File

@@ -0,0 +1,99 @@
# Streaming BitPacker Schemas
## Purpose
Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpackStream`, `readableStreamFromBytes`, the writable-side equivalents). Each schema is the runtime gatekeeper at the corresponding constructor or method boundary so a typo'd flag (`registerDefualts`) or a mis-typed `signal` (a plain object instead of an `AbortSignal`) fails at parse time instead of breaking the chunk pipeline halfway through.
The schemas use `z.custom<T>((v): v is T => …)` for `AbortSignal`, the chunk-processor callback, the progress callback, and the structural `Clock` so referential identity is preserved end-to-end. This matters for cancellation (`signal === user.signal` comparisons elsewhere in the chain), for callback registration (the same function reference must reach progress reporters), and for clock injection (test fakes need the same instance their assertions reference).
## Features
- `streamingBitPackerOptionsSchema` — strict, today only `registerDefaults`
- `streamingPackOptionsSchema` — `chunkSize`, `chunkProcessor`, `onProgress`, `clock`, `signal`
- `streamingUnpackOptionsSchema` — same shape as pack (re-export)
- `readableStreamFromBytesOptionsSchema` — source-side subset (no chunkProcessor, no signal)
- `writablePackOptionsSchema` — sink-side subset (no chunkSize, no signal)
- `writableUnpackOptionsSchema` — same shape as writable pack (re-export)
- Identity-preserving `z.custom` for callables, AbortSignal, and Clock duck-types
- `as unknown as z.ZodType<T>` casts for `exactOptionalPropertyTypes` mismatches
## Usage: Basic Example
```ts
import {
StreamingBitPacker,
streamingBitPackerOptionsSchema,
streamingPackOptionsSchema,
} from "@suaveplan/bitpacker-streaming";
const constructorOpts = streamingBitPackerOptionsSchema.parse({
registerDefaults: true,
});
const packer = new StreamingBitPacker({ x: 255 }, constructorOpts);
const ac = new AbortController();
const callOpts = streamingPackOptionsSchema.parse({
chunkSize: 1024,
signal: ac.signal,
});
const bytes = await packer.packStream({ x: 100 }, callOpts);
```
## Usage: Advanced Example
```ts
import {
StreamingBitPacker,
streamingPackOptionsSchema,
writablePackOptionsSchema,
} from "@suaveplan/bitpacker-streaming";
function safelyPack(packer: StreamingBitPacker, raw: unknown) {
const result = streamingPackOptionsSchema.safeParse(raw);
if (!result.success) {
throw new Error(
`Streaming pack options rejected: ${result.error.message}`
);
}
return packer.packStream({ x: 0 }, result.data);
}
// Writable-side callback wired through the progress channel:
const writableOpts = writablePackOptionsSchema.parse({
chunkProcessor: (chunk) => chunk,
onProgress: (p) => console.log(p),
});
```
## API Reference
### `streamingBitPackerOptionsSchema: z.ZodType<StreamingBitPackerOptions>`
Strict schema for the constructor bundle. `registerDefaults` is an optional boolean.
### `streamingPackOptionsSchema: z.ZodType<StreamingPackOptions>`
Strict schema for the per-call options. `chunkSize` is a positive integer; `chunkProcessor` and `onProgress` are functions (identity-preserved); `clock` is a structural `{ now(): number }`; `signal` is an `AbortSignal` instance.
### `streamingUnpackOptionsSchema: z.ZodType<StreamingUnpackOptions>`
Same shape as `streamingPackOptionsSchema`. `StreamingUnpackOptions` is an interface that extends `StreamingPackOptions` with no additional fields.
### `readableStreamFromBytesOptionsSchema: z.ZodType<ReadableStreamFromBytesOptions>`
Strict schema. Accepts `chunkSize`, `onProgress`, and `clock`. `chunkProcessor` is rejected because it is not part of the source-side `ReadableStream` API.
### `writablePackOptionsSchema: z.ZodType<WritablePackOptions>`
Strict schema. Accepts `chunkProcessor`, `onProgress`, `clock`. `chunkSize` and `signal` are rejected because chunk boundaries and cancellation are controlled upstream by the writable consumer.
### `writableUnpackOptionsSchema: z.ZodType<WritableUnpackOptions>`
Same shape as `writablePackOptionsSchema`.
## Implementation Notes
- **Identity for callables.** All callback fields are validated with `z.custom<Fn>((v): v is Fn => typeof v === "function", …)` so the same instance flows through. `z.function()` in Zod 4 wraps the callable and breaks identity for the chunk pipeline, which depends on `===` comparisons during sequential chunk dispatch.
- **`AbortSignal` instance check.** `z.custom<AbortSignal>((v): v is AbortSignal => v instanceof AbortSignal, …)` rejects duck-typed signals. The cancellation path checks `signal.aborted` via the prototype, so a non-`AbortSignal` would silently fail to cancel.
- **`Clock` is duck-typed.** `clock` is `z.custom<{ now(): number }>`. Tests inject a `FakeClock` from `@suaveplan/testing` that satisfies the shape without inheriting from a class. Reflecting that flexibility in the schema avoids over-coupling.
- **Subset schemas reject extra fields.** `readableStreamFromBytesOptionsSchema` rejects `chunkProcessor`; `writablePackOptionsSchema` rejects `chunkSize`/`signal`. This isn't decorative — it surfaces miscompositions where a caller copies a `StreamingPackOptions` bundle into a writable-side method.

View File

@@ -0,0 +1,107 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import {
readableStreamFromBytesOptionsSchema,
streamingBitPackerOptionsSchema,
streamingPackOptionsSchema,
streamingUnpackOptionsSchema,
writablePackOptionsSchema,
writableUnpackOptionsSchema,
} from "./streaming-bit-packer.schema.js";
describe("streamingBitPackerOptionsSchema", () => {
it("accepts an empty bundle", () => {
expect(streamingBitPackerOptionsSchema.parse({})).toEqual({});
});
it("accepts registerDefaults", () => {
expect(
streamingBitPackerOptionsSchema.parse({ registerDefaults: false })
).toEqual({ registerDefaults: false });
});
it("rejects unknown keys", () => {
expect(() =>
streamingBitPackerOptionsSchema.parse({ unknownKey: true })
).toThrow();
});
});
describe("streamingPackOptionsSchema", () => {
it("accepts an AbortSignal", () => {
const ac = new AbortController();
const opts = { signal: ac.signal, chunkSize: 64 };
const parsed = streamingPackOptionsSchema.parse(opts);
expect(parsed.signal).toBe(ac.signal);
expect(parsed.chunkSize).toBe(64);
});
it("preserves callback identity", () => {
const onProgress = () => undefined;
const parsed = streamingPackOptionsSchema.parse({ onProgress });
expect(parsed.onProgress).toBe(onProgress);
});
it("rejects a non-AbortSignal signal", () => {
expect(() =>
streamingPackOptionsSchema.parse({ signal: { aborted: false } })
).toThrow();
});
it("rejects zero chunkSize", () => {
expect(() =>
streamingPackOptionsSchema.parse({ chunkSize: 0 })
).toThrow();
});
});
describe("streamingUnpackOptionsSchema", () => {
it("accepts the same shape as pack", () => {
const ac = new AbortController();
expect(
streamingUnpackOptionsSchema.parse({
signal: ac.signal,
chunkSize: 32,
})
).toEqual({ signal: ac.signal, chunkSize: 32 });
});
});
describe("readableStreamFromBytesOptionsSchema", () => {
it("accepts the documented fields", () => {
const onProgress = () => undefined;
const parsed = readableStreamFromBytesOptionsSchema.parse({
chunkSize: 256,
onProgress,
});
expect(parsed.chunkSize).toBe(256);
expect(parsed.onProgress).toBe(onProgress);
});
it("rejects a chunkProcessor (not part of the source-side API)", () => {
expect(() =>
readableStreamFromBytesOptionsSchema.parse({
chunkProcessor: () => new Uint8Array(),
})
).toThrow();
});
});
describe("writablePackOptionsSchema", () => {
it("accepts the documented fields", () => {
const chunkProcessor = (b: Uint8Array) => b;
const parsed = writablePackOptionsSchema.parse({ chunkProcessor });
expect(parsed.chunkProcessor).toBe(chunkProcessor);
});
it("rejects chunkSize (controlled upstream)", () => {
expect(() =>
writablePackOptionsSchema.parse({ chunkSize: 64 })
).toThrow();
});
});
describe("writableUnpackOptionsSchema", () => {
it("accepts the same shape as writable pack", () => {
expect(writableUnpackOptionsSchema.parse({})).toEqual({});
});
});

View File

@@ -0,0 +1,106 @@
import z from "zod";
import type {
ReadableStreamFromBytesOptions,
StreamingBitPackerOptions,
StreamingPackOptions,
StreamingUnpackOptions,
WritablePackOptions,
WritableUnpackOptions,
} from "./streaming-bit-packer.js";
const callable = <T>(message: string) =>
z.custom<T>((v): v is T => typeof v === "function", { message });
const clockSchema = z.custom<{ now(): number }>(
(v): v is { now(): number } => {
if (v === null || typeof v !== "object") return false;
return typeof (v as { now: unknown }).now === "function";
},
{ message: "clock must expose now()" }
);
const abortSignalSchema = z.custom<AbortSignal>(
(v): v is AbortSignal => v instanceof AbortSignal,
{ message: "signal must be an AbortSignal" }
);
/**
* Strict Zod schema for {@link StreamingBitPackerOptions}.
*
* Validates the constructor options bundle for `StreamingBitPacker`. Today
* the only field is `registerDefaults`, but using a strict object schema
* means future fields can be added without consumers' typos slipping
* through.
*/
export const streamingBitPackerOptionsSchema = z
.object({
registerDefaults: z.boolean().optional(),
})
.strict() as unknown as z.ZodType<StreamingBitPackerOptions>;
/**
* Strict Zod schema for {@link StreamingPackOptions}.
*
* Validates the per-call options for `packStream` / `packToStream`.
* `chunkProcessor` and `onProgress` are validated as functions via
* `z.custom` so identity is preserved through the pipeline. `clock` and
* `signal` are validated structurally so consumers can pass either a real
* `Clock` from `@suaveplan/types` or any compatible duck-type.
*/
export const streamingPackOptionsSchema = z
.object({
chunkSize: z.number().int().positive().optional(),
chunkProcessor: callable<
NonNullable<StreamingPackOptions["chunkProcessor"]>
>("chunkProcessor must be a function").optional(),
onProgress: callable<NonNullable<StreamingPackOptions["onProgress"]>>(
"onProgress must be a function"
).optional(),
clock: clockSchema.optional(),
signal: abortSignalSchema.optional(),
})
.strict() as unknown as z.ZodType<StreamingPackOptions>;
/** Strict Zod schema for {@link StreamingUnpackOptions} — same shape as pack. */
export const streamingUnpackOptionsSchema =
streamingPackOptionsSchema as unknown as z.ZodType<StreamingUnpackOptions>;
/**
* Strict Zod schema for {@link ReadableStreamFromBytesOptions}.
*
* Validates the options bundle for `readableStreamFromBytes`. Subset of
* `StreamingPackOptions` covering only the fields meaningful for a
* source-side ReadableStream.
*/
export const readableStreamFromBytesOptionsSchema = z
.object({
chunkSize: z.number().int().positive().optional(),
onProgress: callable<
NonNullable<ReadableStreamFromBytesOptions["onProgress"]>
>("onProgress must be a function").optional(),
clock: clockSchema.optional(),
})
.strict() as unknown as z.ZodType<ReadableStreamFromBytesOptions>;
/**
* Strict Zod schema for {@link WritablePackOptions}.
*
* Validates the options bundle for `writablePack` / `writableUnpack`. Same
* fields as `StreamingPackOptions` minus `chunkSize` and `signal` — the
* Writable side already controls chunk boundaries upstream.
*/
export const writablePackOptionsSchema = z
.object({
chunkProcessor: callable<
NonNullable<WritablePackOptions["chunkProcessor"]>
>("chunkProcessor must be a function").optional(),
onProgress: callable<NonNullable<WritablePackOptions["onProgress"]>>(
"onProgress must be a function"
).optional(),
clock: clockSchema.optional(),
})
.strict() as unknown as z.ZodType<WritablePackOptions>;
/** Strict Zod schema for {@link WritableUnpackOptions} — same shape as writable pack. */
export const writableUnpackOptionsSchema =
writablePackOptionsSchema as unknown as z.ZodType<WritableUnpackOptions>;

View File

@@ -0,0 +1,507 @@
import { BITPACKER_CODES, type BitpackerError } from "@suaveplan/bitpacker";
import { describe, expect, it } from "@suaveplan/testing/runner";
import type { Clock, TransformProgress } from "@suaveplan/types";
import { StreamingBitPacker } from "./streaming-bit-packer.js";
const SCHEMA = { x: 255, y: 255 } as const;
type Pair = { x: number; y: number };
function makeFakeClock(stepMs = 50): Clock {
let t = 0;
return () => {
t += stepMs;
return t;
};
}
describe("StreamingBitPacker — packStream / unpackStream", () => {
it("round-trips simple data", { timeout: 5000 }, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const original: Pair = { x: 42, y: 99 };
const bytes = await packer.packStream(original);
const decoded = await packer.unpackStream<Pair>(bytes);
expect(decoded).toEqual(original);
});
it("emits TransformProgress events for every chunk", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const progress: TransformProgress[] = [];
await packer.packStream({ x: 1, y: 2 } as Pair, {
chunkSize: 1,
onProgress: (p) => {
progress.push(p);
},
clock: makeFakeClock(),
});
expect(progress.length).toBeGreaterThanOrEqual(1);
// Last event drained the buffer.
const last = progress[progress.length - 1] as TransformProgress;
expect(last.itemsPending).toBe(0);
expect(last.operation).toBe("pack");
});
it("runs chunkProcessor on every chunk", { timeout: 5000 }, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const xor = (chunk: Uint8Array): Uint8Array => {
const out = new Uint8Array(chunk.length);
for (let i = 0; i < chunk.length; i++) {
out[i] = (chunk[i] as number) ^ 0xff;
}
return out;
};
const original: Pair = { x: 7, y: 8 };
const wire = await packer.packStream(original, {
chunkSize: 1,
chunkProcessor: xor,
});
// Reverse path with same processor (XOR is its own inverse).
const decoded = await packer.unpackStream<Pair>(wire, {
chunkSize: 1,
chunkProcessor: xor,
});
expect(decoded).toEqual(original);
});
it("packStream throws STREAM_ABORTED if signal is already aborted", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const ac = new AbortController();
ac.abort();
try {
await packer.packStream({ x: 1, y: 2 } as Pair, {
signal: ac.signal,
});
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_ABORTED
);
expect((e as BitpackerError).context?.operation).toBe("pack");
}
});
it("packStream throws STREAM_ABORTED if signal aborts mid-pipeline", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const ac = new AbortController();
// chunkProcessor aborts after first chunk.
const trip: typeof packer extends StreamingBitPacker<infer _>
? (chunk: Uint8Array) => Uint8Array
: never = (chunk) => {
ac.abort();
return chunk;
};
try {
await packer.packStream({ x: 1, y: 2 } as Pair, {
chunkSize: 1,
chunkProcessor: trip,
signal: ac.signal,
});
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_ABORTED
);
}
});
it("wraps chunkProcessor errors as STREAM_CHUNK_PROCESSOR_FAILED", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const angry = (): Uint8Array => {
throw new Error("processor blew up");
};
try {
await packer.packStream({ x: 1, y: 2 } as Pair, {
chunkProcessor: angry,
});
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_CHUNK_PROCESSOR_FAILED
);
}
});
});
describe("StreamingBitPacker — createReadableStream", () => {
it("emits all chunks and closes", { timeout: 5000 }, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const buf = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
const stream = packer.createReadableStream(buf, { chunkSize: 3 });
const received: number[] = [];
const reader = stream.getReader();
for (
let res = await reader.read();
!res.done;
res = await reader.read()
) {
for (const b of res.value) received.push(b);
}
expect(new Uint8Array(received)).toEqual(buf);
});
it("uses default chunkSize and defaultClock when options omitted", {
timeout: 5000,
}, async () => {
// No options at all → exercises every default-value branch of the
// parameter destructure (chunkSize, onProgress, clock).
const packer = new StreamingBitPacker(SCHEMA);
const buf = new Uint8Array(2048).fill(7);
const stream = packer.createReadableStream(buf);
const reader = stream.getReader();
let total = 0;
for (
let res = await reader.read();
!res.done;
res = await reader.read()
) {
total += res.value.length;
}
expect(total).toBe(2048);
});
it("emits progress per chunk through onProgress", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const buf = new Uint8Array([1, 2, 3, 4, 5, 6]);
const progress: TransformProgress[] = [];
const stream = packer.createReadableStream(buf, {
chunkSize: 2,
onProgress: (p) => {
progress.push(p);
},
clock: makeFakeClock(),
});
const reader = stream.getReader();
for (
let res = await reader.read();
!res.done;
res = await reader.read()
) {
// drain
}
expect(progress).toHaveLength(3);
expect(progress[2]?.itemsPending).toBe(0);
});
});
describe("StreamingBitPacker — createPackingStream", () => {
it("packs each written item and resolves on close", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const { stream, result } = packer.createPackingStream();
const writer = stream.getWriter();
await writer.write({ x: 1, y: 2 } as Pair);
await writer.write({ x: 3, y: 4 } as Pair);
await writer.close();
const out = await result;
expect(out).toBeInstanceOf(Uint8Array);
expect(out.length).toBeGreaterThan(0);
});
it("rejects result with STREAM_ABORTED on abort()", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const { stream, result } = packer.createPackingStream();
const writer = stream.getWriter();
await writer.abort("test-abort");
try {
await result;
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_ABORTED
);
expect((e as BitpackerError).context?.reason).toBe("test-abort");
}
});
it("rejects result if BitPacker.pack throws on a written item", {
timeout: 5000,
}, async () => {
// Build a packer with NO default strategies → every pack throws.
const packer = new StreamingBitPacker(SCHEMA, {
registerDefaults: false,
});
const { stream, result } = packer.createPackingStream();
const writer = stream.getWriter();
// The write will fail; capture & swallow the error from getWriter
// so we can assert through the result promise.
try {
await writer.write({ x: 1, y: 2 } as Pair);
} catch {
/* expected */
}
try {
await result;
} catch (e) {
expect(e).toBeDefined();
}
});
it("emits progress with itemsPending=0 on each write", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const progress: TransformProgress[] = [];
const { stream, result } = packer.createPackingStream({
onProgress: (p) => {
progress.push(p);
},
clock: makeFakeClock(),
});
const writer = stream.getWriter();
await writer.write({ x: 1, y: 2 } as Pair);
await writer.close();
await result;
expect(progress).toHaveLength(1);
expect(progress[0]?.itemsPending).toBe(0);
expect(progress[0]?.operation).toBe("pack");
});
it("runs chunkProcessor on each packed chunk", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const observed: Uint8Array[] = [];
const { stream, result } = packer.createPackingStream({
chunkProcessor: (c) => {
observed.push(c);
return c;
},
});
const writer = stream.getWriter();
await writer.write({ x: 5, y: 6 } as Pair);
await writer.close();
await result;
expect(observed.length).toBe(1);
});
});
describe("StreamingBitPacker — createUnpackingStream", () => {
it("buffers writes and unpacks on close", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const original: Pair = { x: 200, y: 100 };
const packed = await packer.packStream(original);
const { stream, result } = packer.createUnpackingStream<Pair>();
const writer = stream.getWriter();
// Split the packed bytes across two writes.
const half = Math.floor(packed.length / 2);
await writer.write(packed.slice(0, half));
await writer.write(packed.slice(half));
await writer.close();
const out = await result;
expect(out).toEqual(original);
});
it("rejects with STREAM_INCOMPLETE_DATA when closed empty", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const { stream, result } = packer.createUnpackingStream();
const writer = stream.getWriter();
await writer.close();
try {
await result;
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_INCOMPLETE_DATA
);
}
});
it("rejects with STREAM_ABORTED on abort()", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const { stream, result } = packer.createUnpackingStream();
const writer = stream.getWriter();
await writer.abort("user-cancel");
try {
await result;
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_ABORTED
);
expect((e as BitpackerError).context?.reason).toBe("user-cancel");
}
});
it("rejects if BitPacker.unpack throws on close", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA, {
registerDefaults: false,
});
const { stream, result } = packer.createUnpackingStream();
const writer = stream.getWriter();
// Push some bytes so close attempts an unpack — and unpack fails
// because we skipped strategy registration.
await writer.write(new Uint8Array([1, 2, 3]));
await writer.close();
try {
await result;
} catch (e) {
expect(e).toBeDefined();
}
});
it("runs chunkProcessor on each incoming chunk", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const original: Pair = { x: 1, y: 2 };
const packed = await packer.packStream(original);
const observed: number[] = [];
const { stream, result } = packer.createUnpackingStream<Pair>({
chunkProcessor: (c) => {
observed.push(c.length);
return c;
},
});
const writer = stream.getWriter();
await writer.write(packed);
await writer.close();
const out = await result;
expect(out).toEqual(original);
expect(observed.length).toBe(1);
});
it("emits progress with operation='unpack' on each write", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const original: Pair = { x: 9, y: 10 };
const packed = await packer.packStream(original);
const progress: TransformProgress[] = [];
const { stream, result } = packer.createUnpackingStream<Pair>({
onProgress: (p) => {
progress.push(p);
},
clock: makeFakeClock(),
});
const writer = stream.getWriter();
await writer.write(packed);
await writer.close();
await result;
expect(progress.length).toBe(1);
expect(progress[0]?.operation).toBe("unpack");
});
it("wraps chunkProcessor errors as STREAM_CHUNK_PROCESSOR_FAILED on the writable side", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
const original: Pair = { x: 3, y: 4 };
const packed = await packer.packStream(original);
const { stream, result } = packer.createUnpackingStream({
chunkProcessor: () => {
throw new Error("processor blew up");
},
});
const writer = stream.getWriter();
try {
await writer.write(packed);
} catch {
/* expected */
}
try {
await result;
} catch (e) {
expect((e as BitpackerError).code).toBe(
BITPACKER_CODES.STREAM_CHUNK_PROCESSOR_FAILED
);
}
});
});
describe("StreamingBitPacker — throughput zero-elapsed branch", () => {
const frozen: Clock = () => 0;
it("packStream emits throughput=0 when clock is frozen", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
let captured: TransformProgress | undefined;
await packer.packStream({ x: 1, y: 2 } as Pair, {
chunkSize: 1,
clock: frozen,
onProgress: (p) => {
captured = p;
},
});
expect(captured?.throughput).toBe(0);
});
it("createReadableStream emits throughput=0 when clock is frozen", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
let captured: TransformProgress | undefined;
const stream = packer.createReadableStream(
new Uint8Array([1, 2, 3, 4]),
{
chunkSize: 2,
clock: frozen,
onProgress: (p) => {
captured = p;
},
}
);
const reader = stream.getReader();
for (
let res = await reader.read();
!res.done;
res = await reader.read()
) {
// drain
}
expect(captured?.throughput).toBe(0);
});
it("createPackingStream emits throughput=0 when clock is frozen", {
timeout: 5000,
}, async () => {
const packer = new StreamingBitPacker(SCHEMA);
let captured: TransformProgress | undefined;
const { stream, result } = packer.createPackingStream({
clock: frozen,
onProgress: (p) => {
captured = p;
},
});
const writer = stream.getWriter();
await writer.write({ x: 1, y: 2 } as Pair);
await writer.close();
await result;
expect(captured?.throughput).toBe(0);
});
});
describe("StreamingBitPacker — constructor and exposed BitPacker", () => {
it("exposes the underlying BitPacker via .bitPacker", {
timeout: 5000,
}, () => {
const packer = new StreamingBitPacker(SCHEMA);
expect(packer.bitPacker).toBeDefined();
// Strategies registered by default.
const bytes = packer.bitPacker.pack({ x: 7, y: 8 } as Pair);
expect(bytes).toBeInstanceOf(Uint8Array);
});
it("opt-out via registerDefaults: false leaves no strategies", {
timeout: 5000,
}, () => {
const packer = new StreamingBitPacker(SCHEMA, {
registerDefaults: false,
});
expect(() => packer.bitPacker.pack({ x: 7, y: 8 } as Pair)).toThrow();
});
});

View File

@@ -0,0 +1,413 @@
/**
* `StreamingBitPacker<T>` — class-shaped façade over a chunked
* pack/unpack pipeline.
*
* Wraps an underlying `BitPacker<T>` and exposes the five upstream
* streaming entry points:
*
* 1. `packStream(data, options)` — bit-pack, walk the bytes in chunks,
* optionally run each through `chunkProcessor`, concatenate, return.
* 2. `unpackStream(data, options)` — chunk-walk the bytes, run them
* through `chunkProcessor`, concatenate, bit-unpack, return.
* 3. `createReadableStream(data, options)` — return a WHATWG
* `ReadableStream<Uint8Array>` that emits the buffer in chunks.
* 4. `createPackingStream(options)` — return `{ stream, result }` where
* `stream` is a `WritableStream<unknown>` accepting one item per
* write and `result` resolves to the concatenated packed bytes on
* close.
* 5. `createUnpackingStream(options)` — return `{ stream, result }` where
* `stream` is a `WritableStream<Uint8Array>` accepting raw bytes and
* `result` resolves to the unpacked value on close.
*
* Genesis-side adjustments versus upstream
* `bitpacker/src/streaming/streaming.ts`:
*
* - No `EventEmitter`. Genesis's `BitPacker` does not extend an emitter,
* so we surface progress via the `onProgress` callback shape used
* throughout `bitpacker-streaming`.
* - `BitpackerError` with stable codes (`STREAM_ABORTED`,
* `STREAM_CHUNK_PROCESSOR_FAILED`, `STREAM_INCOMPLETE_DATA`) replaces
* plain `Error` throws.
* - Injectable `Clock` for throughput timing — tests get determinism
* without spinning up real timers.
* - The upstream `validateChunks` option is dropped: genesis `BitPacker`
* does not expose `validate()`/`getValidationErrors()`. Callers wanting
* pre-pack validation run their own validator before calling these
* methods (Zod, etc.).
* - Default strategy registration is opt-out via `registerDefaults: false`
* — by default the new `BitPacker` has the full strategy set, matching
* upstream's implicit behaviour.
* - `TransformProgress` from `@suaveplan/types` replaces the upstream
* ad-hoc progress payload, so a single observability consumer can
* speak the same vocabulary across all `bitpacker-streaming` entry
* points.
*/
import {
BITPACKER_CODES,
BitPacker,
type BitPackerSchema,
BitpackerError,
} from "@suaveplan/bitpacker";
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
import type { Clock, TransformProgress } from "@suaveplan/types";
import { defaultClock } from "../defaults/default-clock.js";
export type ChunkProcessor = (
chunk: Uint8Array
) => Uint8Array | Promise<Uint8Array>;
export interface StreamingPackOptions {
/** Bytes per emitted chunk. Default `1024`. */
chunkSize?: number;
/** Optional per-chunk processor (e.g. compress, encrypt). */
chunkProcessor?: ChunkProcessor;
/** Optional progress callback. */
onProgress?: (progress: TransformProgress) => void;
/** Optional injectable wall clock. Default `defaultClock`. */
clock?: Clock;
/** Optional abort signal. */
signal?: AbortSignal;
}
export interface StreamingUnpackOptions extends StreamingPackOptions {}
export interface ReadableStreamFromBytesOptions {
/** Bytes per emitted chunk. Default `1024`. */
chunkSize?: number;
/** Optional progress callback. */
onProgress?: (progress: TransformProgress) => void;
/** Optional injectable wall clock. Default `defaultClock`. */
clock?: Clock;
}
export interface WritablePackOptions {
/** Optional per-chunk processor (e.g. compress, encrypt). */
chunkProcessor?: ChunkProcessor;
/** Optional progress callback. */
onProgress?: (progress: TransformProgress) => void;
/** Optional injectable wall clock. Default `defaultClock`. */
clock?: Clock;
}
export interface WritableUnpackOptions extends WritablePackOptions {}
export interface StreamingBitPackerOptions {
/**
* Whether to register the default packing strategies on the new
* underlying `BitPacker`. Default `true`. Set `false` if the caller
* is registering a custom strategy set themselves.
*/
readonly registerDefaults?: boolean;
}
export class StreamingBitPacker<T extends BitPackerSchema = BitPackerSchema> {
public readonly bitPacker: BitPacker<T>;
public constructor(schema: T, options: StreamingBitPackerOptions = {}) {
this.bitPacker = new BitPacker<T>(schema);
if (options.registerDefaults !== false) {
registerDefaultStrategies(this.bitPacker.strategyManager);
}
}
/**
* Pack `data` to bytes, walk the bytes in `chunkSize`-byte chunks,
* optionally run each chunk through `chunkProcessor`, and return the
* concatenated result.
*/
public async packStream(
data: unknown,
options: StreamingPackOptions = {}
): Promise<Uint8Array> {
const packed = this.bitPacker.pack(data);
return runChunkPipeline(packed, "pack", options);
}
/**
* Reverse of `packStream`. Walks `data` in chunks (running the
* processor in reverse if supplied), concatenates, and bit-unpacks
* the result.
*/
public async unpackStream<R = unknown>(
data: Uint8Array,
options: StreamingUnpackOptions = {}
): Promise<R> {
const combined = await runChunkPipeline(data, "unpack", options);
return this.bitPacker.unpack<R>(combined);
}
/**
* Return a WHATWG `ReadableStream<Uint8Array>` that emits `data` in
* `chunkSize`-byte chunks. Useful for piping packed bytes into
* another transform (e.g. `pipeThrough(compressionStream)`).
*/
public createReadableStream(
data: Uint8Array,
options: ReadableStreamFromBytesOptions = {}
): ReadableStream<Uint8Array> {
const { chunkSize = 1024, onProgress, clock = defaultClock } = options;
const startTime = clock();
const chunksTotal = Math.ceil(data.length / chunkSize);
return new ReadableStream<Uint8Array>({
start(controller) {
let bytesProcessed = 0;
let itemsProcessed = 0;
for (let i = 0; i < chunksTotal; i++) {
const start = i * chunkSize;
const end = Math.min(start + chunkSize, data.length);
const chunk = data.slice(start, end);
controller.enqueue(chunk);
bytesProcessed += chunk.length;
itemsProcessed++;
if (onProgress) {
const elapsed = (clock() - startTime) / 1000;
onProgress({
itemsProcessed,
bytesProcessed,
itemsPending: chunksTotal - itemsProcessed,
throughput:
elapsed > 0 ? itemsProcessed / elapsed : 0,
operation: "pack",
});
}
}
controller.close();
},
});
}
/**
* Return a `{ stream, result }` pair where each `write(item)` packs
* the item and pushes the bytes into a buffer. On `close()`, `result`
* resolves to the concatenated packed bytes.
*/
public createPackingStream(options: WritablePackOptions = {}): {
stream: WritableStream<unknown>;
result: Promise<Uint8Array>;
} {
const { chunkProcessor, onProgress, clock = defaultClock } = options;
const startTime = clock();
const chunks: Uint8Array[] = [];
let resolveResult!: (value: Uint8Array) => void;
let rejectResult!: (error: Error) => void;
const result = new Promise<Uint8Array>((resolve, reject) => {
resolveResult = resolve;
rejectResult = reject;
});
const packer = this.bitPacker;
const stream = new WritableStream<unknown>({
async write(chunk) {
try {
const packed = packer.pack(chunk);
const processed = chunkProcessor
? await runProcessor(chunkProcessor, packed)
: packed;
chunks.push(processed);
if (onProgress) {
emitWriteProgress(
chunks,
"pack",
startTime,
clock,
onProgress
);
}
} catch (cause) {
rejectResult(cause as Error);
throw cause;
}
},
close() {
resolveResult(concatChunks(chunks));
},
abort(reason) {
rejectResult(
new BitpackerError(
BITPACKER_CODES.STREAM_ABORTED,
"Packing stream aborted",
{ context: { reason: String(reason) } }
)
);
},
});
return { stream, result };
}
/**
* Return a `{ stream, result }` pair where each `write(bytes)` pushes
* the bytes (optionally through `chunkProcessor`) into a buffer. On
* `close()`, the buffer is concatenated and bit-unpacked.
*/
public createUnpackingStream<R = unknown>(
options: WritableUnpackOptions = {}
): { stream: WritableStream<Uint8Array>; result: Promise<R> } {
const { chunkProcessor, onProgress, clock = defaultClock } = options;
const startTime = clock();
const chunks: Uint8Array[] = [];
let resolveResult!: (value: R) => void;
let rejectResult!: (error: Error) => void;
const result = new Promise<R>((resolve, reject) => {
resolveResult = resolve;
rejectResult = reject;
});
const packer = this.bitPacker;
const stream = new WritableStream<Uint8Array>({
async write(chunk) {
try {
const processed = chunkProcessor
? await runProcessor(chunkProcessor, chunk)
: chunk;
chunks.push(processed);
if (onProgress) {
emitWriteProgress(
chunks,
"unpack",
startTime,
clock,
onProgress
);
}
} catch (cause) {
rejectResult(cause as Error);
throw cause;
}
},
close() {
try {
const combined = concatChunks(chunks);
if (combined.length === 0) {
rejectResult(
new BitpackerError(
BITPACKER_CODES.STREAM_INCOMPLETE_DATA,
"Unpacking stream closed with no bytes",
{ context: {} }
)
);
return;
}
resolveResult(packer.unpack<R>(combined));
} catch (cause) {
rejectResult(cause as Error);
}
},
abort(reason) {
rejectResult(
new BitpackerError(
BITPACKER_CODES.STREAM_ABORTED,
"Unpacking stream aborted",
{ context: { reason: String(reason) } }
)
);
},
});
return { stream, result };
}
}
async function runChunkPipeline(
data: Uint8Array,
operation: "pack" | "unpack",
options: StreamingPackOptions
): Promise<Uint8Array> {
const {
chunkSize = 1024,
chunkProcessor,
onProgress,
clock = defaultClock,
signal,
} = options;
if (signal?.aborted) {
throw new BitpackerError(
BITPACKER_CODES.STREAM_ABORTED,
"Stream aborted before processing",
{ context: { operation } }
);
}
const chunksTotal = Math.max(1, Math.ceil(data.length / chunkSize));
const startTime = clock();
const chunks: Uint8Array[] = [];
for (let i = 0; i < chunksTotal; i++) {
if (signal?.aborted) {
throw new BitpackerError(
BITPACKER_CODES.STREAM_ABORTED,
"Stream aborted mid-processing",
{ context: { operation, chunkIndex: i } }
);
}
const start = i * chunkSize;
const end = Math.min(start + chunkSize, data.length);
const chunk = data.slice(start, end);
const processed = chunkProcessor
? await runProcessor(chunkProcessor, chunk)
: chunk;
chunks.push(processed);
if (onProgress) {
const elapsed = (clock() - startTime) / 1000;
onProgress({
itemsProcessed: i + 1,
bytesProcessed: end,
itemsPending: chunksTotal - (i + 1),
throughput: elapsed > 0 ? (i + 1) / elapsed : 0,
operation,
});
}
}
return concatChunks(chunks);
}
async function runProcessor(
processor: ChunkProcessor,
chunk: Uint8Array
): Promise<Uint8Array> {
try {
return await processor(chunk);
} catch (cause) {
throw new BitpackerError(
BITPACKER_CODES.STREAM_CHUNK_PROCESSOR_FAILED,
"chunkProcessor threw",
{
cause: cause as Error,
context: { byteLength: chunk.length },
}
);
}
}
function emitWriteProgress(
chunks: readonly Uint8Array[],
operation: "pack" | "unpack",
startTime: number,
clock: Clock,
onProgress: (progress: TransformProgress) => void
): void {
const bytesProcessed = chunks.reduce((acc, c) => acc + c.length, 0);
const elapsed = (clock() - startTime) / 1000;
onProgress({
itemsProcessed: chunks.length,
bytesProcessed,
itemsPending: 0,
throughput: elapsed > 0 ? chunks.length / elapsed : 0,
operation,
});
}
function concatChunks(chunks: readonly Uint8Array[]): Uint8Array {
const totalLength = chunks.reduce((acc, c) => acc + c.length, 0);
const out = new Uint8Array(totalLength);
let offset = 0;
for (const c of chunks) {
out.set(c, offset);
offset += c.length;
}
return out;
}

View File

@@ -0,0 +1,673 @@
import { BitPacker } from "@suaveplan/bitpacker";
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
import { describe, expect, test } from "@suaveplan/testing/runner";
import {
addFrame,
createFramedPackStream,
createFramedUnpackStream,
createPackTransformStream,
createUnpackTransformStream,
packToChunks,
parseFrames,
unpackFromChunks,
unpackFromChunksAsync,
} from "./index.js";
describe("packToChunks / unpackFromChunks", () => {
test("roundtrips simple data", () => {
const schema = 255;
const data = 42;
const chunks = packToChunks(data, schema, { chunkSize: 2 });
expect(chunks.length).toBeGreaterThanOrEqual(1);
const result = unpackFromChunks<number>(chunks, schema);
expect(result).toBe(42);
});
test("skips empty chunks", () => {
const schema = 255;
const data = 42;
const chunks = [new Uint8Array(0), ...packToChunks(data, schema)];
const result = unpackFromChunks<number>(chunks, schema);
expect(result).toBe(42);
});
test("async roundtrip", async () => {
const schema = "-64";
const data = "hello world";
const chunks = packToChunks(data, schema);
async function* gen() {
for (const c of chunks) yield c;
}
const result = await unpackFromChunksAsync<string>(gen(), schema);
expect(result).toBe("hello world");
});
test("async aborts on signal", async () => {
const schema = 255;
const data = 42;
const chunks = packToChunks(data, schema);
async function* gen() {
for (const c of chunks) yield c;
}
const controller = new AbortController();
controller.abort();
await expect(
unpackFromChunksAsync<number>(gen(), schema, {
signal: controller.signal,
})
).rejects.toThrow("aborted");
});
test("async skips empty chunks", async () => {
const schema = 255;
const data = 42;
const chunks = packToChunks(data, schema);
async function* gen() {
yield new Uint8Array(0);
for (const c of chunks) yield c;
}
const result = await unpackFromChunksAsync<number>(gen(), schema);
expect(result).toBe(42);
});
});
describe("framing", () => {
test("length-prefix framing roundtrips", () => {
const data = new Uint8Array([1, 2, 3, 4, 5]);
const framed = addFrame(data, {
type: "length-prefix",
lengthPrefixSize: 2,
});
expect(framed.length).toBe(7);
const { frames, remainder } = parseFrames(framed, {
type: "length-prefix",
lengthPrefixSize: 2,
});
expect(remainder.length).toBe(0);
expect(frames.length).toBe(1);
expect(frames[0]).toEqual(data);
});
test("delimiter framing roundtrips", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, {
type: "delimiter",
delimiter: new Uint8Array([0xff]),
});
expect(framed[framed.length - 1]).toBe(0xff);
const { frames, remainder } = parseFrames(framed, {
type: "delimiter",
delimiter: new Uint8Array([0xff]),
});
expect(frames.length).toBe(1);
expect(frames[0]).toEqual(data);
expect(remainder.length).toBe(0);
});
test("fixed-size framing", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, { type: "fixed-size", fixedSize: 8 });
expect(framed.length).toBe(8);
});
test("none framing passes through", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, { type: "none" });
expect(framed).toEqual(data);
});
test("length-prefix 1-byte roundtrips", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, {
type: "length-prefix",
lengthPrefixSize: 1,
});
const { frames, remainder } = parseFrames(framed, {
type: "length-prefix",
lengthPrefixSize: 1,
});
expect(remainder.length).toBe(0);
expect(frames[0]).toEqual(data);
});
test("length-prefix 4-byte roundtrips", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, {
type: "length-prefix",
lengthPrefixSize: 4,
});
const { frames, remainder } = parseFrames(framed, {
type: "length-prefix",
lengthPrefixSize: 4,
});
expect(remainder.length).toBe(0);
expect(frames[0]).toEqual(data);
});
test("fixed-size framing roundtrips", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, { type: "fixed-size", fixedSize: 8 });
const { frames, remainder } = parseFrames(framed, {
type: "fixed-size",
fixedSize: 8,
});
expect(remainder.length).toBe(0);
expect(frames[0]).toEqual(framed);
});
test("parseFrames handles partial data", () => {
const buffer = new Uint8Array([0, 5]);
const { frames, remainder } = parseFrames(buffer, {
type: "length-prefix",
lengthPrefixSize: 2,
});
expect(frames.length).toBe(0);
expect(remainder.length).toBe(2);
});
test("parseFrames none type", () => {
const data = new Uint8Array([1, 2, 3]);
const { frames, remainder } = parseFrames(data, { type: "none" });
expect(frames[0]).toEqual(data);
expect(remainder.length).toBe(0);
});
test("parseFrames uses defaults for length-prefix", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, { type: "length-prefix" });
expect(framed.length).toBe(7);
const { frames, remainder } = parseFrames(framed, {
type: "length-prefix",
});
expect(frames[0]).toEqual(data);
expect(remainder.length).toBe(0);
});
test("parseFrames uses defaults for delimiter", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, { type: "delimiter" });
expect(framed[framed.length - 1]).toBe(0);
const { frames, remainder } = parseFrames(framed, {
type: "delimiter",
});
expect(frames[0]).toEqual(data);
expect(remainder.length).toBe(0);
});
test("parseFrames uses defaults for fixed-size", () => {
const data = new Uint8Array([1, 2, 3]);
const framed = addFrame(data, { type: "fixed-size" });
expect(framed.length).toBe(256);
const { frames, remainder } = parseFrames(framed, {
type: "fixed-size",
});
expect(frames.length).toBe(1);
expect(remainder.length).toBe(0);
});
test("parseFrames delimiter at start of buffer", () => {
const delimiter = new Uint8Array([0xff]);
const buffer = new Uint8Array([0xff]);
const { frames, remainder } = parseFrames(buffer, {
type: "delimiter",
delimiter,
});
expect(frames.length).toBe(0);
expect(remainder.length).toBe(0);
});
test("parseFrames empty buffer with none", () => {
const { frames, remainder } = parseFrames(new Uint8Array(0), {
type: "none",
});
expect(frames.length).toBe(0);
expect(remainder.length).toBe(0);
});
test("parseFrames empty buffer with delimiter", () => {
const { frames, remainder } = parseFrames(new Uint8Array(0), {
type: "delimiter",
delimiter: new Uint8Array([0xff]),
});
expect(frames.length).toBe(0);
expect(remainder.length).toBe(0);
});
test("parseFrames empty buffer with fixed-size", () => {
const { frames, remainder } = parseFrames(new Uint8Array(0), {
type: "fixed-size",
fixedSize: 8,
});
expect(frames.length).toBe(0);
expect(remainder.length).toBe(0);
});
test("parseFrames empty buffer with length-prefix", () => {
const { frames, remainder } = parseFrames(new Uint8Array(0), {
type: "length-prefix",
lengthPrefixSize: 4,
});
expect(frames.length).toBe(0);
expect(remainder.length).toBe(0);
});
});
describe("TransformStreams", () => {
test("createPackTransformStream with batchSize > 1", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const stream = createPackTransformStream(bp, schema, { batchSize: 2 });
const writer = stream.writable.getWriter();
await writer.write(1);
await writer.write(2);
await writer.close();
const reader = stream.readable.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
expect(chunks.length).toBe(2);
});
test("createPackTransformStream flushes remaining batch", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const stream = createPackTransformStream(bp, schema, { batchSize: 2 });
const writer = stream.writable.getWriter();
await writer.write(1);
await writer.close();
const reader = stream.readable.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
expect(chunks.length).toBe(1);
});
test("createPackTransformStream calls onProgress", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
let time = 1000;
Date.now = () => time++;
try {
const stream = createPackTransformStream(bp, schema, {
batchSize: 1,
onProgress: (p) => calls.push(p),
});
const writer = stream.writable.getWriter();
await writer.write(1);
await writer.close();
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[0]).toMatchObject({
itemsProcessed: expect.any(Number),
bytesProcessed: expect.any(Number),
operation: "pack",
});
expect(calls[calls.length - 1]).toMatchObject({
throughput: expect.any(Number),
});
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
});
test("createPackTransformStream handles zero duration", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
Date.now = () => 1000;
try {
const stream = createPackTransformStream(bp, schema, {
batchSize: 1,
onProgress: (p) => calls.push(p),
});
const writer = stream.writable.getWriter();
await writer.write(1);
await writer.close();
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[calls.length - 1]).toMatchObject({
throughput: 0,
operation: "pack",
});
});
test("createUnpackTransformStream decodes data", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const packStream = createPackTransformStream(bp, schema);
const unpackStream = createUnpackTransformStream(bp, schema);
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(42);
});
test("createUnpackTransformStream decodes framed data in transform", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const packStream = createFramedPackStream(bp, schema, {
type: "length-prefix",
lengthPrefixSize: 4,
});
const unpackStream = createUnpackTransformStream(bp, schema, {
headerSize: 4,
});
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(42);
});
test("createUnpackTransformStream calls onProgress", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
let time = 1000;
Date.now = () => time++;
try {
const packStream = createPackTransformStream(bp, schema);
const unpackStream = createUnpackTransformStream(bp, schema, {
onProgress: (p) => calls.push(p),
});
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(42);
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[0]).toMatchObject({ operation: "unpack" });
expect(calls[calls.length - 1]).toMatchObject({
throughput: expect.any(Number),
});
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
});
test("createUnpackTransformStream handles zero duration", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
Date.now = () => 1000;
try {
const packStream = createPackTransformStream(bp, schema);
const unpackStream = createUnpackTransformStream(bp, schema, {
onProgress: (p) => calls.push(p),
});
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
await reader.read();
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[calls.length - 1]).toMatchObject({
throughput: 0,
operation: "unpack",
});
});
test("createUnpackTransformStream flushes incomplete valid frame", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const unpackStream = createUnpackTransformStream(bp, schema, {
headerSize: 4,
});
const writer = unpackStream.writable.getWriter();
// 4-byte header indicating length=1, but no payload
await writer.write(new Uint8Array([0, 0, 0, 1]));
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(0);
});
test("createUnpackTransformStream flushes incomplete invalid frame", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = "f32";
const unpackStream = createUnpackTransformStream(bp, schema, {
headerSize: 4,
});
const writer = unpackStream.writable.getWriter();
// 3 bytes is less than header size (4), so parseFrames leaves it in buffer
// f32 needs 4 bytes, so unpacking 3 bytes should fail
await writer.write(new Uint8Array([0, 0, 0]));
await writer.close();
const reader = unpackStream.readable.getReader();
const { done } = await reader.read();
expect(done).toBe(true);
});
test("createFramedPackStream emits framed data", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const stream = createFramedPackStream(bp, schema, {
type: "length-prefix",
lengthPrefixSize: 2,
});
const writer = stream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = stream.readable.getReader();
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
}
expect(chunks.length).toBeGreaterThan(0);
expect(chunks[0]!.length).toBeGreaterThan(2);
});
test("createFramedPackStream uses default framing", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const stream = createFramedPackStream(bp, schema);
const writer = stream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = stream.readable.getReader();
const { value } = await reader.read();
expect(value!.length).toBeGreaterThan(1);
});
test("createFramedPackStream calls onProgress", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
let time = 1000;
Date.now = () => time++;
try {
const stream = createFramedPackStream(
bp,
schema,
{ type: "length-prefix", lengthPrefixSize: 2 },
{ onProgress: (p) => calls.push(p) }
);
const writer = stream.writable.getWriter();
await writer.write(42);
await writer.close();
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[0]).toMatchObject({ operation: "pack" });
expect(calls[calls.length - 1]).toMatchObject({
throughput: expect.any(Number),
});
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
});
test("createFramedPackStream handles zero duration", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
Date.now = () => 1000;
try {
const stream = createFramedPackStream(
bp,
schema,
{ type: "length-prefix", lengthPrefixSize: 2 },
{ onProgress: (p) => calls.push(p) }
);
const writer = stream.writable.getWriter();
await writer.write(42);
await writer.close();
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[calls.length - 1]).toMatchObject({
throughput: 0,
operation: "pack",
});
});
test("createFramedUnpackStream decodes framed data", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const packStream = createFramedPackStream(bp, schema, {
type: "length-prefix",
lengthPrefixSize: 2,
});
const unpackStream = createFramedUnpackStream(bp, schema, {
type: "length-prefix",
lengthPrefixSize: 2,
});
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(42);
});
test("createFramedUnpackStream uses default framing", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const packStream = createFramedPackStream(bp, schema);
const unpackStream = createFramedUnpackStream(bp, schema);
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(42);
});
test("createFramedUnpackStream calls onProgress", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
let time = 1000;
Date.now = () => time++;
try {
const packStream = createFramedPackStream(bp, schema, {
type: "length-prefix",
lengthPrefixSize: 2,
});
const unpackStream = createFramedUnpackStream(
bp,
schema,
{ type: "length-prefix", lengthPrefixSize: 2 },
{ onProgress: (p) => calls.push(p) }
);
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
const { value } = await reader.read();
expect(value).toBe(42);
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[0]).toMatchObject({ operation: "unpack" });
expect(calls[calls.length - 1]).toMatchObject({
throughput: expect.any(Number),
});
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
});
test("createFramedUnpackStream handles zero duration", async () => {
const bp = new BitPacker();
registerDefaultStrategies(bp.strategyManager);
const schema = 255;
const calls: unknown[] = [];
const originalNow = Date.now;
Date.now = () => 1000;
try {
const packStream = createFramedPackStream(bp, schema, {
type: "length-prefix",
lengthPrefixSize: 2,
});
const unpackStream = createFramedUnpackStream(
bp,
schema,
{ type: "length-prefix", lengthPrefixSize: 2 },
{ onProgress: (p) => calls.push(p) }
);
packStream.readable.pipeTo(unpackStream.writable);
const writer = packStream.writable.getWriter();
await writer.write(42);
await writer.close();
const reader = unpackStream.readable.getReader();
await reader.read();
} finally {
Date.now = originalNow;
}
expect(calls.length).toBeGreaterThan(0);
expect(calls[calls.length - 1]).toMatchObject({
throughput: 0,
operation: "unpack",
});
});
});

View File

@@ -0,0 +1,53 @@
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs --> - transform.schema — Zod schemas for the option bundl…
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- transform.schema — Zod schemas for the option bundl…
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- transform.schema — Zod schemas for the option bundl…
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- transform.schema — Zod schemas for the option bundl…
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
## Transform Streams
**Path:** `src/transform/transform.ts`
**Export:** `createPackTransformStream`, `createUnpackTransformStream`, `createFramedPackStream`, `createFramedUnpackStream`, `parseFrames`, `addFrame`
### What it does
True `TransformStream` support for BitPacker with backpressure, batching, progress callbacks, and multiple framing strategies.
### Key API
| Function | Description |
|----------|-------------|
| `createPackTransformStream(bp, schema, options?)` | Transform objects → `Uint8Array` chunks |
| `createUnpackTransformStream(bp, schema, options?)` | Transform `Uint8Array` → objects (length-prefix framing) |
| `createFramedPackStream(bp, schema, framing?, options?)` | Pack with explicit framing config |
| `createFramedUnpackStream(bp, schema, framing?, options?)` | Unpack with explicit framing config |
| `parseFrames(buffer, framing)` | Extract complete frames from a buffer |
| `addFrame(data, framing)` | Add framing to a message |
### Framing Types
- `length-prefix` — prefix with 1/2/4 byte length (default)
- `delimiter` — suffix with delimiter bytes
- `fixed-size` — pad/truncate to fixed size
- `none` — no framing (single message per chunk)
### Depends on
- `BitPacker` from `@suaveplan/bitpacker`
### Used by
- Network streaming pipelines
- File serialization with chunked I/O

View File

@@ -0,0 +1,79 @@
# Transform Stream Schemas
## Purpose
Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the framing helpers (`addFrame`, `parseFrames`, `createFramedPackStream`, `createFramedUnpackStream`). Each schema is the runtime gatekeeper at the corresponding boundary — `framingConfigSchema` validates the frame-strategy descriptor, and `packTransformOptionsSchema` / `unpackTransformOptionsSchema` validate the per-call tuning bundles.
The framing schema deliberately does NOT enforce cross-field requirements (e.g. "if `type === 'length-prefix'` then `lengthPrefixSize` is required"). The runtime parser tolerates the extra fields, and forcing the cross-field check at parse time would force callers to always specify `lengthPrefixSize: 4` even when defaulting it. Strict-mode rejects unknown keys; the runtime parser is the authority on which combinations make sense.
## Features
- `framingConfigSchema` — discriminated by `type` (`length-prefix` / `delimiter` / `fixed-size` / `none`)
- `packTransformOptionsSchema` — `batchSize`, `highWaterMark`, `onProgress`
- `unpackTransformOptionsSchema` — `highWaterMark`, `onProgress`, `headerSize`
- Identity-preserving `z.custom` for `onProgress`
- `Uint8Array` instance check for `delimiter` (rejects array-of-numbers literals)
- `as unknown as z.ZodType<T>` casts for EOPT mismatches
## Usage: Basic Example
```ts
import {
framingConfigSchema,
packTransformOptionsSchema,
} from "@suaveplan/bitpacker-streaming";
const framing = framingConfigSchema.parse({
type: "length-prefix",
lengthPrefixSize: 4,
});
const packOpts = packTransformOptionsSchema.parse({
batchSize: 4,
highWaterMark: 16,
onProgress: (p) => console.log(p),
});
```
## Usage: Advanced Example
```ts
import {
addFrame,
framingConfigSchema,
parseFrames,
unpackTransformOptionsSchema,
} from "@suaveplan/bitpacker-streaming";
// Validate framing at the boundary, then trust the parsed shape downstream:
function withFraming(rawFraming: unknown, payload: Uint8Array) {
const cfg = framingConfigSchema.parse(rawFraming);
return addFrame(payload, cfg);
}
// Per-call unpack options, including a length-prefix header width:
const unpackOpts = unpackTransformOptionsSchema.parse({
highWaterMark: 32,
headerSize: 4,
});
```
## API Reference
### `framingConfigSchema: z.ZodType<FramingConfig>`
Strict schema for the `FramingConfig` discriminator. `type` is one of four literal tokens. `lengthPrefixSize` is restricted to `1 | 2 | 4`. `delimiter` must be a `Uint8Array` (not a `number[]` literal). `fixedSize` is a positive integer.
### `packTransformOptionsSchema: z.ZodType<PackTransformOptions>`
Strict schema. `batchSize` and `highWaterMark` are positive integers. `onProgress` is a function preserved by identity.
### `unpackTransformOptionsSchema: z.ZodType<UnpackTransformOptions>`
Strict schema. `highWaterMark` and `headerSize` are positive integers. `onProgress` is a function preserved by identity.
## Implementation Notes
- **No cross-field validation.** A `length-prefix` framing without `lengthPrefixSize` is technically incomplete, but the runtime parser defaults it to `4` and tolerates the omission. Forcing a cross-field rule at parse time would over-constrain callers.
- **`Uint8Array` instance check on `delimiter`.** A `[0x0a]` literal looks like a delimiter but does not implement the typed-array surface the parser uses (`.subarray`, `.byteLength`). Refusing it at parse time prevents a silent type coercion that would surface as an obscure runtime error.
- **`as unknown as` casts.** Same EOPT mismatch as elsewhere in this change. Runtime semantics are identical.

View File

@@ -0,0 +1,82 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import {
framingConfigSchema,
packTransformOptionsSchema,
unpackTransformOptionsSchema,
} from "./transform.schema.js";
describe("framingConfigSchema", () => {
it("accepts length-prefix framing", () => {
const cfg = {
type: "length-prefix" as const,
lengthPrefixSize: 4 as const,
};
expect(framingConfigSchema.parse(cfg)).toEqual(cfg);
});
it("accepts delimiter framing with a Uint8Array delimiter", () => {
const cfg = {
type: "delimiter" as const,
delimiter: new Uint8Array([0]),
};
const parsed = framingConfigSchema.parse(cfg);
expect(parsed.type).toBe("delimiter");
expect(parsed.delimiter).toBe(cfg.delimiter);
});
it("accepts fixed-size framing", () => {
const cfg = { type: "fixed-size" as const, fixedSize: 64 };
expect(framingConfigSchema.parse(cfg)).toEqual(cfg);
});
it("accepts the degenerate `none` type", () => {
const cfg = { type: "none" as const };
expect(framingConfigSchema.parse(cfg)).toEqual(cfg);
});
it("rejects an unknown type token", () => {
expect(() => framingConfigSchema.parse({ type: "ndjson" })).toThrow();
});
it("rejects a non-Uint8Array delimiter", () => {
expect(() =>
framingConfigSchema.parse({
type: "delimiter",
delimiter: [0],
})
).toThrow();
});
it("rejects an unsupported lengthPrefixSize", () => {
expect(() =>
framingConfigSchema.parse({
type: "length-prefix",
lengthPrefixSize: 8,
})
).toThrow();
});
});
describe("packTransformOptionsSchema", () => {
it("accepts an empty bundle", () => {
expect(packTransformOptionsSchema.parse({})).toEqual({});
});
it("accepts batchSize and highWaterMark", () => {
const opts = { batchSize: 4, highWaterMark: 32 };
expect(packTransformOptionsSchema.parse(opts)).toEqual(opts);
});
it("rejects non-positive batchSize", () => {
expect(() =>
packTransformOptionsSchema.parse({ batchSize: 0 })
).toThrow();
});
});
describe("unpackTransformOptionsSchema", () => {
it("accepts headerSize", () => {
const opts = { headerSize: 4 };
expect(unpackTransformOptionsSchema.parse(opts)).toEqual(opts);
});
});

View File

@@ -0,0 +1,73 @@
import type { FramingConfig } from "@suaveplan/types";
import z from "zod";
import type {
PackTransformOptions,
UnpackTransformOptions,
} from "./transform.types.js";
const callable = <T>(message: string) =>
z.custom<T>((v): v is T => typeof v === "function", { message });
const uint8ArraySchema = z.custom<Uint8Array>(
(v): v is Uint8Array => v instanceof Uint8Array,
{ message: "delimiter must be a Uint8Array" }
);
/**
* Strict Zod schema for {@link FramingConfig}.
*
* Validates the framing-strategy descriptor that drives `parseFrames`,
* `addFrame`, and the framed pack/unpack streams. Cross-field requirements
* are NOT enforced here (e.g. `lengthPrefixSize` is only meaningful for
* `type: "length-prefix"`) because the runtime parser tolerates the extra
* fields. Strict-mode rejects unknown keys.
*/
export const framingConfigSchema = z
.object({
type: z.union([
z.literal("length-prefix"),
z.literal("delimiter"),
z.literal("fixed-size"),
z.literal("none"),
]),
lengthPrefixSize: z
.union([z.literal(1), z.literal(2), z.literal(4)])
.optional(),
delimiter: uint8ArraySchema.optional(),
fixedSize: z.number().int().positive().optional(),
})
.strict() as unknown as z.ZodType<FramingConfig>;
/**
* Strict Zod schema for {@link PackTransformOptions}.
*
* Validates `createPackTransformStream`'s options bundle. `batchSize` and
* `highWaterMark` must be positive integers; `onProgress` is a function
* preserved by identity.
*/
export const packTransformOptionsSchema = z
.object({
batchSize: z.number().int().positive().optional(),
highWaterMark: z.number().int().positive().optional(),
onProgress: callable<NonNullable<PackTransformOptions["onProgress"]>>(
"onProgress must be a function"
).optional(),
})
.strict() as unknown as z.ZodType<PackTransformOptions>;
/**
* Strict Zod schema for {@link UnpackTransformOptions}.
*
* Validates `createUnpackTransformStream`'s options bundle. `headerSize` is
* the length-prefix width when the function infers a `length-prefix`
* framing config.
*/
export const unpackTransformOptionsSchema = z
.object({
highWaterMark: z.number().int().positive().optional(),
onProgress: callable<NonNullable<UnpackTransformOptions["onProgress"]>>(
"onProgress must be a function"
).optional(),
headerSize: z.number().int().positive().optional(),
})
.strict() as unknown as z.ZodType<UnpackTransformOptions>;

View File

@@ -0,0 +1,365 @@
import type { BitPacker, BitPackerSchema } from "@suaveplan/bitpacker";
import type {
FrameParseResult,
FramingConfig,
PackTransformOptions,
UnpackTransformOptions,
} from "./transform.types.js";
export * from "./transform.types.js";
export function createPackTransformStream<T extends BitPackerSchema>(
bitPacker: BitPacker,
schema: T,
options: PackTransformOptions = {}
): TransformStream<unknown, Uint8Array> {
const { batchSize = 1, highWaterMark = 16, onProgress } = options;
let itemsProcessed = 0;
let bytesProcessed = 0;
let startTime = Date.now();
let batch: unknown[] = [];
const updateProgress = () => {
if (onProgress) {
const duration = (Date.now() - startTime) / 1000;
onProgress({
itemsProcessed,
bytesProcessed,
itemsPending: batch.length,
throughput: duration > 0 ? itemsProcessed / duration : 0,
operation: "pack",
});
}
};
return new TransformStream<unknown, Uint8Array>(
{
start() {
startTime = Date.now();
},
transform(chunk, controller) {
batch.push(chunk);
if (batch.length >= batchSize) {
for (const item of batch) {
const packed = bitPacker.pack(item, schema);
bytesProcessed += packed.length;
itemsProcessed++;
controller.enqueue(packed);
}
batch = [];
updateProgress();
}
},
flush(controller) {
for (const item of batch) {
const packed = bitPacker.pack(item, schema);
bytesProcessed += packed.length;
itemsProcessed++;
controller.enqueue(packed);
}
batch = [];
updateProgress();
},
},
{ highWaterMark },
{ highWaterMark }
);
}
export function createUnpackTransformStream<T extends BitPackerSchema>(
bitPacker: BitPacker,
schema: T,
options: UnpackTransformOptions = {}
): TransformStream<Uint8Array, unknown> {
const { highWaterMark = 16, onProgress, headerSize = 4 } = options;
let itemsProcessed = 0;
let bytesProcessed = 0;
let startTime = Date.now();
let buffer: Uint8Array = new Uint8Array(0);
const framing: FramingConfig = {
type: "length-prefix",
lengthPrefixSize: headerSize as 1 | 2 | 4,
};
const updateProgress = () => {
if (onProgress) {
const duration = (Date.now() - startTime) / 1000;
onProgress({
itemsProcessed,
bytesProcessed,
itemsPending: 0,
throughput: duration > 0 ? itemsProcessed / duration : 0,
operation: "unpack",
});
}
};
return new TransformStream<Uint8Array, unknown>(
{
start() {
startTime = Date.now();
},
transform(chunk, controller) {
const newBuffer = new Uint8Array(buffer.length + chunk.length);
newBuffer.set(buffer);
newBuffer.set(chunk, buffer.length);
buffer = newBuffer;
const { frames, remainder } = parseFrames(buffer, framing);
buffer = remainder;
for (const frame of frames) {
const unpacked = bitPacker.unpack(frame, schema);
bytesProcessed += frame.length;
itemsProcessed++;
controller.enqueue(unpacked);
}
updateProgress();
},
flush(controller) {
if (buffer.length > 0) {
try {
const unpacked = bitPacker.unpack(buffer, schema);
bytesProcessed += buffer.length;
itemsProcessed++;
controller.enqueue(unpacked);
} catch {
// Incomplete data, ignore
}
}
updateProgress();
},
},
{ highWaterMark },
{ highWaterMark }
);
}
export function createFramedPackStream<T extends BitPackerSchema>(
bitPacker: BitPacker,
schema: T,
framing: FramingConfig = { type: "length-prefix", lengthPrefixSize: 4 },
options: PackTransformOptions = {}
): TransformStream<unknown, Uint8Array> {
const { highWaterMark = 16, onProgress } = options;
let itemsProcessed = 0;
let bytesProcessed = 0;
let startTime = Date.now();
return new TransformStream<unknown, Uint8Array>(
{
start() {
startTime = Date.now();
},
transform(chunk, controller) {
const packed = bitPacker.pack(chunk, schema);
const framed = addFrame(packed, framing);
bytesProcessed += framed.length;
itemsProcessed++;
controller.enqueue(framed);
if (onProgress) {
const duration = (Date.now() - startTime) / 1000;
onProgress({
itemsProcessed,
bytesProcessed,
itemsPending: 0,
throughput:
duration > 0 ? itemsProcessed / duration : 0,
operation: "pack",
});
}
},
},
{ highWaterMark },
{ highWaterMark }
);
}
export function createFramedUnpackStream<T extends BitPackerSchema>(
bitPacker: BitPacker,
schema: T,
framing: FramingConfig = { type: "length-prefix", lengthPrefixSize: 4 },
options: UnpackTransformOptions = {}
): TransformStream<Uint8Array, unknown> {
const { highWaterMark = 16, onProgress } = options;
let itemsProcessed = 0;
let bytesProcessed = 0;
let startTime = Date.now();
let buffer: Uint8Array = new Uint8Array(0);
return new TransformStream<Uint8Array, unknown>(
{
start() {
startTime = Date.now();
},
transform(chunk, controller) {
const newBuffer = new Uint8Array(buffer.length + chunk.length);
newBuffer.set(buffer);
newBuffer.set(chunk, buffer.length);
buffer = newBuffer;
const { frames, remainder } = parseFrames(buffer, framing);
buffer = remainder;
for (const frame of frames) {
const unpacked = bitPacker.unpack(frame, schema);
bytesProcessed += frame.length;
itemsProcessed++;
controller.enqueue(unpacked);
}
if (onProgress) {
const duration = (Date.now() - startTime) / 1000;
onProgress({
itemsProcessed,
bytesProcessed,
itemsPending: 0,
throughput:
duration > 0 ? itemsProcessed / duration : 0,
operation: "unpack",
});
}
},
flush() {
// With framing.type === "none", parseFrames always consumes
// the entire buffer, so there is never any remainder here.
},
},
{ highWaterMark },
{ highWaterMark }
);
}
export function parseFrames(
buffer: Uint8Array,
framing: FramingConfig
): FrameParseResult {
const frames: Uint8Array[] = [];
let offset = 0;
switch (framing.type) {
case "length-prefix": {
const prefixSize = framing.lengthPrefixSize ?? 4;
while (offset + prefixSize <= buffer.length) {
const length = readLengthPrefix(buffer, offset, prefixSize);
if (offset + prefixSize + length > buffer.length) break;
frames.push(
buffer.slice(
offset + prefixSize,
offset + prefixSize + length
)
);
offset += prefixSize + length;
}
break;
}
case "delimiter": {
const delimiter = framing.delimiter ?? new Uint8Array([0]);
let start = 0;
for (let i = 0; i <= buffer.length - delimiter.length; i++) {
let match = true;
for (let j = 0; j < delimiter.length; j++) {
if (buffer[i + j] !== delimiter[j]) {
match = false;
break;
}
}
if (match) {
if (i > start) frames.push(buffer.slice(start, i));
start = i + delimiter.length;
i = start - 1;
}
}
offset = start;
break;
}
case "fixed-size": {
const size = framing.fixedSize ?? 256;
while (offset + size <= buffer.length) {
frames.push(buffer.slice(offset, offset + size));
offset += size;
}
break;
}
case "none":
if (buffer.length > 0) {
frames.push(buffer);
offset = buffer.length;
}
break;
}
return { frames, remainder: buffer.slice(offset) };
}
export function addFrame(data: Uint8Array, framing: FramingConfig): Uint8Array {
switch (framing.type) {
case "length-prefix": {
const prefixSize = framing.lengthPrefixSize ?? 4;
const framed = new Uint8Array(prefixSize + data.length);
writeLengthPrefix(framed, 0, data.length, prefixSize);
framed.set(data, prefixSize);
return framed;
}
case "delimiter": {
const delimiter = framing.delimiter ?? new Uint8Array([0]);
const framed = new Uint8Array(data.length + delimiter.length);
framed.set(data);
framed.set(delimiter, data.length);
return framed;
}
case "fixed-size": {
const size = framing.fixedSize ?? 256;
const framed = new Uint8Array(size);
framed.set(data.slice(0, size));
return framed;
}
default:
return data;
}
}
function readLengthPrefix(
buffer: Uint8Array,
offset: number,
size: 1 | 2 | 4
): number {
switch (size) {
case 1:
return buffer[offset]!;
case 2:
return (buffer[offset]! << 8) | buffer[offset + 1]!;
case 4:
return (
(buffer[offset]! << 24) |
(buffer[offset + 1]! << 16) |
(buffer[offset + 2]! << 8) |
buffer[offset + 3]!
);
}
}
function writeLengthPrefix(
buffer: Uint8Array,
offset: number,
length: number,
size: 1 | 2 | 4
): void {
switch (size) {
case 1:
buffer[offset] = length & 0xff;
break;
case 2:
buffer[offset] = (length >> 8) & 0xff;
buffer[offset + 1] = length & 0xff;
break;
case 4:
buffer[offset] = (length >> 24) & 0xff;
buffer[offset + 1] = (length >> 16) & 0xff;
buffer[offset + 2] = (length >> 8) & 0xff;
buffer[offset + 3] = length & 0xff;
break;
}
}

View File

@@ -0,0 +1,8 @@
## transform.types
**Path:** `bitpacker-streaming/src/transform/transform.types.ts`
**Export:** See source for details.
### What it does
Part of the @suaveplan/ package.

View File

@@ -0,0 +1,55 @@
/**
* Bitpacker-streaming option bundles.
*
* The five generic streaming primitives — `TransformProgress`,
* `FramingConfig`, `FrameParseResult`, `StreamTransformer`,
* `PipelineStage` — live in `@suaveplan/types` so any data-pipeline
* package can speak the same vocabulary. Only the bitpack/unpack-flavored
* options stay here.
*/
import type {
FrameParseResult,
FramingConfig,
PipelineStage,
StreamTransformer,
TransformProgress,
} from "@suaveplan/types";
export type {
FrameParseResult,
FramingConfig,
PipelineStage,
StreamTransformer,
TransformProgress,
};
/**
* Legacy alias kept so downstream code that imports
* `BitPackerTransformer` keeps compiling. New code should import
* `StreamTransformer` from `@suaveplan/types` directly.
*
* @deprecated Use `StreamTransformer` from `@suaveplan/types`.
*/
export type BitPackerTransformer<TIn, TOut> = StreamTransformer<TIn, TOut>;
export interface PackTransformOptions {
/** Number of items to pack per emitted chunk. Default `1`. */
batchSize?: number;
/** WHATWG TransformStream high-water mark. Default `16`. */
highWaterMark?: number;
/** Optional progress callback. */
onProgress?: (progress: TransformProgress) => void;
}
export interface UnpackTransformOptions {
/** WHATWG TransformStream high-water mark. Default `16`. */
highWaterMark?: number;
/** Optional progress callback. */
onProgress?: (progress: TransformProgress) => void;
/**
* Length-prefix width when `createUnpackTransformStream` infers a
* `length-prefix` framing config. Default `4`.
*/
headerSize?: number;
}

View File

@@ -0,0 +1,12 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "../../../../tsconfig.base.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"lib": ["ES2022"],
"types": ["node"]
},
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
}

View File

@@ -0,0 +1,37 @@
import { defineConfig } from "vite";
import dts from "vite-plugin-dts";
export default defineConfig({
root: __dirname,
cacheDir: "../../../node_modules/.vite/packages/bitpacker",
plugins: [
dts({
entryRoot: "src",
tsconfigPath: "./tsconfig.lib.json",
insertTypesEntry: true,
}),
],
build: {
outDir: "./dist",
emptyOutDir: true,
reportCompressedSize: true,
sourcemap: true,
lib: { entry: "src/index.ts", formats: ["es"] },
rollupOptions: {
treeshake: true,
output: {
preserveModules: true,
preserveModulesRoot: "src",
entryFileNames: "[name].js",
},
external: [
"tslib",
"zod",
"@opentelemetry/api",
"@suaveplan/error",
"@suaveplan/types",
/^node:/,
],
},
},
});

View File

@@ -0,0 +1,9 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
coverage: {
provider: "istanbul",
},
},
});