Initial Commit
This commit is contained in:
89
packages/foundation/bitpacker/bitpacker-streaming/README.md
Normal file
89
packages/foundation/bitpacker/bitpacker-streaming/README.md
Normal file
@@ -0,0 +1,89 @@
|
||||
# @suaveplan/bitpacker-streaming - Bitpacker streaming plugin
|
||||
|
||||
## Summary
|
||||
|
||||
Streaming APIs for bitpacker including chunked packing/unpacking and true TransformStream support with framing.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [defaults](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [register-schemas](./src/register-schemas/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
|
||||
- [register-schemas](./src/register-schemas/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
|
||||
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - transform.schema — Zod schemas for the option bundl…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - transform.schema — Zod schemas for the option bundl…
|
||||
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
|
||||
<!-- end:auto-docs --> - [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
|
||||
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
|
||||
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
|
||||
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
|
||||
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
|
||||
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
|
||||
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- transform.schema — Zod schemas for the option bundl…
|
||||
- [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
|
||||
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
|
||||
- [default-clock](./src/defaults/default-clock.md) — <!-- begin:auto-docs --> _(no submodules)_- [otel](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_ - [instruments](./src/otel/instruments.md) — <!-- begin:auto-docs --> _(no submodules)_- [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [streaming-bit-packer](./src/streaming-bit-packer/streaming-bit-packer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [streaming-bit-packer.schema](./src/streaming-bit-packer/streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ - [transform](./src/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ - [transform.schema](./src/transform/transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- [index](./src/index.md) — **Path:** `bitpacker-streaming/src/index.ts` **Export:** See source for details.
|
||||
- [register-schemas](./src/register-schemas.md) — Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `reg…
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/bitpacker-streaming
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```typescript
|
||||
import { BitPacker } from "@suaveplan/bitpacker";
|
||||
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
|
||||
import { packToChunks, unpackFromChunks, createFramedPackStream, createFramedUnpackStream } from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
|
||||
// Chunked packing
|
||||
const chunks = packToChunks(42, schema, { chunkSize: 2 });
|
||||
const value = unpackFromChunks<number>(chunks, schema);
|
||||
|
||||
// Transform streams with framing
|
||||
const packStream = createFramedPackStream(bp, schema);
|
||||
const unpackStream = createFramedUnpackStream(bp, schema);
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read(); // 42
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- `src/index.ts` — `packToChunks`, `unpackFromChunks`, `unpackFromChunksAsync`
|
||||
- `src/transform/transform.ts` — `TransformStream` factories with length-prefix, delimiter, fixed-size, and no-framing options
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"name": "@suaveplan/bitpacker-streaming",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Bitpacker streaming plugin",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./register-schemas": {
|
||||
"import": {
|
||||
"types": "./dist/register-schemas/index.d.ts",
|
||||
"default": "./dist/register-schemas/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/register-schemas/index.d.ts",
|
||||
"default": "./dist/register-schemas/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/"
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/bitpacker": "workspace:^",
|
||||
"@suaveplan/bitpacker-strategies": "workspace:^",
|
||||
"@suaveplan/schema-registry": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
# defaultClock
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`defaultClock` is the single point in `@suaveplan/bitpacker-streaming` where the wall clock is read. Every other module that needs a time reading either accepts a `Clock` from the caller or imports this default. Centralising the `Date.now()` call this way makes the package deterministic-by-default in tests (callers inject `FakeClock`) while the production hot path still reads the real wall clock.
|
||||
|
||||
This pattern matches the convention used by `@suaveplan/bitpacker-network` and `@suaveplan/bitpacker-compressed`. Suaveplan enforces a "no module-scope `Date.now()`/`Math.random()`" rule outside allow-listed defaults — this file is one of those allow-listed defaults.
|
||||
|
||||
## Features
|
||||
|
||||
- Single named export `defaultClock: Clock` typed via `@suaveplan/types`.
|
||||
- Pure function — no side effects beyond reading the wall clock.
|
||||
- Idiomatic Suaveplan convention for clock injection.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { defaultClock } from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
const now = defaultClock();
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
import { StreamingBitPacker } from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
// In tests, inject a deterministic clock instead of the default.
|
||||
const fakeClock: Clock = (() => {
|
||||
let t = 0;
|
||||
return () => {
|
||||
t += 100;
|
||||
return t;
|
||||
};
|
||||
})();
|
||||
|
||||
const packer = new StreamingBitPacker({ x: 255 });
|
||||
await packer.packStream({ x: 1 }, { clock: fakeClock });
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `defaultClock: Clock`
|
||||
|
||||
Returns the current wall-clock time in milliseconds (`Date.now()`).
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The `Clock` type is `() => number`. suaveplan prefers function types over class instances for monotonic primitives because they are trivially serialisable, comparable, and substitutable in tests without dependency injection frameworks.
|
||||
|
||||
The reason this lives in `defaults/` rather than as a static class member is the suaveplan lint rule that bans module-scope `Date.now()` calls in production code. By scoping the call to a single named, lint-allow-listed module, we keep the rest of the package free of static analysis exceptions.
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Package-default `Clock` for streaming throughput timing.
|
||||
*
|
||||
* Centralising the single `Date.now()` call here means the rest of the
|
||||
* package never reads the wall clock directly — every site that needs a
|
||||
* time reading either accepts an injectable `Clock` from the caller or
|
||||
* imports this default. This is the same convention used by
|
||||
* `@suaveplan/bitpacker-network` and `@suaveplan/bitpacker-compressed`.
|
||||
*/
|
||||
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
|
||||
/**
|
||||
* The package-wide default `Clock`. Returns `Date.now()`. Tests should
|
||||
* inject a `FakeClock` from `@suaveplan/testing` instead of using this.
|
||||
*/
|
||||
export const defaultClock: Clock = () => Date.now();
|
||||
@@ -0,0 +1,8 @@
|
||||
## index
|
||||
|
||||
**Path:** `bitpacker-streaming/src/index.ts`
|
||||
**Export:** See source for details.
|
||||
|
||||
### What it does
|
||||
|
||||
Part of the @suaveplan/ package.
|
||||
@@ -0,0 +1,99 @@
|
||||
import { BitPacker } from "@suaveplan/bitpacker";
|
||||
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
|
||||
import type { BitPackerSchema } from "@suaveplan/types";
|
||||
|
||||
export interface StreamingOptions {
|
||||
readonly chunkSize?: number;
|
||||
readonly signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export function packToChunks(
|
||||
data: unknown,
|
||||
schema: BitPackerSchema,
|
||||
options: StreamingOptions = {}
|
||||
): Uint8Array[] {
|
||||
const { chunkSize = 1024 } = options;
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const bytes = bp.pack(data, schema);
|
||||
const chunks: Uint8Array[] = [];
|
||||
for (let offset = 0; offset < bytes.length; offset += chunkSize) {
|
||||
chunks.push(bytes.slice(offset, offset + chunkSize));
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
|
||||
export function unpackFromChunks<T = unknown>(
|
||||
chunks: Iterable<Uint8Array>,
|
||||
schema: BitPackerSchema
|
||||
): T {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const arrays: Uint8Array[] = [];
|
||||
let totalLength = 0;
|
||||
for (const chunk of chunks) {
|
||||
if (chunk.length === 0) continue;
|
||||
arrays.push(chunk);
|
||||
totalLength += chunk.length;
|
||||
}
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
combined.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
return bp.unpack(combined, schema) as T;
|
||||
}
|
||||
|
||||
export async function unpackFromChunksAsync<T = unknown>(
|
||||
chunks: AsyncIterable<Uint8Array>,
|
||||
schema: BitPackerSchema,
|
||||
options: StreamingOptions = {}
|
||||
): Promise<T> {
|
||||
const { signal } = options;
|
||||
const arrays: Uint8Array[] = [];
|
||||
let totalLength = 0;
|
||||
for await (const chunk of chunks) {
|
||||
if (signal?.aborted) throw new Error("Stream aborted");
|
||||
if (chunk.length === 0) continue;
|
||||
arrays.push(chunk);
|
||||
totalLength += chunk.length;
|
||||
}
|
||||
const combined = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const arr of arrays) {
|
||||
combined.set(arr, offset);
|
||||
offset += arr.length;
|
||||
}
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
return bp.unpack(combined, schema) as T;
|
||||
}
|
||||
|
||||
export { defaultClock } from "./defaults/default-clock.js";
|
||||
export {
|
||||
type ChunkProcessor,
|
||||
type ReadableStreamFromBytesOptions,
|
||||
StreamingBitPacker,
|
||||
type StreamingBitPackerOptions,
|
||||
type StreamingPackOptions,
|
||||
type StreamingUnpackOptions,
|
||||
type WritablePackOptions,
|
||||
type WritableUnpackOptions,
|
||||
} from "./streaming-bit-packer/streaming-bit-packer.js";
|
||||
export {
|
||||
addFrame,
|
||||
type BitPackerTransformer,
|
||||
createFramedPackStream,
|
||||
createFramedUnpackStream,
|
||||
createPackTransformStream,
|
||||
createUnpackTransformStream,
|
||||
type FrameParseResult,
|
||||
type FramingConfig,
|
||||
type PackTransformOptions,
|
||||
type PipelineStage,
|
||||
parseFrames,
|
||||
type StreamTransformer,
|
||||
type TransformProgress,
|
||||
type UnpackTransformOptions,
|
||||
} from "./transform/transform.js";
|
||||
@@ -0,0 +1,84 @@
|
||||
# Bitpacker Streaming OTel Instruments
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
OTel meter, tracer, counter, and histogram declarations for `@suaveplan/bitpacker-streaming`. Ships canonical instrumentation primitives for the streaming pack/unpack pipeline plus the framing layer (`length-prefix`, `delimiter`, `fixed-size`, `none`). Centralizing the metric/span vocabulary here lets every entry point — `StreamingBitPacker.packStream`, `createPackTransformStream`, `createFramedUnpackStream` — emit consistent observability data.
|
||||
|
||||
The metric and span names follow the suaveplan convention with snake-case for the multi-word package: tracer/meter named `@suaveplan/bitpacker-streaming`; spans named `bitpacker_streaming.<operation>`; counters/histograms named `bitpacker_streaming.<scope>.<metric>`.
|
||||
|
||||
## Features
|
||||
|
||||
- `getBitPackerStreamingTracer()` — returns the package's tracer
|
||||
- `streamingOpsTotal` — counter labelled by `op` and `success`
|
||||
- `streamingOpDurationMs` — histogram of pack/unpack duration
|
||||
- `streamingChunkBytes` — histogram of bytes-per-chunk on the pipeline
|
||||
- `streamingFramesTotal` — counter labelled by `framing` strategy
|
||||
- `BITPACKER_STREAMING_SPAN_NAMES` — four canonical span names
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
BITPACKER_STREAMING_SPAN_NAMES,
|
||||
streamingOpsTotal,
|
||||
getBitPackerStreamingTracer,
|
||||
} from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
const tracer = getBitPackerStreamingTracer();
|
||||
tracer.startActiveSpan(BITPACKER_STREAMING_SPAN_NAMES.pack, (span) => {
|
||||
try {
|
||||
// pack work
|
||||
streamingOpsTotal.add(1, { op: "pack", success: "true" });
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
streamingChunkBytes,
|
||||
streamingFramesTotal,
|
||||
} from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
function recordChunk(chunk: Uint8Array, framing: "length-prefix" | "delimiter") {
|
||||
streamingChunkBytes.record(chunk.byteLength, { op: "pack" });
|
||||
streamingFramesTotal.add(1, { framing });
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `getBitPackerStreamingTracer(): Tracer`
|
||||
|
||||
Returns the OTel `Tracer` instance for the package.
|
||||
|
||||
### `streamingOpsTotal: Counter`
|
||||
|
||||
Counter labelled by `op: "pack" | "unpack"` and `success: "true" | "false"`.
|
||||
|
||||
### `streamingOpDurationMs: Histogram`
|
||||
|
||||
Histogram of elapsed milliseconds per pack/unpack call, labelled by `op`.
|
||||
|
||||
### `streamingChunkBytes: Histogram`
|
||||
|
||||
Histogram of bytes per chunk on the pipeline, labelled by `op`.
|
||||
|
||||
### `streamingFramesTotal: Counter`
|
||||
|
||||
Counter labelled by `framing: "length-prefix" | "delimiter" | "fixed-size" | "none"`.
|
||||
|
||||
### `BITPACKER_STREAMING_SPAN_NAMES`
|
||||
|
||||
Const map: `{ pack, unpack, framedPack, framedUnpack }` → canonical span names.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- **Per-chunk events, not per-chunk spans.** A streaming pack producing 10k chunks would produce 10k spans if each were its own; instead, chunks are recorded as `streamingChunkBytes` histogram observations and as nested span events on the parent pack/unpack span.
|
||||
- **Framing is a counter label.** `streamingFramesTotal{framing}` lets dashboards group frame throughput by framing strategy without spawning separate counters per type.
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
BITPACKER_STREAMING_SPAN_NAMES,
|
||||
getBitPackerStreamingTracer,
|
||||
streamingChunkBytes,
|
||||
streamingFramesTotal,
|
||||
streamingOpDurationMs,
|
||||
streamingOpsTotal,
|
||||
} from "./instruments.js";
|
||||
|
||||
describe("OTel instruments — declarations", () => {
|
||||
it("exposes the canonical span names", () => {
|
||||
expect(BITPACKER_STREAMING_SPAN_NAMES.pack).toBe(
|
||||
"bitpacker_streaming.pack"
|
||||
);
|
||||
expect(BITPACKER_STREAMING_SPAN_NAMES.unpack).toBe(
|
||||
"bitpacker_streaming.unpack"
|
||||
);
|
||||
expect(BITPACKER_STREAMING_SPAN_NAMES.framedPack).toBe(
|
||||
"bitpacker_streaming.framed_pack"
|
||||
);
|
||||
expect(BITPACKER_STREAMING_SPAN_NAMES.framedUnpack).toBe(
|
||||
"bitpacker_streaming.framed_unpack"
|
||||
);
|
||||
});
|
||||
|
||||
it("counters and histograms are constructed without error", () => {
|
||||
expect(streamingOpsTotal).toBeDefined();
|
||||
expect(streamingOpDurationMs).toBeDefined();
|
||||
expect(streamingChunkBytes).toBeDefined();
|
||||
expect(streamingFramesTotal).toBeDefined();
|
||||
});
|
||||
|
||||
it("tracer is reachable", () => {
|
||||
expect(getBitPackerStreamingTracer()).toBeDefined();
|
||||
});
|
||||
|
||||
it("counters accept the documented labels", () => {
|
||||
expect(() =>
|
||||
streamingOpsTotal.add(1, { op: "pack", success: "true" })
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
streamingFramesTotal.add(1, { framing: "length-prefix" })
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { metrics, type Tracer, trace } from "@opentelemetry/api";
|
||||
|
||||
const meter = metrics.getMeter("@suaveplan/bitpacker-streaming");
|
||||
|
||||
export function getBitPackerStreamingTracer(): Tracer {
|
||||
return trace.getTracer("@suaveplan/bitpacker-streaming");
|
||||
}
|
||||
|
||||
/**
|
||||
* Counter incremented once per top-level streaming pack/unpack call. Labels:
|
||||
* `op` (`pack` | `unpack`) and `success` (`true` | `false`).
|
||||
*/
|
||||
export const streamingOpsTotal = meter.createCounter(
|
||||
"bitpacker_streaming.ops.total",
|
||||
{
|
||||
description: "Total streaming pack/unpack operations",
|
||||
unit: "{ops}",
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* Histogram of streaming-op duration in milliseconds, labelled by `op`.
|
||||
*/
|
||||
export const streamingOpDurationMs = meter.createHistogram(
|
||||
"bitpacker_streaming.op.duration_ms",
|
||||
{
|
||||
description: "Duration of streaming pack/unpack in milliseconds",
|
||||
unit: "ms",
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* Histogram of bytes-per-chunk emitted on the streaming pipeline, labelled
|
||||
* by `op`.
|
||||
*/
|
||||
export const streamingChunkBytes = meter.createHistogram(
|
||||
"bitpacker_streaming.chunk.bytes",
|
||||
{
|
||||
description: "Bytes per chunk on the streaming pipeline",
|
||||
unit: "By",
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* Counter incremented per frame parsed by the framing pipeline, labelled by
|
||||
* `framing` (`length-prefix` | `delimiter` | `fixed-size` | `none`).
|
||||
*/
|
||||
export const streamingFramesTotal = meter.createCounter(
|
||||
"bitpacker_streaming.frames.total",
|
||||
{
|
||||
description: "Total frames parsed by the streaming framing pipeline",
|
||||
unit: "{frames}",
|
||||
}
|
||||
);
|
||||
|
||||
export const BITPACKER_STREAMING_SPAN_NAMES = {
|
||||
pack: "bitpacker_streaming.pack",
|
||||
unpack: "bitpacker_streaming.unpack",
|
||||
framedPack: "bitpacker_streaming.framed_pack",
|
||||
framedUnpack: "bitpacker_streaming.framed_unpack",
|
||||
} as const;
|
||||
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
BITPACKER_STREAMING_SCHEMA_VERSION,
|
||||
BITPACKER_STREAMING_SUBJECT,
|
||||
registerBitPackerStreamingSchemas,
|
||||
} from "./register-schemas.js";
|
||||
@@ -0,0 +1,66 @@
|
||||
# Register Schemas — `@suaveplan/bitpacker-streaming`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Idempotent integration with `@suaveplan/schema-registry` for the `@suaveplan/bitpacker-streaming` package. Exposes `registerBitPackerStreamingSchemas(registry)` which converts the package's nine public boundary schemas into JSON Schema form and registers them under `bitpacker:@suaveplan/bitpacker-streaming` at version 1. Re-registration on the same registry is a no-op.
|
||||
|
||||
The nine schemas covered are: the six streaming-bit-packer option bundles (`streamingBitPackerOptions`, `streamingPackOptions`, `streamingUnpackOptions`, `readableStreamFromBytesOptions`, `writablePackOptions`, `writableUnpackOptions`) plus the three transform-stream schemas (`framingConfig`, `packTransformOptions`, `unpackTransformOptions`).
|
||||
|
||||
## Features
|
||||
|
||||
- `BITPACKER_STREAMING_SUBJECT` constant — `"bitpacker:@suaveplan/bitpacker-streaming"`
|
||||
- `BITPACKER_STREAMING_SCHEMA_VERSION` constant — `1`
|
||||
- `registerBitPackerStreamingSchemas(registry)` — idempotent registration
|
||||
- Nine-key payload covering every option bundle the package accepts
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { createRegistry } from "@suaveplan/schema-registry";
|
||||
import {
|
||||
BITPACKER_STREAMING_SUBJECT,
|
||||
registerBitPackerStreamingSchemas,
|
||||
} from "@suaveplan/bitpacker-streaming/register-schemas";
|
||||
|
||||
const registry = createRegistry();
|
||||
registerBitPackerStreamingSchemas(registry);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { createRegistry, SchemaRegistry } from "@suaveplan/schema-registry";
|
||||
import { registerBitPackerSchemas } from "@suaveplan/bitpacker/register-schemas";
|
||||
import { registerBitPackerStreamingSchemas } from "@suaveplan/bitpacker-streaming/register-schemas";
|
||||
|
||||
const registry = createRegistry();
|
||||
registerBitPackerSchemas(registry);
|
||||
registerBitPackerStreamingSchemas(registry);
|
||||
|
||||
const payload = registry.getLatest("bitpacker:@suaveplan/bitpacker-streaming")
|
||||
.schema as Record<string, unknown>;
|
||||
// payload.framingConfig, payload.streamingPackOptions, …
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `BITPACKER_STREAMING_SUBJECT: "bitpacker:@suaveplan/bitpacker-streaming"`
|
||||
|
||||
Const string.
|
||||
|
||||
### `BITPACKER_STREAMING_SCHEMA_VERSION: 1`
|
||||
|
||||
Const number.
|
||||
|
||||
### `registerBitPackerStreamingSchemas(registry: SchemaRegistry): void`
|
||||
|
||||
Idempotent registration. Returns without writing if the subject is already present.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- **Nine-key payload.** Each key corresponds to one of the package's exported schemas.
|
||||
- **`unrepresentable: "any"`.** Required because the schemas use `z.custom` for `AbortSignal`, callables, and `Uint8Array` checks.
|
||||
@@ -0,0 +1,44 @@
|
||||
import { createRegistry } from "@suaveplan/schema-registry";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
BITPACKER_STREAMING_SCHEMA_VERSION,
|
||||
BITPACKER_STREAMING_SUBJECT,
|
||||
registerBitPackerStreamingSchemas,
|
||||
} from "./register-schemas.js";
|
||||
|
||||
describe("registerBitPackerStreamingSchemas", () => {
|
||||
it("registers under the namespaced subject at version 1", () => {
|
||||
const registry = createRegistry();
|
||||
registerBitPackerStreamingSchemas(registry);
|
||||
expect(registry.getLatest(BITPACKER_STREAMING_SUBJECT).version).toBe(
|
||||
BITPACKER_STREAMING_SCHEMA_VERSION
|
||||
);
|
||||
});
|
||||
|
||||
it("includes every public schema in the payload", () => {
|
||||
const registry = createRegistry();
|
||||
registerBitPackerStreamingSchemas(registry);
|
||||
const payload = registry.getLatest(BITPACKER_STREAMING_SUBJECT)
|
||||
.schema as Record<string, unknown>;
|
||||
for (const key of [
|
||||
"streamingBitPackerOptions",
|
||||
"streamingPackOptions",
|
||||
"streamingUnpackOptions",
|
||||
"readableStreamFromBytesOptions",
|
||||
"writablePackOptions",
|
||||
"writableUnpackOptions",
|
||||
"framingConfig",
|
||||
"packTransformOptions",
|
||||
"unpackTransformOptions",
|
||||
]) {
|
||||
expect(payload).toHaveProperty(key);
|
||||
}
|
||||
});
|
||||
|
||||
it("is idempotent across repeated calls", () => {
|
||||
const registry = createRegistry();
|
||||
registerBitPackerStreamingSchemas(registry);
|
||||
registerBitPackerStreamingSchemas(registry);
|
||||
expect(registry.getLatest(BITPACKER_STREAMING_SUBJECT).version).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import type { SchemaRegistry } from "@suaveplan/schema-registry";
|
||||
import z from "zod";
|
||||
import {
|
||||
readableStreamFromBytesOptionsSchema,
|
||||
streamingBitPackerOptionsSchema,
|
||||
streamingPackOptionsSchema,
|
||||
streamingUnpackOptionsSchema,
|
||||
writablePackOptionsSchema,
|
||||
writableUnpackOptionsSchema,
|
||||
} from "../streaming-bit-packer/streaming-bit-packer.schema.js";
|
||||
import {
|
||||
framingConfigSchema,
|
||||
packTransformOptionsSchema,
|
||||
unpackTransformOptionsSchema,
|
||||
} from "../transform/transform.schema.js";
|
||||
|
||||
export const BITPACKER_STREAMING_SUBJECT =
|
||||
"bitpacker:@suaveplan/bitpacker-streaming" as const;
|
||||
export const BITPACKER_STREAMING_SCHEMA_VERSION = 1;
|
||||
|
||||
interface Payload {
|
||||
readonly streamingBitPackerOptions: unknown;
|
||||
readonly streamingPackOptions: unknown;
|
||||
readonly streamingUnpackOptions: unknown;
|
||||
readonly readableStreamFromBytesOptions: unknown;
|
||||
readonly writablePackOptions: unknown;
|
||||
readonly writableUnpackOptions: unknown;
|
||||
readonly framingConfig: unknown;
|
||||
readonly packTransformOptions: unknown;
|
||||
readonly unpackTransformOptions: unknown;
|
||||
}
|
||||
|
||||
function buildPayload(): Payload {
|
||||
const opts = { unrepresentable: "any" } as const;
|
||||
return {
|
||||
streamingBitPackerOptions: z.toJSONSchema(
|
||||
streamingBitPackerOptionsSchema,
|
||||
opts
|
||||
),
|
||||
streamingPackOptions: z.toJSONSchema(streamingPackOptionsSchema, opts),
|
||||
streamingUnpackOptions: z.toJSONSchema(
|
||||
streamingUnpackOptionsSchema,
|
||||
opts
|
||||
),
|
||||
readableStreamFromBytesOptions: z.toJSONSchema(
|
||||
readableStreamFromBytesOptionsSchema,
|
||||
opts
|
||||
),
|
||||
writablePackOptions: z.toJSONSchema(writablePackOptionsSchema, opts),
|
||||
writableUnpackOptions: z.toJSONSchema(
|
||||
writableUnpackOptionsSchema,
|
||||
opts
|
||||
),
|
||||
framingConfig: z.toJSONSchema(framingConfigSchema, opts),
|
||||
packTransformOptions: z.toJSONSchema(packTransformOptionsSchema, opts),
|
||||
unpackTransformOptions: z.toJSONSchema(
|
||||
unpackTransformOptionsSchema,
|
||||
opts
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotently register `@suaveplan/bitpacker-streaming`'s public Zod
|
||||
* schemas with the given `SchemaRegistry`.
|
||||
*/
|
||||
export function registerBitPackerStreamingSchemas(
|
||||
registry: SchemaRegistry
|
||||
): void {
|
||||
if (registry.listSubjects().includes(BITPACKER_STREAMING_SUBJECT)) {
|
||||
return;
|
||||
}
|
||||
registry.register(
|
||||
BITPACKER_STREAMING_SUBJECT,
|
||||
buildPayload(),
|
||||
"json-schema",
|
||||
{
|
||||
package: "@suaveplan/bitpacker-streaming",
|
||||
namespace: "bitpacker",
|
||||
}
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
# StreamingBitPacker
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs --> - streaming-bit-packer.schema — Zod schemas for the o…
|
||||
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
|
||||
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
|
||||
- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
|
||||
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
|
||||
- streaming-bit-packer.schema — Zod schemas for the o…
|
||||
|
||||
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
- [streaming-bit-packer.schema](./streaming-bit-packer.schema.md) — Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpa…
|
||||
|
||||
## Purpose
|
||||
|
||||
`StreamingBitPacker<T>` is the class-shaped façade that lets callers process pack/unpack operations as chunked pipelines instead of single-shot calls. It owns an underlying `BitPacker<T>` and surfaces five entry points that mirror the upstream `bitpacker/src/streaming/streaming.ts` API: `packStream`, `unpackStream`, `createReadableStream`, `createPackingStream`, and `createUnpackingStream`. Wire-format compatibility is preserved — bytes produced by `packStream` round-trip through `unpackStream` exactly as bytes produced by `BitPacker.pack` round-trip through `BitPacker.unpack`.
|
||||
|
||||
The streaming surface is for callers that need progress callbacks, per-chunk processing (compression, encryption), or backpressure semantics — anything where the single-shot `pack/unpack` is too coarse. For everything else, prefer the simpler chunk helpers (`packToChunks` / `unpackFromChunks`) that already live in this package's `index.ts`.
|
||||
|
||||
## Features
|
||||
|
||||
- Five entry points covering the chunked-pipeline matrix: synchronous-with-known-input (`packStream`/`unpackStream`), readable-side (`createReadableStream`), and writable-side with in-flight pack/unpack (`createPackingStream`/`createUnpackingStream`).
|
||||
- Optional `chunkProcessor` runs between bit-packing and emission, so the same façade composes cleanly with downstream compression/encryption stages without callers needing to wire the WHATWG streams API by hand.
|
||||
- Progress reported via the canonical `TransformProgress` shape from `@suaveplan/types` — observability tools speak one vocabulary across every Suaveplan pipeline.
|
||||
- Injectable `Clock` for throughput timing; tests run with a `FakeClock` to avoid real-timer flakiness.
|
||||
- `BitpackerError` codes (`STREAM_ABORTED`, `STREAM_CHUNK_PROCESSOR_FAILED`, `STREAM_INCOMPLETE_DATA`) for every failure mode, replacing upstream's plain `Error` throws.
|
||||
- Default packing strategies registered automatically (opt-out via `registerDefaults: false`).
|
||||
- AbortSignal supported on all `*Stream` methods — caller can cancel mid-pipeline.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { StreamingBitPacker } from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
const packer = new StreamingBitPacker({ x: 255, y: 255 });
|
||||
const packed = await packer.packStream({ x: 42, y: 99 });
|
||||
const decoded = await packer.unpackStream<{ x: number; y: number }>(packed);
|
||||
// → { x: 42, y: 99 }
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { StreamingBitPacker } from "@suaveplan/bitpacker-streaming";
|
||||
import type { TransformProgress } from "@suaveplan/types";
|
||||
|
||||
const packer = new StreamingBitPacker({
|
||||
position: { x: "f32", y: "f32", z: "f32" },
|
||||
velocity: { x: "f32", y: "f32", z: "f32" },
|
||||
});
|
||||
|
||||
const progressLog: TransformProgress[] = [];
|
||||
const onProgress = (p: TransformProgress): void => {
|
||||
progressLog.push(p);
|
||||
};
|
||||
|
||||
// chunkProcessor runs between bit-pack and emission — compose with any
|
||||
// downstream codec without touching the WHATWG streams API directly.
|
||||
const xor = (chunk: Uint8Array): Uint8Array => {
|
||||
const out = new Uint8Array(chunk.length);
|
||||
for (let i = 0; i < chunk.length; i++) out[i] = (chunk[i] as number) ^ 0xaa;
|
||||
return out;
|
||||
};
|
||||
|
||||
const packed = await packer.packStream(
|
||||
{
|
||||
position: { x: 1, y: 2, z: 3 },
|
||||
velocity: { x: 0.1, y: 0, z: 0 },
|
||||
},
|
||||
{
|
||||
chunkSize: 4,
|
||||
chunkProcessor: xor,
|
||||
onProgress,
|
||||
}
|
||||
);
|
||||
|
||||
// Reverse path with the same processor undoes the XOR before unpack.
|
||||
const decoded = await packer.unpackStream(packed, {
|
||||
chunkSize: 4,
|
||||
chunkProcessor: xor,
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `new StreamingBitPacker<T>(schema, options?)`
|
||||
|
||||
`schema` — a `BitPackerSchema` describing the data shape.
|
||||
`options.registerDefaults?` — `boolean`. Default `true`. When `false`, the underlying `BitPacker` ships with no strategies; the caller must register their own before any pack/unpack call.
|
||||
|
||||
### `packStream(data, options?) → Promise<Uint8Array>`
|
||||
|
||||
Bit-packs `data`, walks the bytes in `chunkSize`-byte chunks (default `1024`), runs each chunk through `options.chunkProcessor` if supplied, and returns the concatenated bytes. `onProgress` fires once per chunk. Throws `BitpackerError(STREAM_ABORTED)` if `options.signal` aborts before or mid-loop.
|
||||
|
||||
### `unpackStream<R>(data, options?) → Promise<R>`
|
||||
|
||||
Reverse of `packStream`. Walks `data` in chunks (running the inverse processor if supplied), concatenates, and bit-unpacks.
|
||||
|
||||
### `createReadableStream(data, options?) → ReadableStream<Uint8Array>`
|
||||
|
||||
Returns a WHATWG `ReadableStream<Uint8Array>` that emits `data` in chunks. Useful for piping into another transform (`pipeThrough(compressionStream)`). Emits all chunks during `start()` and immediately closes — back-pressure is delegated to the consumer.
|
||||
|
||||
### `createPackingStream(options?) → { stream, result }`
|
||||
|
||||
`stream` is a `WritableStream<unknown>` that packs each `write(item)` and pushes the bytes into a buffer. `result` resolves to the concatenated packed bytes when the stream closes. If `chunkProcessor` is supplied, each packed chunk is run through it before being buffered. `result` rejects with `BitpackerError(STREAM_ABORTED)` on `abort()`.
|
||||
|
||||
### `createUnpackingStream<R>(options?) → { stream, result }`
|
||||
|
||||
`stream` is a `WritableStream<Uint8Array>` that pushes each `write(bytes)` (optionally through `chunkProcessor`) into a buffer. On `close()`, the buffer is concatenated and bit-unpacked. `result` rejects with `BitpackerError(STREAM_INCOMPLETE_DATA)` if the stream closes empty, or `STREAM_ABORTED` on `abort()`.
|
||||
|
||||
### Inherited
|
||||
|
||||
`bitPacker` — public readonly. The underlying `BitPacker<T>` instance. Useful for callers wanting to register a custom strategy or schema.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The port drops upstream's `EventEmitter` inheritance because `BitPacker` does not extend an emitter and the `'progress'` event fires the same payload `onProgress` would have received — the callback is the simpler shape. This also avoids the duplicate-listener footgun upstream callers occasionally hit.
|
||||
|
||||
`validateChunks` from upstream is gone. `BitPacker` does not expose `validate()` or `getValidationErrors()` — those upstream helpers were thin wrappers over schema-introspection logic that suaveplan pushes onto `@suaveplan/validation` callers. If pre-pack validation is needed, run a Zod schema at the call site before invoking `packStream`.
|
||||
|
||||
The `runChunkPipeline` helper is shared between `packStream` and `unpackStream` because the per-chunk loop logic — chunk slice, optional processor, progress emit — is identical in both directions. Diverging the two paths would duplicate ~30 lines of logic for no semantic gain.
|
||||
|
||||
`createPackingStream` and `createUnpackingStream` deliberately do not auto-emit on `start` (unlike `createReadableStream`) — they are _writable_ sides and emission timing is fully driven by caller writes. The `result` promise pattern lets callers `await` the closure without needing a separate completion signal.
|
||||
@@ -0,0 +1,99 @@
|
||||
# Streaming BitPacker Schemas
|
||||
|
||||
## Purpose
|
||||
|
||||
Zod schemas for the option bundles consumed by `StreamingBitPacker` and its peer streaming helpers (`packStream`, `unpackStream`, `readableStreamFromBytes`, the writable-side equivalents). Each schema is the runtime gatekeeper at the corresponding constructor or method boundary so a typo'd flag (`registerDefualts`) or a mis-typed `signal` (a plain object instead of an `AbortSignal`) fails at parse time instead of breaking the chunk pipeline halfway through.
|
||||
|
||||
The schemas use `z.custom<T>((v): v is T => …)` for `AbortSignal`, the chunk-processor callback, the progress callback, and the structural `Clock` so referential identity is preserved end-to-end. This matters for cancellation (`signal === user.signal` comparisons elsewhere in the chain), for callback registration (the same function reference must reach progress reporters), and for clock injection (test fakes need the same instance their assertions reference).
|
||||
|
||||
## Features
|
||||
|
||||
- `streamingBitPackerOptionsSchema` — strict, today only `registerDefaults`
|
||||
- `streamingPackOptionsSchema` — `chunkSize`, `chunkProcessor`, `onProgress`, `clock`, `signal`
|
||||
- `streamingUnpackOptionsSchema` — same shape as pack (re-export)
|
||||
- `readableStreamFromBytesOptionsSchema` — source-side subset (no chunkProcessor, no signal)
|
||||
- `writablePackOptionsSchema` — sink-side subset (no chunkSize, no signal)
|
||||
- `writableUnpackOptionsSchema` — same shape as writable pack (re-export)
|
||||
- Identity-preserving `z.custom` for callables, AbortSignal, and Clock duck-types
|
||||
- `as unknown as z.ZodType<T>` casts for `exactOptionalPropertyTypes` mismatches
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
StreamingBitPacker,
|
||||
streamingBitPackerOptionsSchema,
|
||||
streamingPackOptionsSchema,
|
||||
} from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
const constructorOpts = streamingBitPackerOptionsSchema.parse({
|
||||
registerDefaults: true,
|
||||
});
|
||||
const packer = new StreamingBitPacker({ x: 255 }, constructorOpts);
|
||||
|
||||
const ac = new AbortController();
|
||||
const callOpts = streamingPackOptionsSchema.parse({
|
||||
chunkSize: 1024,
|
||||
signal: ac.signal,
|
||||
});
|
||||
const bytes = await packer.packStream({ x: 100 }, callOpts);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
StreamingBitPacker,
|
||||
streamingPackOptionsSchema,
|
||||
writablePackOptionsSchema,
|
||||
} from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
function safelyPack(packer: StreamingBitPacker, raw: unknown) {
|
||||
const result = streamingPackOptionsSchema.safeParse(raw);
|
||||
if (!result.success) {
|
||||
throw new Error(
|
||||
`Streaming pack options rejected: ${result.error.message}`
|
||||
);
|
||||
}
|
||||
return packer.packStream({ x: 0 }, result.data);
|
||||
}
|
||||
|
||||
// Writable-side callback wired through the progress channel:
|
||||
const writableOpts = writablePackOptionsSchema.parse({
|
||||
chunkProcessor: (chunk) => chunk,
|
||||
onProgress: (p) => console.log(p),
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `streamingBitPackerOptionsSchema: z.ZodType<StreamingBitPackerOptions>`
|
||||
|
||||
Strict schema for the constructor bundle. `registerDefaults` is an optional boolean.
|
||||
|
||||
### `streamingPackOptionsSchema: z.ZodType<StreamingPackOptions>`
|
||||
|
||||
Strict schema for the per-call options. `chunkSize` is a positive integer; `chunkProcessor` and `onProgress` are functions (identity-preserved); `clock` is a structural `{ now(): number }`; `signal` is an `AbortSignal` instance.
|
||||
|
||||
### `streamingUnpackOptionsSchema: z.ZodType<StreamingUnpackOptions>`
|
||||
|
||||
Same shape as `streamingPackOptionsSchema`. `StreamingUnpackOptions` is an interface that extends `StreamingPackOptions` with no additional fields.
|
||||
|
||||
### `readableStreamFromBytesOptionsSchema: z.ZodType<ReadableStreamFromBytesOptions>`
|
||||
|
||||
Strict schema. Accepts `chunkSize`, `onProgress`, and `clock`. `chunkProcessor` is rejected because it is not part of the source-side `ReadableStream` API.
|
||||
|
||||
### `writablePackOptionsSchema: z.ZodType<WritablePackOptions>`
|
||||
|
||||
Strict schema. Accepts `chunkProcessor`, `onProgress`, `clock`. `chunkSize` and `signal` are rejected because chunk boundaries and cancellation are controlled upstream by the writable consumer.
|
||||
|
||||
### `writableUnpackOptionsSchema: z.ZodType<WritableUnpackOptions>`
|
||||
|
||||
Same shape as `writablePackOptionsSchema`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- **Identity for callables.** All callback fields are validated with `z.custom<Fn>((v): v is Fn => typeof v === "function", …)` so the same instance flows through. `z.function()` in Zod 4 wraps the callable and breaks identity for the chunk pipeline, which depends on `===` comparisons during sequential chunk dispatch.
|
||||
- **`AbortSignal` instance check.** `z.custom<AbortSignal>((v): v is AbortSignal => v instanceof AbortSignal, …)` rejects duck-typed signals. The cancellation path checks `signal.aborted` via the prototype, so a non-`AbortSignal` would silently fail to cancel.
|
||||
- **`Clock` is duck-typed.** `clock` is `z.custom<{ now(): number }>`. Tests inject a `FakeClock` from `@suaveplan/testing` that satisfies the shape without inheriting from a class. Reflecting that flexibility in the schema avoids over-coupling.
|
||||
- **Subset schemas reject extra fields.** `readableStreamFromBytesOptionsSchema` rejects `chunkProcessor`; `writablePackOptionsSchema` rejects `chunkSize`/`signal`. This isn't decorative — it surfaces miscompositions where a caller copies a `StreamingPackOptions` bundle into a writable-side method.
|
||||
@@ -0,0 +1,107 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
readableStreamFromBytesOptionsSchema,
|
||||
streamingBitPackerOptionsSchema,
|
||||
streamingPackOptionsSchema,
|
||||
streamingUnpackOptionsSchema,
|
||||
writablePackOptionsSchema,
|
||||
writableUnpackOptionsSchema,
|
||||
} from "./streaming-bit-packer.schema.js";
|
||||
|
||||
describe("streamingBitPackerOptionsSchema", () => {
|
||||
it("accepts an empty bundle", () => {
|
||||
expect(streamingBitPackerOptionsSchema.parse({})).toEqual({});
|
||||
});
|
||||
|
||||
it("accepts registerDefaults", () => {
|
||||
expect(
|
||||
streamingBitPackerOptionsSchema.parse({ registerDefaults: false })
|
||||
).toEqual({ registerDefaults: false });
|
||||
});
|
||||
|
||||
it("rejects unknown keys", () => {
|
||||
expect(() =>
|
||||
streamingBitPackerOptionsSchema.parse({ unknownKey: true })
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("streamingPackOptionsSchema", () => {
|
||||
it("accepts an AbortSignal", () => {
|
||||
const ac = new AbortController();
|
||||
const opts = { signal: ac.signal, chunkSize: 64 };
|
||||
const parsed = streamingPackOptionsSchema.parse(opts);
|
||||
expect(parsed.signal).toBe(ac.signal);
|
||||
expect(parsed.chunkSize).toBe(64);
|
||||
});
|
||||
|
||||
it("preserves callback identity", () => {
|
||||
const onProgress = () => undefined;
|
||||
const parsed = streamingPackOptionsSchema.parse({ onProgress });
|
||||
expect(parsed.onProgress).toBe(onProgress);
|
||||
});
|
||||
|
||||
it("rejects a non-AbortSignal signal", () => {
|
||||
expect(() =>
|
||||
streamingPackOptionsSchema.parse({ signal: { aborted: false } })
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects zero chunkSize", () => {
|
||||
expect(() =>
|
||||
streamingPackOptionsSchema.parse({ chunkSize: 0 })
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("streamingUnpackOptionsSchema", () => {
|
||||
it("accepts the same shape as pack", () => {
|
||||
const ac = new AbortController();
|
||||
expect(
|
||||
streamingUnpackOptionsSchema.parse({
|
||||
signal: ac.signal,
|
||||
chunkSize: 32,
|
||||
})
|
||||
).toEqual({ signal: ac.signal, chunkSize: 32 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("readableStreamFromBytesOptionsSchema", () => {
|
||||
it("accepts the documented fields", () => {
|
||||
const onProgress = () => undefined;
|
||||
const parsed = readableStreamFromBytesOptionsSchema.parse({
|
||||
chunkSize: 256,
|
||||
onProgress,
|
||||
});
|
||||
expect(parsed.chunkSize).toBe(256);
|
||||
expect(parsed.onProgress).toBe(onProgress);
|
||||
});
|
||||
|
||||
it("rejects a chunkProcessor (not part of the source-side API)", () => {
|
||||
expect(() =>
|
||||
readableStreamFromBytesOptionsSchema.parse({
|
||||
chunkProcessor: () => new Uint8Array(),
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("writablePackOptionsSchema", () => {
|
||||
it("accepts the documented fields", () => {
|
||||
const chunkProcessor = (b: Uint8Array) => b;
|
||||
const parsed = writablePackOptionsSchema.parse({ chunkProcessor });
|
||||
expect(parsed.chunkProcessor).toBe(chunkProcessor);
|
||||
});
|
||||
|
||||
it("rejects chunkSize (controlled upstream)", () => {
|
||||
expect(() =>
|
||||
writablePackOptionsSchema.parse({ chunkSize: 64 })
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("writableUnpackOptionsSchema", () => {
|
||||
it("accepts the same shape as writable pack", () => {
|
||||
expect(writableUnpackOptionsSchema.parse({})).toEqual({});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import z from "zod";
|
||||
import type {
|
||||
ReadableStreamFromBytesOptions,
|
||||
StreamingBitPackerOptions,
|
||||
StreamingPackOptions,
|
||||
StreamingUnpackOptions,
|
||||
WritablePackOptions,
|
||||
WritableUnpackOptions,
|
||||
} from "./streaming-bit-packer.js";
|
||||
|
||||
const callable = <T>(message: string) =>
|
||||
z.custom<T>((v): v is T => typeof v === "function", { message });
|
||||
|
||||
const clockSchema = z.custom<{ now(): number }>(
|
||||
(v): v is { now(): number } => {
|
||||
if (v === null || typeof v !== "object") return false;
|
||||
return typeof (v as { now: unknown }).now === "function";
|
||||
},
|
||||
{ message: "clock must expose now()" }
|
||||
);
|
||||
|
||||
const abortSignalSchema = z.custom<AbortSignal>(
|
||||
(v): v is AbortSignal => v instanceof AbortSignal,
|
||||
{ message: "signal must be an AbortSignal" }
|
||||
);
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link StreamingBitPackerOptions}.
|
||||
*
|
||||
* Validates the constructor options bundle for `StreamingBitPacker`. Today
|
||||
* the only field is `registerDefaults`, but using a strict object schema
|
||||
* means future fields can be added without consumers' typos slipping
|
||||
* through.
|
||||
*/
|
||||
export const streamingBitPackerOptionsSchema = z
|
||||
.object({
|
||||
registerDefaults: z.boolean().optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<StreamingBitPackerOptions>;
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link StreamingPackOptions}.
|
||||
*
|
||||
* Validates the per-call options for `packStream` / `packToStream`.
|
||||
* `chunkProcessor` and `onProgress` are validated as functions via
|
||||
* `z.custom` so identity is preserved through the pipeline. `clock` and
|
||||
* `signal` are validated structurally so consumers can pass either a real
|
||||
* `Clock` from `@suaveplan/types` or any compatible duck-type.
|
||||
*/
|
||||
export const streamingPackOptionsSchema = z
|
||||
.object({
|
||||
chunkSize: z.number().int().positive().optional(),
|
||||
chunkProcessor: callable<
|
||||
NonNullable<StreamingPackOptions["chunkProcessor"]>
|
||||
>("chunkProcessor must be a function").optional(),
|
||||
onProgress: callable<NonNullable<StreamingPackOptions["onProgress"]>>(
|
||||
"onProgress must be a function"
|
||||
).optional(),
|
||||
clock: clockSchema.optional(),
|
||||
signal: abortSignalSchema.optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<StreamingPackOptions>;
|
||||
|
||||
/** Strict Zod schema for {@link StreamingUnpackOptions} — same shape as pack. */
|
||||
export const streamingUnpackOptionsSchema =
|
||||
streamingPackOptionsSchema as unknown as z.ZodType<StreamingUnpackOptions>;
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link ReadableStreamFromBytesOptions}.
|
||||
*
|
||||
* Validates the options bundle for `readableStreamFromBytes`. Subset of
|
||||
* `StreamingPackOptions` covering only the fields meaningful for a
|
||||
* source-side ReadableStream.
|
||||
*/
|
||||
export const readableStreamFromBytesOptionsSchema = z
|
||||
.object({
|
||||
chunkSize: z.number().int().positive().optional(),
|
||||
onProgress: callable<
|
||||
NonNullable<ReadableStreamFromBytesOptions["onProgress"]>
|
||||
>("onProgress must be a function").optional(),
|
||||
clock: clockSchema.optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<ReadableStreamFromBytesOptions>;
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link WritablePackOptions}.
|
||||
*
|
||||
* Validates the options bundle for `writablePack` / `writableUnpack`. Same
|
||||
* fields as `StreamingPackOptions` minus `chunkSize` and `signal` — the
|
||||
* Writable side already controls chunk boundaries upstream.
|
||||
*/
|
||||
export const writablePackOptionsSchema = z
|
||||
.object({
|
||||
chunkProcessor: callable<
|
||||
NonNullable<WritablePackOptions["chunkProcessor"]>
|
||||
>("chunkProcessor must be a function").optional(),
|
||||
onProgress: callable<NonNullable<WritablePackOptions["onProgress"]>>(
|
||||
"onProgress must be a function"
|
||||
).optional(),
|
||||
clock: clockSchema.optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<WritablePackOptions>;
|
||||
|
||||
/** Strict Zod schema for {@link WritableUnpackOptions} — same shape as writable pack. */
|
||||
export const writableUnpackOptionsSchema =
|
||||
writablePackOptionsSchema as unknown as z.ZodType<WritableUnpackOptions>;
|
||||
@@ -0,0 +1,507 @@
|
||||
import { BITPACKER_CODES, type BitpackerError } from "@suaveplan/bitpacker";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { Clock, TransformProgress } from "@suaveplan/types";
|
||||
import { StreamingBitPacker } from "./streaming-bit-packer.js";
|
||||
|
||||
const SCHEMA = { x: 255, y: 255 } as const;
|
||||
type Pair = { x: number; y: number };
|
||||
|
||||
function makeFakeClock(stepMs = 50): Clock {
|
||||
let t = 0;
|
||||
return () => {
|
||||
t += stepMs;
|
||||
return t;
|
||||
};
|
||||
}
|
||||
|
||||
describe("StreamingBitPacker — packStream / unpackStream", () => {
|
||||
it("round-trips simple data", { timeout: 5000 }, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const original: Pair = { x: 42, y: 99 };
|
||||
const bytes = await packer.packStream(original);
|
||||
const decoded = await packer.unpackStream<Pair>(bytes);
|
||||
expect(decoded).toEqual(original);
|
||||
});
|
||||
|
||||
it("emits TransformProgress events for every chunk", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const progress: TransformProgress[] = [];
|
||||
await packer.packStream({ x: 1, y: 2 } as Pair, {
|
||||
chunkSize: 1,
|
||||
onProgress: (p) => {
|
||||
progress.push(p);
|
||||
},
|
||||
clock: makeFakeClock(),
|
||||
});
|
||||
expect(progress.length).toBeGreaterThanOrEqual(1);
|
||||
// Last event drained the buffer.
|
||||
const last = progress[progress.length - 1] as TransformProgress;
|
||||
expect(last.itemsPending).toBe(0);
|
||||
expect(last.operation).toBe("pack");
|
||||
});
|
||||
|
||||
it("runs chunkProcessor on every chunk", { timeout: 5000 }, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const xor = (chunk: Uint8Array): Uint8Array => {
|
||||
const out = new Uint8Array(chunk.length);
|
||||
for (let i = 0; i < chunk.length; i++) {
|
||||
out[i] = (chunk[i] as number) ^ 0xff;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const original: Pair = { x: 7, y: 8 };
|
||||
const wire = await packer.packStream(original, {
|
||||
chunkSize: 1,
|
||||
chunkProcessor: xor,
|
||||
});
|
||||
// Reverse path with same processor (XOR is its own inverse).
|
||||
const decoded = await packer.unpackStream<Pair>(wire, {
|
||||
chunkSize: 1,
|
||||
chunkProcessor: xor,
|
||||
});
|
||||
expect(decoded).toEqual(original);
|
||||
});
|
||||
|
||||
it("packStream throws STREAM_ABORTED if signal is already aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
try {
|
||||
await packer.packStream({ x: 1, y: 2 } as Pair, {
|
||||
signal: ac.signal,
|
||||
});
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_ABORTED
|
||||
);
|
||||
expect((e as BitpackerError).context?.operation).toBe("pack");
|
||||
}
|
||||
});
|
||||
|
||||
it("packStream throws STREAM_ABORTED if signal aborts mid-pipeline", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const ac = new AbortController();
|
||||
// chunkProcessor aborts after first chunk.
|
||||
const trip: typeof packer extends StreamingBitPacker<infer _>
|
||||
? (chunk: Uint8Array) => Uint8Array
|
||||
: never = (chunk) => {
|
||||
ac.abort();
|
||||
return chunk;
|
||||
};
|
||||
try {
|
||||
await packer.packStream({ x: 1, y: 2 } as Pair, {
|
||||
chunkSize: 1,
|
||||
chunkProcessor: trip,
|
||||
signal: ac.signal,
|
||||
});
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps chunkProcessor errors as STREAM_CHUNK_PROCESSOR_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const angry = (): Uint8Array => {
|
||||
throw new Error("processor blew up");
|
||||
};
|
||||
try {
|
||||
await packer.packStream({ x: 1, y: 2 } as Pair, {
|
||||
chunkProcessor: angry,
|
||||
});
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_CHUNK_PROCESSOR_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingBitPacker — createReadableStream", () => {
|
||||
it("emits all chunks and closes", { timeout: 5000 }, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const buf = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
|
||||
const stream = packer.createReadableStream(buf, { chunkSize: 3 });
|
||||
const received: number[] = [];
|
||||
const reader = stream.getReader();
|
||||
for (
|
||||
let res = await reader.read();
|
||||
!res.done;
|
||||
res = await reader.read()
|
||||
) {
|
||||
for (const b of res.value) received.push(b);
|
||||
}
|
||||
expect(new Uint8Array(received)).toEqual(buf);
|
||||
});
|
||||
|
||||
it("uses default chunkSize and defaultClock when options omitted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// No options at all → exercises every default-value branch of the
|
||||
// parameter destructure (chunkSize, onProgress, clock).
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const buf = new Uint8Array(2048).fill(7);
|
||||
const stream = packer.createReadableStream(buf);
|
||||
const reader = stream.getReader();
|
||||
let total = 0;
|
||||
for (
|
||||
let res = await reader.read();
|
||||
!res.done;
|
||||
res = await reader.read()
|
||||
) {
|
||||
total += res.value.length;
|
||||
}
|
||||
expect(total).toBe(2048);
|
||||
});
|
||||
|
||||
it("emits progress per chunk through onProgress", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const buf = new Uint8Array([1, 2, 3, 4, 5, 6]);
|
||||
const progress: TransformProgress[] = [];
|
||||
const stream = packer.createReadableStream(buf, {
|
||||
chunkSize: 2,
|
||||
onProgress: (p) => {
|
||||
progress.push(p);
|
||||
},
|
||||
clock: makeFakeClock(),
|
||||
});
|
||||
const reader = stream.getReader();
|
||||
for (
|
||||
let res = await reader.read();
|
||||
!res.done;
|
||||
res = await reader.read()
|
||||
) {
|
||||
// drain
|
||||
}
|
||||
expect(progress).toHaveLength(3);
|
||||
expect(progress[2]?.itemsPending).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingBitPacker — createPackingStream", () => {
|
||||
it("packs each written item and resolves on close", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const { stream, result } = packer.createPackingStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.write({ x: 1, y: 2 } as Pair);
|
||||
await writer.write({ x: 3, y: 4 } as Pair);
|
||||
await writer.close();
|
||||
const out = await result;
|
||||
expect(out).toBeInstanceOf(Uint8Array);
|
||||
expect(out.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects result with STREAM_ABORTED on abort()", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const { stream, result } = packer.createPackingStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.abort("test-abort");
|
||||
try {
|
||||
await result;
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_ABORTED
|
||||
);
|
||||
expect((e as BitpackerError).context?.reason).toBe("test-abort");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects result if BitPacker.pack throws on a written item", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// Build a packer with NO default strategies → every pack throws.
|
||||
const packer = new StreamingBitPacker(SCHEMA, {
|
||||
registerDefaults: false,
|
||||
});
|
||||
const { stream, result } = packer.createPackingStream();
|
||||
const writer = stream.getWriter();
|
||||
// The write will fail; capture & swallow the error from getWriter
|
||||
// so we can assert through the result promise.
|
||||
try {
|
||||
await writer.write({ x: 1, y: 2 } as Pair);
|
||||
} catch {
|
||||
/* expected */
|
||||
}
|
||||
try {
|
||||
await result;
|
||||
} catch (e) {
|
||||
expect(e).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("emits progress with itemsPending=0 on each write", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const progress: TransformProgress[] = [];
|
||||
const { stream, result } = packer.createPackingStream({
|
||||
onProgress: (p) => {
|
||||
progress.push(p);
|
||||
},
|
||||
clock: makeFakeClock(),
|
||||
});
|
||||
const writer = stream.getWriter();
|
||||
await writer.write({ x: 1, y: 2 } as Pair);
|
||||
await writer.close();
|
||||
await result;
|
||||
expect(progress).toHaveLength(1);
|
||||
expect(progress[0]?.itemsPending).toBe(0);
|
||||
expect(progress[0]?.operation).toBe("pack");
|
||||
});
|
||||
|
||||
it("runs chunkProcessor on each packed chunk", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const observed: Uint8Array[] = [];
|
||||
const { stream, result } = packer.createPackingStream({
|
||||
chunkProcessor: (c) => {
|
||||
observed.push(c);
|
||||
return c;
|
||||
},
|
||||
});
|
||||
const writer = stream.getWriter();
|
||||
await writer.write({ x: 5, y: 6 } as Pair);
|
||||
await writer.close();
|
||||
await result;
|
||||
expect(observed.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingBitPacker — createUnpackingStream", () => {
|
||||
it("buffers writes and unpacks on close", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const original: Pair = { x: 200, y: 100 };
|
||||
const packed = await packer.packStream(original);
|
||||
const { stream, result } = packer.createUnpackingStream<Pair>();
|
||||
const writer = stream.getWriter();
|
||||
// Split the packed bytes across two writes.
|
||||
const half = Math.floor(packed.length / 2);
|
||||
await writer.write(packed.slice(0, half));
|
||||
await writer.write(packed.slice(half));
|
||||
await writer.close();
|
||||
const out = await result;
|
||||
expect(out).toEqual(original);
|
||||
});
|
||||
|
||||
it("rejects with STREAM_INCOMPLETE_DATA when closed empty", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const { stream, result } = packer.createUnpackingStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.close();
|
||||
try {
|
||||
await result;
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_INCOMPLETE_DATA
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects with STREAM_ABORTED on abort()", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const { stream, result } = packer.createUnpackingStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.abort("user-cancel");
|
||||
try {
|
||||
await result;
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_ABORTED
|
||||
);
|
||||
expect((e as BitpackerError).context?.reason).toBe("user-cancel");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects if BitPacker.unpack throws on close", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA, {
|
||||
registerDefaults: false,
|
||||
});
|
||||
const { stream, result } = packer.createUnpackingStream();
|
||||
const writer = stream.getWriter();
|
||||
// Push some bytes so close attempts an unpack — and unpack fails
|
||||
// because we skipped strategy registration.
|
||||
await writer.write(new Uint8Array([1, 2, 3]));
|
||||
await writer.close();
|
||||
try {
|
||||
await result;
|
||||
} catch (e) {
|
||||
expect(e).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("runs chunkProcessor on each incoming chunk", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const original: Pair = { x: 1, y: 2 };
|
||||
const packed = await packer.packStream(original);
|
||||
const observed: number[] = [];
|
||||
const { stream, result } = packer.createUnpackingStream<Pair>({
|
||||
chunkProcessor: (c) => {
|
||||
observed.push(c.length);
|
||||
return c;
|
||||
},
|
||||
});
|
||||
const writer = stream.getWriter();
|
||||
await writer.write(packed);
|
||||
await writer.close();
|
||||
const out = await result;
|
||||
expect(out).toEqual(original);
|
||||
expect(observed.length).toBe(1);
|
||||
});
|
||||
|
||||
it("emits progress with operation='unpack' on each write", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const original: Pair = { x: 9, y: 10 };
|
||||
const packed = await packer.packStream(original);
|
||||
const progress: TransformProgress[] = [];
|
||||
const { stream, result } = packer.createUnpackingStream<Pair>({
|
||||
onProgress: (p) => {
|
||||
progress.push(p);
|
||||
},
|
||||
clock: makeFakeClock(),
|
||||
});
|
||||
const writer = stream.getWriter();
|
||||
await writer.write(packed);
|
||||
await writer.close();
|
||||
await result;
|
||||
expect(progress.length).toBe(1);
|
||||
expect(progress[0]?.operation).toBe("unpack");
|
||||
});
|
||||
|
||||
it("wraps chunkProcessor errors as STREAM_CHUNK_PROCESSOR_FAILED on the writable side", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
const original: Pair = { x: 3, y: 4 };
|
||||
const packed = await packer.packStream(original);
|
||||
const { stream, result } = packer.createUnpackingStream({
|
||||
chunkProcessor: () => {
|
||||
throw new Error("processor blew up");
|
||||
},
|
||||
});
|
||||
const writer = stream.getWriter();
|
||||
try {
|
||||
await writer.write(packed);
|
||||
} catch {
|
||||
/* expected */
|
||||
}
|
||||
try {
|
||||
await result;
|
||||
} catch (e) {
|
||||
expect((e as BitpackerError).code).toBe(
|
||||
BITPACKER_CODES.STREAM_CHUNK_PROCESSOR_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingBitPacker — throughput zero-elapsed branch", () => {
|
||||
const frozen: Clock = () => 0;
|
||||
|
||||
it("packStream emits throughput=0 when clock is frozen", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
let captured: TransformProgress | undefined;
|
||||
await packer.packStream({ x: 1, y: 2 } as Pair, {
|
||||
chunkSize: 1,
|
||||
clock: frozen,
|
||||
onProgress: (p) => {
|
||||
captured = p;
|
||||
},
|
||||
});
|
||||
expect(captured?.throughput).toBe(0);
|
||||
});
|
||||
|
||||
it("createReadableStream emits throughput=0 when clock is frozen", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
let captured: TransformProgress | undefined;
|
||||
const stream = packer.createReadableStream(
|
||||
new Uint8Array([1, 2, 3, 4]),
|
||||
{
|
||||
chunkSize: 2,
|
||||
clock: frozen,
|
||||
onProgress: (p) => {
|
||||
captured = p;
|
||||
},
|
||||
}
|
||||
);
|
||||
const reader = stream.getReader();
|
||||
for (
|
||||
let res = await reader.read();
|
||||
!res.done;
|
||||
res = await reader.read()
|
||||
) {
|
||||
// drain
|
||||
}
|
||||
expect(captured?.throughput).toBe(0);
|
||||
});
|
||||
|
||||
it("createPackingStream emits throughput=0 when clock is frozen", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
let captured: TransformProgress | undefined;
|
||||
const { stream, result } = packer.createPackingStream({
|
||||
clock: frozen,
|
||||
onProgress: (p) => {
|
||||
captured = p;
|
||||
},
|
||||
});
|
||||
const writer = stream.getWriter();
|
||||
await writer.write({ x: 1, y: 2 } as Pair);
|
||||
await writer.close();
|
||||
await result;
|
||||
expect(captured?.throughput).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingBitPacker — constructor and exposed BitPacker", () => {
|
||||
it("exposes the underlying BitPacker via .bitPacker", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA);
|
||||
expect(packer.bitPacker).toBeDefined();
|
||||
// Strategies registered by default.
|
||||
const bytes = packer.bitPacker.pack({ x: 7, y: 8 } as Pair);
|
||||
expect(bytes).toBeInstanceOf(Uint8Array);
|
||||
});
|
||||
|
||||
it("opt-out via registerDefaults: false leaves no strategies", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const packer = new StreamingBitPacker(SCHEMA, {
|
||||
registerDefaults: false,
|
||||
});
|
||||
expect(() => packer.bitPacker.pack({ x: 7, y: 8 } as Pair)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,413 @@
|
||||
/**
|
||||
* `StreamingBitPacker<T>` — class-shaped façade over a chunked
|
||||
* pack/unpack pipeline.
|
||||
*
|
||||
* Wraps an underlying `BitPacker<T>` and exposes the five upstream
|
||||
* streaming entry points:
|
||||
*
|
||||
* 1. `packStream(data, options)` — bit-pack, walk the bytes in chunks,
|
||||
* optionally run each through `chunkProcessor`, concatenate, return.
|
||||
* 2. `unpackStream(data, options)` — chunk-walk the bytes, run them
|
||||
* through `chunkProcessor`, concatenate, bit-unpack, return.
|
||||
* 3. `createReadableStream(data, options)` — return a WHATWG
|
||||
* `ReadableStream<Uint8Array>` that emits the buffer in chunks.
|
||||
* 4. `createPackingStream(options)` — return `{ stream, result }` where
|
||||
* `stream` is a `WritableStream<unknown>` accepting one item per
|
||||
* write and `result` resolves to the concatenated packed bytes on
|
||||
* close.
|
||||
* 5. `createUnpackingStream(options)` — return `{ stream, result }` where
|
||||
* `stream` is a `WritableStream<Uint8Array>` accepting raw bytes and
|
||||
* `result` resolves to the unpacked value on close.
|
||||
*
|
||||
* Genesis-side adjustments versus upstream
|
||||
* `bitpacker/src/streaming/streaming.ts`:
|
||||
*
|
||||
* - No `EventEmitter`. Genesis's `BitPacker` does not extend an emitter,
|
||||
* so we surface progress via the `onProgress` callback shape used
|
||||
* throughout `bitpacker-streaming`.
|
||||
* - `BitpackerError` with stable codes (`STREAM_ABORTED`,
|
||||
* `STREAM_CHUNK_PROCESSOR_FAILED`, `STREAM_INCOMPLETE_DATA`) replaces
|
||||
* plain `Error` throws.
|
||||
* - Injectable `Clock` for throughput timing — tests get determinism
|
||||
* without spinning up real timers.
|
||||
* - The upstream `validateChunks` option is dropped: genesis `BitPacker`
|
||||
* does not expose `validate()`/`getValidationErrors()`. Callers wanting
|
||||
* pre-pack validation run their own validator before calling these
|
||||
* methods (Zod, etc.).
|
||||
* - Default strategy registration is opt-out via `registerDefaults: false`
|
||||
* — by default the new `BitPacker` has the full strategy set, matching
|
||||
* upstream's implicit behaviour.
|
||||
* - `TransformProgress` from `@suaveplan/types` replaces the upstream
|
||||
* ad-hoc progress payload, so a single observability consumer can
|
||||
* speak the same vocabulary across all `bitpacker-streaming` entry
|
||||
* points.
|
||||
*/
|
||||
|
||||
import {
|
||||
BITPACKER_CODES,
|
||||
BitPacker,
|
||||
type BitPackerSchema,
|
||||
BitpackerError,
|
||||
} from "@suaveplan/bitpacker";
|
||||
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
|
||||
import type { Clock, TransformProgress } from "@suaveplan/types";
|
||||
import { defaultClock } from "../defaults/default-clock.js";
|
||||
|
||||
export type ChunkProcessor = (
|
||||
chunk: Uint8Array
|
||||
) => Uint8Array | Promise<Uint8Array>;
|
||||
|
||||
export interface StreamingPackOptions {
|
||||
/** Bytes per emitted chunk. Default `1024`. */
|
||||
chunkSize?: number;
|
||||
/** Optional per-chunk processor (e.g. compress, encrypt). */
|
||||
chunkProcessor?: ChunkProcessor;
|
||||
/** Optional progress callback. */
|
||||
onProgress?: (progress: TransformProgress) => void;
|
||||
/** Optional injectable wall clock. Default `defaultClock`. */
|
||||
clock?: Clock;
|
||||
/** Optional abort signal. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface StreamingUnpackOptions extends StreamingPackOptions {}
|
||||
|
||||
export interface ReadableStreamFromBytesOptions {
|
||||
/** Bytes per emitted chunk. Default `1024`. */
|
||||
chunkSize?: number;
|
||||
/** Optional progress callback. */
|
||||
onProgress?: (progress: TransformProgress) => void;
|
||||
/** Optional injectable wall clock. Default `defaultClock`. */
|
||||
clock?: Clock;
|
||||
}
|
||||
|
||||
export interface WritablePackOptions {
|
||||
/** Optional per-chunk processor (e.g. compress, encrypt). */
|
||||
chunkProcessor?: ChunkProcessor;
|
||||
/** Optional progress callback. */
|
||||
onProgress?: (progress: TransformProgress) => void;
|
||||
/** Optional injectable wall clock. Default `defaultClock`. */
|
||||
clock?: Clock;
|
||||
}
|
||||
|
||||
export interface WritableUnpackOptions extends WritablePackOptions {}
|
||||
|
||||
export interface StreamingBitPackerOptions {
|
||||
/**
|
||||
* Whether to register the default packing strategies on the new
|
||||
* underlying `BitPacker`. Default `true`. Set `false` if the caller
|
||||
* is registering a custom strategy set themselves.
|
||||
*/
|
||||
readonly registerDefaults?: boolean;
|
||||
}
|
||||
|
||||
export class StreamingBitPacker<T extends BitPackerSchema = BitPackerSchema> {
|
||||
public readonly bitPacker: BitPacker<T>;
|
||||
|
||||
public constructor(schema: T, options: StreamingBitPackerOptions = {}) {
|
||||
this.bitPacker = new BitPacker<T>(schema);
|
||||
if (options.registerDefaults !== false) {
|
||||
registerDefaultStrategies(this.bitPacker.strategyManager);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pack `data` to bytes, walk the bytes in `chunkSize`-byte chunks,
|
||||
* optionally run each chunk through `chunkProcessor`, and return the
|
||||
* concatenated result.
|
||||
*/
|
||||
public async packStream(
|
||||
data: unknown,
|
||||
options: StreamingPackOptions = {}
|
||||
): Promise<Uint8Array> {
|
||||
const packed = this.bitPacker.pack(data);
|
||||
return runChunkPipeline(packed, "pack", options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse of `packStream`. Walks `data` in chunks (running the
|
||||
* processor in reverse if supplied), concatenates, and bit-unpacks
|
||||
* the result.
|
||||
*/
|
||||
public async unpackStream<R = unknown>(
|
||||
data: Uint8Array,
|
||||
options: StreamingUnpackOptions = {}
|
||||
): Promise<R> {
|
||||
const combined = await runChunkPipeline(data, "unpack", options);
|
||||
return this.bitPacker.unpack<R>(combined);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a WHATWG `ReadableStream<Uint8Array>` that emits `data` in
|
||||
* `chunkSize`-byte chunks. Useful for piping packed bytes into
|
||||
* another transform (e.g. `pipeThrough(compressionStream)`).
|
||||
*/
|
||||
public createReadableStream(
|
||||
data: Uint8Array,
|
||||
options: ReadableStreamFromBytesOptions = {}
|
||||
): ReadableStream<Uint8Array> {
|
||||
const { chunkSize = 1024, onProgress, clock = defaultClock } = options;
|
||||
const startTime = clock();
|
||||
const chunksTotal = Math.ceil(data.length / chunkSize);
|
||||
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
let bytesProcessed = 0;
|
||||
let itemsProcessed = 0;
|
||||
for (let i = 0; i < chunksTotal; i++) {
|
||||
const start = i * chunkSize;
|
||||
const end = Math.min(start + chunkSize, data.length);
|
||||
const chunk = data.slice(start, end);
|
||||
controller.enqueue(chunk);
|
||||
bytesProcessed += chunk.length;
|
||||
itemsProcessed++;
|
||||
if (onProgress) {
|
||||
const elapsed = (clock() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed,
|
||||
bytesProcessed,
|
||||
itemsPending: chunksTotal - itemsProcessed,
|
||||
throughput:
|
||||
elapsed > 0 ? itemsProcessed / elapsed : 0,
|
||||
operation: "pack",
|
||||
});
|
||||
}
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a `{ stream, result }` pair where each `write(item)` packs
|
||||
* the item and pushes the bytes into a buffer. On `close()`, `result`
|
||||
* resolves to the concatenated packed bytes.
|
||||
*/
|
||||
public createPackingStream(options: WritablePackOptions = {}): {
|
||||
stream: WritableStream<unknown>;
|
||||
result: Promise<Uint8Array>;
|
||||
} {
|
||||
const { chunkProcessor, onProgress, clock = defaultClock } = options;
|
||||
const startTime = clock();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let resolveResult!: (value: Uint8Array) => void;
|
||||
let rejectResult!: (error: Error) => void;
|
||||
const result = new Promise<Uint8Array>((resolve, reject) => {
|
||||
resolveResult = resolve;
|
||||
rejectResult = reject;
|
||||
});
|
||||
|
||||
const packer = this.bitPacker;
|
||||
const stream = new WritableStream<unknown>({
|
||||
async write(chunk) {
|
||||
try {
|
||||
const packed = packer.pack(chunk);
|
||||
const processed = chunkProcessor
|
||||
? await runProcessor(chunkProcessor, packed)
|
||||
: packed;
|
||||
chunks.push(processed);
|
||||
if (onProgress) {
|
||||
emitWriteProgress(
|
||||
chunks,
|
||||
"pack",
|
||||
startTime,
|
||||
clock,
|
||||
onProgress
|
||||
);
|
||||
}
|
||||
} catch (cause) {
|
||||
rejectResult(cause as Error);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
close() {
|
||||
resolveResult(concatChunks(chunks));
|
||||
},
|
||||
abort(reason) {
|
||||
rejectResult(
|
||||
new BitpackerError(
|
||||
BITPACKER_CODES.STREAM_ABORTED,
|
||||
"Packing stream aborted",
|
||||
{ context: { reason: String(reason) } }
|
||||
)
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
return { stream, result };
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a `{ stream, result }` pair where each `write(bytes)` pushes
|
||||
* the bytes (optionally through `chunkProcessor`) into a buffer. On
|
||||
* `close()`, the buffer is concatenated and bit-unpacked.
|
||||
*/
|
||||
public createUnpackingStream<R = unknown>(
|
||||
options: WritableUnpackOptions = {}
|
||||
): { stream: WritableStream<Uint8Array>; result: Promise<R> } {
|
||||
const { chunkProcessor, onProgress, clock = defaultClock } = options;
|
||||
const startTime = clock();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let resolveResult!: (value: R) => void;
|
||||
let rejectResult!: (error: Error) => void;
|
||||
const result = new Promise<R>((resolve, reject) => {
|
||||
resolveResult = resolve;
|
||||
rejectResult = reject;
|
||||
});
|
||||
|
||||
const packer = this.bitPacker;
|
||||
const stream = new WritableStream<Uint8Array>({
|
||||
async write(chunk) {
|
||||
try {
|
||||
const processed = chunkProcessor
|
||||
? await runProcessor(chunkProcessor, chunk)
|
||||
: chunk;
|
||||
chunks.push(processed);
|
||||
if (onProgress) {
|
||||
emitWriteProgress(
|
||||
chunks,
|
||||
"unpack",
|
||||
startTime,
|
||||
clock,
|
||||
onProgress
|
||||
);
|
||||
}
|
||||
} catch (cause) {
|
||||
rejectResult(cause as Error);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
close() {
|
||||
try {
|
||||
const combined = concatChunks(chunks);
|
||||
if (combined.length === 0) {
|
||||
rejectResult(
|
||||
new BitpackerError(
|
||||
BITPACKER_CODES.STREAM_INCOMPLETE_DATA,
|
||||
"Unpacking stream closed with no bytes",
|
||||
{ context: {} }
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
resolveResult(packer.unpack<R>(combined));
|
||||
} catch (cause) {
|
||||
rejectResult(cause as Error);
|
||||
}
|
||||
},
|
||||
abort(reason) {
|
||||
rejectResult(
|
||||
new BitpackerError(
|
||||
BITPACKER_CODES.STREAM_ABORTED,
|
||||
"Unpacking stream aborted",
|
||||
{ context: { reason: String(reason) } }
|
||||
)
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
return { stream, result };
|
||||
}
|
||||
}
|
||||
|
||||
async function runChunkPipeline(
|
||||
data: Uint8Array,
|
||||
operation: "pack" | "unpack",
|
||||
options: StreamingPackOptions
|
||||
): Promise<Uint8Array> {
|
||||
const {
|
||||
chunkSize = 1024,
|
||||
chunkProcessor,
|
||||
onProgress,
|
||||
clock = defaultClock,
|
||||
signal,
|
||||
} = options;
|
||||
|
||||
if (signal?.aborted) {
|
||||
throw new BitpackerError(
|
||||
BITPACKER_CODES.STREAM_ABORTED,
|
||||
"Stream aborted before processing",
|
||||
{ context: { operation } }
|
||||
);
|
||||
}
|
||||
|
||||
const chunksTotal = Math.max(1, Math.ceil(data.length / chunkSize));
|
||||
const startTime = clock();
|
||||
const chunks: Uint8Array[] = [];
|
||||
|
||||
for (let i = 0; i < chunksTotal; i++) {
|
||||
if (signal?.aborted) {
|
||||
throw new BitpackerError(
|
||||
BITPACKER_CODES.STREAM_ABORTED,
|
||||
"Stream aborted mid-processing",
|
||||
{ context: { operation, chunkIndex: i } }
|
||||
);
|
||||
}
|
||||
const start = i * chunkSize;
|
||||
const end = Math.min(start + chunkSize, data.length);
|
||||
const chunk = data.slice(start, end);
|
||||
const processed = chunkProcessor
|
||||
? await runProcessor(chunkProcessor, chunk)
|
||||
: chunk;
|
||||
chunks.push(processed);
|
||||
if (onProgress) {
|
||||
const elapsed = (clock() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed: i + 1,
|
||||
bytesProcessed: end,
|
||||
itemsPending: chunksTotal - (i + 1),
|
||||
throughput: elapsed > 0 ? (i + 1) / elapsed : 0,
|
||||
operation,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return concatChunks(chunks);
|
||||
}
|
||||
|
||||
async function runProcessor(
|
||||
processor: ChunkProcessor,
|
||||
chunk: Uint8Array
|
||||
): Promise<Uint8Array> {
|
||||
try {
|
||||
return await processor(chunk);
|
||||
} catch (cause) {
|
||||
throw new BitpackerError(
|
||||
BITPACKER_CODES.STREAM_CHUNK_PROCESSOR_FAILED,
|
||||
"chunkProcessor threw",
|
||||
{
|
||||
cause: cause as Error,
|
||||
context: { byteLength: chunk.length },
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function emitWriteProgress(
|
||||
chunks: readonly Uint8Array[],
|
||||
operation: "pack" | "unpack",
|
||||
startTime: number,
|
||||
clock: Clock,
|
||||
onProgress: (progress: TransformProgress) => void
|
||||
): void {
|
||||
const bytesProcessed = chunks.reduce((acc, c) => acc + c.length, 0);
|
||||
const elapsed = (clock() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed: chunks.length,
|
||||
bytesProcessed,
|
||||
itemsPending: 0,
|
||||
throughput: elapsed > 0 ? chunks.length / elapsed : 0,
|
||||
operation,
|
||||
});
|
||||
}
|
||||
|
||||
function concatChunks(chunks: readonly Uint8Array[]): Uint8Array {
|
||||
const totalLength = chunks.reduce((acc, c) => acc + c.length, 0);
|
||||
const out = new Uint8Array(totalLength);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, offset);
|
||||
offset += c.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
import { BitPacker } from "@suaveplan/bitpacker";
|
||||
import { registerDefaultStrategies } from "@suaveplan/bitpacker-strategies";
|
||||
import { describe, expect, test } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
addFrame,
|
||||
createFramedPackStream,
|
||||
createFramedUnpackStream,
|
||||
createPackTransformStream,
|
||||
createUnpackTransformStream,
|
||||
packToChunks,
|
||||
parseFrames,
|
||||
unpackFromChunks,
|
||||
unpackFromChunksAsync,
|
||||
} from "./index.js";
|
||||
|
||||
describe("packToChunks / unpackFromChunks", () => {
|
||||
test("roundtrips simple data", () => {
|
||||
const schema = 255;
|
||||
const data = 42;
|
||||
const chunks = packToChunks(data, schema, { chunkSize: 2 });
|
||||
expect(chunks.length).toBeGreaterThanOrEqual(1);
|
||||
const result = unpackFromChunks<number>(chunks, schema);
|
||||
expect(result).toBe(42);
|
||||
});
|
||||
test("skips empty chunks", () => {
|
||||
const schema = 255;
|
||||
const data = 42;
|
||||
const chunks = [new Uint8Array(0), ...packToChunks(data, schema)];
|
||||
const result = unpackFromChunks<number>(chunks, schema);
|
||||
expect(result).toBe(42);
|
||||
});
|
||||
test("async roundtrip", async () => {
|
||||
const schema = "-64";
|
||||
const data = "hello world";
|
||||
const chunks = packToChunks(data, schema);
|
||||
async function* gen() {
|
||||
for (const c of chunks) yield c;
|
||||
}
|
||||
const result = await unpackFromChunksAsync<string>(gen(), schema);
|
||||
expect(result).toBe("hello world");
|
||||
});
|
||||
test("async aborts on signal", async () => {
|
||||
const schema = 255;
|
||||
const data = 42;
|
||||
const chunks = packToChunks(data, schema);
|
||||
async function* gen() {
|
||||
for (const c of chunks) yield c;
|
||||
}
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(
|
||||
unpackFromChunksAsync<number>(gen(), schema, {
|
||||
signal: controller.signal,
|
||||
})
|
||||
).rejects.toThrow("aborted");
|
||||
});
|
||||
test("async skips empty chunks", async () => {
|
||||
const schema = 255;
|
||||
const data = 42;
|
||||
const chunks = packToChunks(data, schema);
|
||||
async function* gen() {
|
||||
yield new Uint8Array(0);
|
||||
for (const c of chunks) yield c;
|
||||
}
|
||||
const result = await unpackFromChunksAsync<number>(gen(), schema);
|
||||
expect(result).toBe(42);
|
||||
});
|
||||
});
|
||||
|
||||
describe("framing", () => {
|
||||
test("length-prefix framing roundtrips", () => {
|
||||
const data = new Uint8Array([1, 2, 3, 4, 5]);
|
||||
const framed = addFrame(data, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
expect(framed.length).toBe(7);
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
expect(remainder.length).toBe(0);
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0]).toEqual(data);
|
||||
});
|
||||
test("delimiter framing roundtrips", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, {
|
||||
type: "delimiter",
|
||||
delimiter: new Uint8Array([0xff]),
|
||||
});
|
||||
expect(framed[framed.length - 1]).toBe(0xff);
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "delimiter",
|
||||
delimiter: new Uint8Array([0xff]),
|
||||
});
|
||||
expect(frames.length).toBe(1);
|
||||
expect(frames[0]).toEqual(data);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("fixed-size framing", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, { type: "fixed-size", fixedSize: 8 });
|
||||
expect(framed.length).toBe(8);
|
||||
});
|
||||
test("none framing passes through", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, { type: "none" });
|
||||
expect(framed).toEqual(data);
|
||||
});
|
||||
test("length-prefix 1-byte roundtrips", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 1,
|
||||
});
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 1,
|
||||
});
|
||||
expect(remainder.length).toBe(0);
|
||||
expect(frames[0]).toEqual(data);
|
||||
});
|
||||
test("length-prefix 4-byte roundtrips", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 4,
|
||||
});
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 4,
|
||||
});
|
||||
expect(remainder.length).toBe(0);
|
||||
expect(frames[0]).toEqual(data);
|
||||
});
|
||||
test("fixed-size framing roundtrips", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, { type: "fixed-size", fixedSize: 8 });
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "fixed-size",
|
||||
fixedSize: 8,
|
||||
});
|
||||
expect(remainder.length).toBe(0);
|
||||
expect(frames[0]).toEqual(framed);
|
||||
});
|
||||
test("parseFrames handles partial data", () => {
|
||||
const buffer = new Uint8Array([0, 5]);
|
||||
const { frames, remainder } = parseFrames(buffer, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
expect(frames.length).toBe(0);
|
||||
expect(remainder.length).toBe(2);
|
||||
});
|
||||
test("parseFrames none type", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const { frames, remainder } = parseFrames(data, { type: "none" });
|
||||
expect(frames[0]).toEqual(data);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames uses defaults for length-prefix", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, { type: "length-prefix" });
|
||||
expect(framed.length).toBe(7);
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "length-prefix",
|
||||
});
|
||||
expect(frames[0]).toEqual(data);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames uses defaults for delimiter", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, { type: "delimiter" });
|
||||
expect(framed[framed.length - 1]).toBe(0);
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "delimiter",
|
||||
});
|
||||
expect(frames[0]).toEqual(data);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames uses defaults for fixed-size", () => {
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const framed = addFrame(data, { type: "fixed-size" });
|
||||
expect(framed.length).toBe(256);
|
||||
const { frames, remainder } = parseFrames(framed, {
|
||||
type: "fixed-size",
|
||||
});
|
||||
expect(frames.length).toBe(1);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames delimiter at start of buffer", () => {
|
||||
const delimiter = new Uint8Array([0xff]);
|
||||
const buffer = new Uint8Array([0xff]);
|
||||
const { frames, remainder } = parseFrames(buffer, {
|
||||
type: "delimiter",
|
||||
delimiter,
|
||||
});
|
||||
expect(frames.length).toBe(0);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames empty buffer with none", () => {
|
||||
const { frames, remainder } = parseFrames(new Uint8Array(0), {
|
||||
type: "none",
|
||||
});
|
||||
expect(frames.length).toBe(0);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames empty buffer with delimiter", () => {
|
||||
const { frames, remainder } = parseFrames(new Uint8Array(0), {
|
||||
type: "delimiter",
|
||||
delimiter: new Uint8Array([0xff]),
|
||||
});
|
||||
expect(frames.length).toBe(0);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames empty buffer with fixed-size", () => {
|
||||
const { frames, remainder } = parseFrames(new Uint8Array(0), {
|
||||
type: "fixed-size",
|
||||
fixedSize: 8,
|
||||
});
|
||||
expect(frames.length).toBe(0);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
test("parseFrames empty buffer with length-prefix", () => {
|
||||
const { frames, remainder } = parseFrames(new Uint8Array(0), {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 4,
|
||||
});
|
||||
expect(frames.length).toBe(0);
|
||||
expect(remainder.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("TransformStreams", () => {
|
||||
test("createPackTransformStream with batchSize > 1", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const stream = createPackTransformStream(bp, schema, { batchSize: 2 });
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(1);
|
||||
await writer.write(2);
|
||||
await writer.close();
|
||||
|
||||
const reader = stream.readable.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
expect(chunks.length).toBe(2);
|
||||
});
|
||||
test("createPackTransformStream flushes remaining batch", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const stream = createPackTransformStream(bp, schema, { batchSize: 2 });
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(1);
|
||||
await writer.close();
|
||||
|
||||
const reader = stream.readable.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
expect(chunks.length).toBe(1);
|
||||
});
|
||||
test("createPackTransformStream calls onProgress", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
let time = 1000;
|
||||
Date.now = () => time++;
|
||||
try {
|
||||
const stream = createPackTransformStream(bp, schema, {
|
||||
batchSize: 1,
|
||||
onProgress: (p) => calls.push(p),
|
||||
});
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(1);
|
||||
await writer.close();
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[0]).toMatchObject({
|
||||
itemsProcessed: expect.any(Number),
|
||||
bytesProcessed: expect.any(Number),
|
||||
operation: "pack",
|
||||
});
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: expect.any(Number),
|
||||
});
|
||||
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
|
||||
});
|
||||
test("createPackTransformStream handles zero duration", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
Date.now = () => 1000;
|
||||
try {
|
||||
const stream = createPackTransformStream(bp, schema, {
|
||||
batchSize: 1,
|
||||
onProgress: (p) => calls.push(p),
|
||||
});
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(1);
|
||||
await writer.close();
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: 0,
|
||||
operation: "pack",
|
||||
});
|
||||
});
|
||||
test("createUnpackTransformStream decodes data", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
|
||||
const packStream = createPackTransformStream(bp, schema);
|
||||
const unpackStream = createUnpackTransformStream(bp, schema);
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(42);
|
||||
});
|
||||
test("createUnpackTransformStream decodes framed data in transform", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
|
||||
const packStream = createFramedPackStream(bp, schema, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 4,
|
||||
});
|
||||
const unpackStream = createUnpackTransformStream(bp, schema, {
|
||||
headerSize: 4,
|
||||
});
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(42);
|
||||
});
|
||||
test("createUnpackTransformStream calls onProgress", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
let time = 1000;
|
||||
Date.now = () => time++;
|
||||
try {
|
||||
const packStream = createPackTransformStream(bp, schema);
|
||||
const unpackStream = createUnpackTransformStream(bp, schema, {
|
||||
onProgress: (p) => calls.push(p),
|
||||
});
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(42);
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[0]).toMatchObject({ operation: "unpack" });
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: expect.any(Number),
|
||||
});
|
||||
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
|
||||
});
|
||||
test("createUnpackTransformStream handles zero duration", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
Date.now = () => 1000;
|
||||
try {
|
||||
const packStream = createPackTransformStream(bp, schema);
|
||||
const unpackStream = createUnpackTransformStream(bp, schema, {
|
||||
onProgress: (p) => calls.push(p),
|
||||
});
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
await reader.read();
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: 0,
|
||||
operation: "unpack",
|
||||
});
|
||||
});
|
||||
test("createUnpackTransformStream flushes incomplete valid frame", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
|
||||
const unpackStream = createUnpackTransformStream(bp, schema, {
|
||||
headerSize: 4,
|
||||
});
|
||||
const writer = unpackStream.writable.getWriter();
|
||||
// 4-byte header indicating length=1, but no payload
|
||||
await writer.write(new Uint8Array([0, 0, 0, 1]));
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(0);
|
||||
});
|
||||
test("createUnpackTransformStream flushes incomplete invalid frame", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = "f32";
|
||||
|
||||
const unpackStream = createUnpackTransformStream(bp, schema, {
|
||||
headerSize: 4,
|
||||
});
|
||||
const writer = unpackStream.writable.getWriter();
|
||||
// 3 bytes is less than header size (4), so parseFrames leaves it in buffer
|
||||
// f32 needs 4 bytes, so unpacking 3 bytes should fail
|
||||
await writer.write(new Uint8Array([0, 0, 0]));
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { done } = await reader.read();
|
||||
expect(done).toBe(true);
|
||||
});
|
||||
|
||||
test("createFramedPackStream emits framed data", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const stream = createFramedPackStream(bp, schema, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = stream.readable.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
expect(chunks.length).toBeGreaterThan(0);
|
||||
expect(chunks[0]!.length).toBeGreaterThan(2);
|
||||
});
|
||||
test("createFramedPackStream uses default framing", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const stream = createFramedPackStream(bp, schema);
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = stream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value!.length).toBeGreaterThan(1);
|
||||
});
|
||||
test("createFramedPackStream calls onProgress", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
let time = 1000;
|
||||
Date.now = () => time++;
|
||||
try {
|
||||
const stream = createFramedPackStream(
|
||||
bp,
|
||||
schema,
|
||||
{ type: "length-prefix", lengthPrefixSize: 2 },
|
||||
{ onProgress: (p) => calls.push(p) }
|
||||
);
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[0]).toMatchObject({ operation: "pack" });
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: expect.any(Number),
|
||||
});
|
||||
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
|
||||
});
|
||||
test("createFramedPackStream handles zero duration", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
Date.now = () => 1000;
|
||||
try {
|
||||
const stream = createFramedPackStream(
|
||||
bp,
|
||||
schema,
|
||||
{ type: "length-prefix", lengthPrefixSize: 2 },
|
||||
{ onProgress: (p) => calls.push(p) }
|
||||
);
|
||||
const writer = stream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: 0,
|
||||
operation: "pack",
|
||||
});
|
||||
});
|
||||
test("createFramedUnpackStream decodes framed data", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
|
||||
const packStream = createFramedPackStream(bp, schema, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
const unpackStream = createFramedUnpackStream(bp, schema, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(42);
|
||||
});
|
||||
test("createFramedUnpackStream uses default framing", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const packStream = createFramedPackStream(bp, schema);
|
||||
const unpackStream = createFramedUnpackStream(bp, schema);
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(42);
|
||||
});
|
||||
test("createFramedUnpackStream calls onProgress", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
let time = 1000;
|
||||
Date.now = () => time++;
|
||||
try {
|
||||
const packStream = createFramedPackStream(bp, schema, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
const unpackStream = createFramedUnpackStream(
|
||||
bp,
|
||||
schema,
|
||||
{ type: "length-prefix", lengthPrefixSize: 2 },
|
||||
{ onProgress: (p) => calls.push(p) }
|
||||
);
|
||||
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
const { value } = await reader.read();
|
||||
expect(value).toBe(42);
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[0]).toMatchObject({ operation: "unpack" });
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: expect.any(Number),
|
||||
});
|
||||
expect(calls[calls.length - 1]!.throughput).toBeGreaterThan(0);
|
||||
});
|
||||
test("createFramedUnpackStream handles zero duration", async () => {
|
||||
const bp = new BitPacker();
|
||||
registerDefaultStrategies(bp.strategyManager);
|
||||
const schema = 255;
|
||||
const calls: unknown[] = [];
|
||||
const originalNow = Date.now;
|
||||
Date.now = () => 1000;
|
||||
try {
|
||||
const packStream = createFramedPackStream(bp, schema, {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 2,
|
||||
});
|
||||
const unpackStream = createFramedUnpackStream(
|
||||
bp,
|
||||
schema,
|
||||
{ type: "length-prefix", lengthPrefixSize: 2 },
|
||||
{ onProgress: (p) => calls.push(p) }
|
||||
);
|
||||
|
||||
packStream.readable.pipeTo(unpackStream.writable);
|
||||
|
||||
const writer = packStream.writable.getWriter();
|
||||
await writer.write(42);
|
||||
await writer.close();
|
||||
|
||||
const reader = unpackStream.readable.getReader();
|
||||
await reader.read();
|
||||
} finally {
|
||||
Date.now = originalNow;
|
||||
}
|
||||
|
||||
expect(calls.length).toBeGreaterThan(0);
|
||||
expect(calls[calls.length - 1]).toMatchObject({
|
||||
throughput: 0,
|
||||
operation: "unpack",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs --> - transform.schema — Zod schemas for the option bundl…
|
||||
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- transform.schema — Zod schemas for the option bundl…
|
||||
|
||||
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
|
||||
- transform.schema — Zod schemas for the option bundl…
|
||||
|
||||
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
|
||||
- transform.schema — Zod schemas for the option bundl…
|
||||
|
||||
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
- [transform.schema](./transform.schema.md) — Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the fram…
|
||||
|
||||
## Transform Streams
|
||||
|
||||
**Path:** `src/transform/transform.ts`
|
||||
**Export:** `createPackTransformStream`, `createUnpackTransformStream`, `createFramedPackStream`, `createFramedUnpackStream`, `parseFrames`, `addFrame`
|
||||
|
||||
### What it does
|
||||
|
||||
True `TransformStream` support for BitPacker with backpressure, batching, progress callbacks, and multiple framing strategies.
|
||||
|
||||
### Key API
|
||||
|
||||
| Function | Description |
|
||||
|----------|-------------|
|
||||
| `createPackTransformStream(bp, schema, options?)` | Transform objects → `Uint8Array` chunks |
|
||||
| `createUnpackTransformStream(bp, schema, options?)` | Transform `Uint8Array` → objects (length-prefix framing) |
|
||||
| `createFramedPackStream(bp, schema, framing?, options?)` | Pack with explicit framing config |
|
||||
| `createFramedUnpackStream(bp, schema, framing?, options?)` | Unpack with explicit framing config |
|
||||
| `parseFrames(buffer, framing)` | Extract complete frames from a buffer |
|
||||
| `addFrame(data, framing)` | Add framing to a message |
|
||||
|
||||
### Framing Types
|
||||
|
||||
- `length-prefix` — prefix with 1/2/4 byte length (default)
|
||||
- `delimiter` — suffix with delimiter bytes
|
||||
- `fixed-size` — pad/truncate to fixed size
|
||||
- `none` — no framing (single message per chunk)
|
||||
|
||||
### Depends on
|
||||
|
||||
- `BitPacker` from `@suaveplan/bitpacker`
|
||||
|
||||
### Used by
|
||||
|
||||
- Network streaming pipelines
|
||||
- File serialization with chunked I/O
|
||||
@@ -0,0 +1,79 @@
|
||||
# Transform Stream Schemas
|
||||
|
||||
## Purpose
|
||||
|
||||
Zod schemas for the option bundles consumed by `createPackTransformStream`, `createUnpackTransformStream`, and the framing helpers (`addFrame`, `parseFrames`, `createFramedPackStream`, `createFramedUnpackStream`). Each schema is the runtime gatekeeper at the corresponding boundary — `framingConfigSchema` validates the frame-strategy descriptor, and `packTransformOptionsSchema` / `unpackTransformOptionsSchema` validate the per-call tuning bundles.
|
||||
|
||||
The framing schema deliberately does NOT enforce cross-field requirements (e.g. "if `type === 'length-prefix'` then `lengthPrefixSize` is required"). The runtime parser tolerates the extra fields, and forcing the cross-field check at parse time would force callers to always specify `lengthPrefixSize: 4` even when defaulting it. Strict-mode rejects unknown keys; the runtime parser is the authority on which combinations make sense.
|
||||
|
||||
## Features
|
||||
|
||||
- `framingConfigSchema` — discriminated by `type` (`length-prefix` / `delimiter` / `fixed-size` / `none`)
|
||||
- `packTransformOptionsSchema` — `batchSize`, `highWaterMark`, `onProgress`
|
||||
- `unpackTransformOptionsSchema` — `highWaterMark`, `onProgress`, `headerSize`
|
||||
- Identity-preserving `z.custom` for `onProgress`
|
||||
- `Uint8Array` instance check for `delimiter` (rejects array-of-numbers literals)
|
||||
- `as unknown as z.ZodType<T>` casts for EOPT mismatches
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
framingConfigSchema,
|
||||
packTransformOptionsSchema,
|
||||
} from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
const framing = framingConfigSchema.parse({
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 4,
|
||||
});
|
||||
|
||||
const packOpts = packTransformOptionsSchema.parse({
|
||||
batchSize: 4,
|
||||
highWaterMark: 16,
|
||||
onProgress: (p) => console.log(p),
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
addFrame,
|
||||
framingConfigSchema,
|
||||
parseFrames,
|
||||
unpackTransformOptionsSchema,
|
||||
} from "@suaveplan/bitpacker-streaming";
|
||||
|
||||
// Validate framing at the boundary, then trust the parsed shape downstream:
|
||||
function withFraming(rawFraming: unknown, payload: Uint8Array) {
|
||||
const cfg = framingConfigSchema.parse(rawFraming);
|
||||
return addFrame(payload, cfg);
|
||||
}
|
||||
|
||||
// Per-call unpack options, including a length-prefix header width:
|
||||
const unpackOpts = unpackTransformOptionsSchema.parse({
|
||||
highWaterMark: 32,
|
||||
headerSize: 4,
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `framingConfigSchema: z.ZodType<FramingConfig>`
|
||||
|
||||
Strict schema for the `FramingConfig` discriminator. `type` is one of four literal tokens. `lengthPrefixSize` is restricted to `1 | 2 | 4`. `delimiter` must be a `Uint8Array` (not a `number[]` literal). `fixedSize` is a positive integer.
|
||||
|
||||
### `packTransformOptionsSchema: z.ZodType<PackTransformOptions>`
|
||||
|
||||
Strict schema. `batchSize` and `highWaterMark` are positive integers. `onProgress` is a function preserved by identity.
|
||||
|
||||
### `unpackTransformOptionsSchema: z.ZodType<UnpackTransformOptions>`
|
||||
|
||||
Strict schema. `highWaterMark` and `headerSize` are positive integers. `onProgress` is a function preserved by identity.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- **No cross-field validation.** A `length-prefix` framing without `lengthPrefixSize` is technically incomplete, but the runtime parser defaults it to `4` and tolerates the omission. Forcing a cross-field rule at parse time would over-constrain callers.
|
||||
- **`Uint8Array` instance check on `delimiter`.** A `[0x0a]` literal looks like a delimiter but does not implement the typed-array surface the parser uses (`.subarray`, `.byteLength`). Refusing it at parse time prevents a silent type coercion that would surface as an obscure runtime error.
|
||||
- **`as unknown as` casts.** Same EOPT mismatch as elsewhere in this change. Runtime semantics are identical.
|
||||
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
framingConfigSchema,
|
||||
packTransformOptionsSchema,
|
||||
unpackTransformOptionsSchema,
|
||||
} from "./transform.schema.js";
|
||||
|
||||
describe("framingConfigSchema", () => {
|
||||
it("accepts length-prefix framing", () => {
|
||||
const cfg = {
|
||||
type: "length-prefix" as const,
|
||||
lengthPrefixSize: 4 as const,
|
||||
};
|
||||
expect(framingConfigSchema.parse(cfg)).toEqual(cfg);
|
||||
});
|
||||
|
||||
it("accepts delimiter framing with a Uint8Array delimiter", () => {
|
||||
const cfg = {
|
||||
type: "delimiter" as const,
|
||||
delimiter: new Uint8Array([0]),
|
||||
};
|
||||
const parsed = framingConfigSchema.parse(cfg);
|
||||
expect(parsed.type).toBe("delimiter");
|
||||
expect(parsed.delimiter).toBe(cfg.delimiter);
|
||||
});
|
||||
|
||||
it("accepts fixed-size framing", () => {
|
||||
const cfg = { type: "fixed-size" as const, fixedSize: 64 };
|
||||
expect(framingConfigSchema.parse(cfg)).toEqual(cfg);
|
||||
});
|
||||
|
||||
it("accepts the degenerate `none` type", () => {
|
||||
const cfg = { type: "none" as const };
|
||||
expect(framingConfigSchema.parse(cfg)).toEqual(cfg);
|
||||
});
|
||||
|
||||
it("rejects an unknown type token", () => {
|
||||
expect(() => framingConfigSchema.parse({ type: "ndjson" })).toThrow();
|
||||
});
|
||||
|
||||
it("rejects a non-Uint8Array delimiter", () => {
|
||||
expect(() =>
|
||||
framingConfigSchema.parse({
|
||||
type: "delimiter",
|
||||
delimiter: [0],
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects an unsupported lengthPrefixSize", () => {
|
||||
expect(() =>
|
||||
framingConfigSchema.parse({
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: 8,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("packTransformOptionsSchema", () => {
|
||||
it("accepts an empty bundle", () => {
|
||||
expect(packTransformOptionsSchema.parse({})).toEqual({});
|
||||
});
|
||||
|
||||
it("accepts batchSize and highWaterMark", () => {
|
||||
const opts = { batchSize: 4, highWaterMark: 32 };
|
||||
expect(packTransformOptionsSchema.parse(opts)).toEqual(opts);
|
||||
});
|
||||
|
||||
it("rejects non-positive batchSize", () => {
|
||||
expect(() =>
|
||||
packTransformOptionsSchema.parse({ batchSize: 0 })
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("unpackTransformOptionsSchema", () => {
|
||||
it("accepts headerSize", () => {
|
||||
const opts = { headerSize: 4 };
|
||||
expect(unpackTransformOptionsSchema.parse(opts)).toEqual(opts);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { FramingConfig } from "@suaveplan/types";
|
||||
import z from "zod";
|
||||
import type {
|
||||
PackTransformOptions,
|
||||
UnpackTransformOptions,
|
||||
} from "./transform.types.js";
|
||||
|
||||
const callable = <T>(message: string) =>
|
||||
z.custom<T>((v): v is T => typeof v === "function", { message });
|
||||
|
||||
const uint8ArraySchema = z.custom<Uint8Array>(
|
||||
(v): v is Uint8Array => v instanceof Uint8Array,
|
||||
{ message: "delimiter must be a Uint8Array" }
|
||||
);
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link FramingConfig}.
|
||||
*
|
||||
* Validates the framing-strategy descriptor that drives `parseFrames`,
|
||||
* `addFrame`, and the framed pack/unpack streams. Cross-field requirements
|
||||
* are NOT enforced here (e.g. `lengthPrefixSize` is only meaningful for
|
||||
* `type: "length-prefix"`) because the runtime parser tolerates the extra
|
||||
* fields. Strict-mode rejects unknown keys.
|
||||
*/
|
||||
export const framingConfigSchema = z
|
||||
.object({
|
||||
type: z.union([
|
||||
z.literal("length-prefix"),
|
||||
z.literal("delimiter"),
|
||||
z.literal("fixed-size"),
|
||||
z.literal("none"),
|
||||
]),
|
||||
lengthPrefixSize: z
|
||||
.union([z.literal(1), z.literal(2), z.literal(4)])
|
||||
.optional(),
|
||||
delimiter: uint8ArraySchema.optional(),
|
||||
fixedSize: z.number().int().positive().optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<FramingConfig>;
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link PackTransformOptions}.
|
||||
*
|
||||
* Validates `createPackTransformStream`'s options bundle. `batchSize` and
|
||||
* `highWaterMark` must be positive integers; `onProgress` is a function
|
||||
* preserved by identity.
|
||||
*/
|
||||
export const packTransformOptionsSchema = z
|
||||
.object({
|
||||
batchSize: z.number().int().positive().optional(),
|
||||
highWaterMark: z.number().int().positive().optional(),
|
||||
onProgress: callable<NonNullable<PackTransformOptions["onProgress"]>>(
|
||||
"onProgress must be a function"
|
||||
).optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<PackTransformOptions>;
|
||||
|
||||
/**
|
||||
* Strict Zod schema for {@link UnpackTransformOptions}.
|
||||
*
|
||||
* Validates `createUnpackTransformStream`'s options bundle. `headerSize` is
|
||||
* the length-prefix width when the function infers a `length-prefix`
|
||||
* framing config.
|
||||
*/
|
||||
export const unpackTransformOptionsSchema = z
|
||||
.object({
|
||||
highWaterMark: z.number().int().positive().optional(),
|
||||
onProgress: callable<NonNullable<UnpackTransformOptions["onProgress"]>>(
|
||||
"onProgress must be a function"
|
||||
).optional(),
|
||||
headerSize: z.number().int().positive().optional(),
|
||||
})
|
||||
.strict() as unknown as z.ZodType<UnpackTransformOptions>;
|
||||
@@ -0,0 +1,365 @@
|
||||
import type { BitPacker, BitPackerSchema } from "@suaveplan/bitpacker";
|
||||
import type {
|
||||
FrameParseResult,
|
||||
FramingConfig,
|
||||
PackTransformOptions,
|
||||
UnpackTransformOptions,
|
||||
} from "./transform.types.js";
|
||||
|
||||
export * from "./transform.types.js";
|
||||
|
||||
export function createPackTransformStream<T extends BitPackerSchema>(
|
||||
bitPacker: BitPacker,
|
||||
schema: T,
|
||||
options: PackTransformOptions = {}
|
||||
): TransformStream<unknown, Uint8Array> {
|
||||
const { batchSize = 1, highWaterMark = 16, onProgress } = options;
|
||||
let itemsProcessed = 0;
|
||||
let bytesProcessed = 0;
|
||||
let startTime = Date.now();
|
||||
let batch: unknown[] = [];
|
||||
|
||||
const updateProgress = () => {
|
||||
if (onProgress) {
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed,
|
||||
bytesProcessed,
|
||||
itemsPending: batch.length,
|
||||
throughput: duration > 0 ? itemsProcessed / duration : 0,
|
||||
operation: "pack",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
return new TransformStream<unknown, Uint8Array>(
|
||||
{
|
||||
start() {
|
||||
startTime = Date.now();
|
||||
},
|
||||
transform(chunk, controller) {
|
||||
batch.push(chunk);
|
||||
if (batch.length >= batchSize) {
|
||||
for (const item of batch) {
|
||||
const packed = bitPacker.pack(item, schema);
|
||||
bytesProcessed += packed.length;
|
||||
itemsProcessed++;
|
||||
controller.enqueue(packed);
|
||||
}
|
||||
batch = [];
|
||||
updateProgress();
|
||||
}
|
||||
},
|
||||
flush(controller) {
|
||||
for (const item of batch) {
|
||||
const packed = bitPacker.pack(item, schema);
|
||||
bytesProcessed += packed.length;
|
||||
itemsProcessed++;
|
||||
controller.enqueue(packed);
|
||||
}
|
||||
batch = [];
|
||||
updateProgress();
|
||||
},
|
||||
},
|
||||
{ highWaterMark },
|
||||
{ highWaterMark }
|
||||
);
|
||||
}
|
||||
|
||||
export function createUnpackTransformStream<T extends BitPackerSchema>(
|
||||
bitPacker: BitPacker,
|
||||
schema: T,
|
||||
options: UnpackTransformOptions = {}
|
||||
): TransformStream<Uint8Array, unknown> {
|
||||
const { highWaterMark = 16, onProgress, headerSize = 4 } = options;
|
||||
let itemsProcessed = 0;
|
||||
let bytesProcessed = 0;
|
||||
let startTime = Date.now();
|
||||
let buffer: Uint8Array = new Uint8Array(0);
|
||||
|
||||
const framing: FramingConfig = {
|
||||
type: "length-prefix",
|
||||
lengthPrefixSize: headerSize as 1 | 2 | 4,
|
||||
};
|
||||
|
||||
const updateProgress = () => {
|
||||
if (onProgress) {
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed,
|
||||
bytesProcessed,
|
||||
itemsPending: 0,
|
||||
throughput: duration > 0 ? itemsProcessed / duration : 0,
|
||||
operation: "unpack",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
return new TransformStream<Uint8Array, unknown>(
|
||||
{
|
||||
start() {
|
||||
startTime = Date.now();
|
||||
},
|
||||
transform(chunk, controller) {
|
||||
const newBuffer = new Uint8Array(buffer.length + chunk.length);
|
||||
newBuffer.set(buffer);
|
||||
newBuffer.set(chunk, buffer.length);
|
||||
buffer = newBuffer;
|
||||
|
||||
const { frames, remainder } = parseFrames(buffer, framing);
|
||||
buffer = remainder;
|
||||
|
||||
for (const frame of frames) {
|
||||
const unpacked = bitPacker.unpack(frame, schema);
|
||||
bytesProcessed += frame.length;
|
||||
itemsProcessed++;
|
||||
controller.enqueue(unpacked);
|
||||
}
|
||||
updateProgress();
|
||||
},
|
||||
flush(controller) {
|
||||
if (buffer.length > 0) {
|
||||
try {
|
||||
const unpacked = bitPacker.unpack(buffer, schema);
|
||||
bytesProcessed += buffer.length;
|
||||
itemsProcessed++;
|
||||
controller.enqueue(unpacked);
|
||||
} catch {
|
||||
// Incomplete data, ignore
|
||||
}
|
||||
}
|
||||
updateProgress();
|
||||
},
|
||||
},
|
||||
{ highWaterMark },
|
||||
{ highWaterMark }
|
||||
);
|
||||
}
|
||||
|
||||
export function createFramedPackStream<T extends BitPackerSchema>(
|
||||
bitPacker: BitPacker,
|
||||
schema: T,
|
||||
framing: FramingConfig = { type: "length-prefix", lengthPrefixSize: 4 },
|
||||
options: PackTransformOptions = {}
|
||||
): TransformStream<unknown, Uint8Array> {
|
||||
const { highWaterMark = 16, onProgress } = options;
|
||||
let itemsProcessed = 0;
|
||||
let bytesProcessed = 0;
|
||||
let startTime = Date.now();
|
||||
|
||||
return new TransformStream<unknown, Uint8Array>(
|
||||
{
|
||||
start() {
|
||||
startTime = Date.now();
|
||||
},
|
||||
transform(chunk, controller) {
|
||||
const packed = bitPacker.pack(chunk, schema);
|
||||
const framed = addFrame(packed, framing);
|
||||
bytesProcessed += framed.length;
|
||||
itemsProcessed++;
|
||||
controller.enqueue(framed);
|
||||
|
||||
if (onProgress) {
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed,
|
||||
bytesProcessed,
|
||||
itemsPending: 0,
|
||||
throughput:
|
||||
duration > 0 ? itemsProcessed / duration : 0,
|
||||
operation: "pack",
|
||||
});
|
||||
}
|
||||
},
|
||||
},
|
||||
{ highWaterMark },
|
||||
{ highWaterMark }
|
||||
);
|
||||
}
|
||||
|
||||
export function createFramedUnpackStream<T extends BitPackerSchema>(
|
||||
bitPacker: BitPacker,
|
||||
schema: T,
|
||||
framing: FramingConfig = { type: "length-prefix", lengthPrefixSize: 4 },
|
||||
options: UnpackTransformOptions = {}
|
||||
): TransformStream<Uint8Array, unknown> {
|
||||
const { highWaterMark = 16, onProgress } = options;
|
||||
let itemsProcessed = 0;
|
||||
let bytesProcessed = 0;
|
||||
let startTime = Date.now();
|
||||
let buffer: Uint8Array = new Uint8Array(0);
|
||||
|
||||
return new TransformStream<Uint8Array, unknown>(
|
||||
{
|
||||
start() {
|
||||
startTime = Date.now();
|
||||
},
|
||||
transform(chunk, controller) {
|
||||
const newBuffer = new Uint8Array(buffer.length + chunk.length);
|
||||
newBuffer.set(buffer);
|
||||
newBuffer.set(chunk, buffer.length);
|
||||
buffer = newBuffer;
|
||||
|
||||
const { frames, remainder } = parseFrames(buffer, framing);
|
||||
buffer = remainder;
|
||||
|
||||
for (const frame of frames) {
|
||||
const unpacked = bitPacker.unpack(frame, schema);
|
||||
bytesProcessed += frame.length;
|
||||
itemsProcessed++;
|
||||
controller.enqueue(unpacked);
|
||||
}
|
||||
|
||||
if (onProgress) {
|
||||
const duration = (Date.now() - startTime) / 1000;
|
||||
onProgress({
|
||||
itemsProcessed,
|
||||
bytesProcessed,
|
||||
itemsPending: 0,
|
||||
throughput:
|
||||
duration > 0 ? itemsProcessed / duration : 0,
|
||||
operation: "unpack",
|
||||
});
|
||||
}
|
||||
},
|
||||
flush() {
|
||||
// With framing.type === "none", parseFrames always consumes
|
||||
// the entire buffer, so there is never any remainder here.
|
||||
},
|
||||
},
|
||||
{ highWaterMark },
|
||||
{ highWaterMark }
|
||||
);
|
||||
}
|
||||
|
||||
export function parseFrames(
|
||||
buffer: Uint8Array,
|
||||
framing: FramingConfig
|
||||
): FrameParseResult {
|
||||
const frames: Uint8Array[] = [];
|
||||
let offset = 0;
|
||||
|
||||
switch (framing.type) {
|
||||
case "length-prefix": {
|
||||
const prefixSize = framing.lengthPrefixSize ?? 4;
|
||||
while (offset + prefixSize <= buffer.length) {
|
||||
const length = readLengthPrefix(buffer, offset, prefixSize);
|
||||
if (offset + prefixSize + length > buffer.length) break;
|
||||
frames.push(
|
||||
buffer.slice(
|
||||
offset + prefixSize,
|
||||
offset + prefixSize + length
|
||||
)
|
||||
);
|
||||
offset += prefixSize + length;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "delimiter": {
|
||||
const delimiter = framing.delimiter ?? new Uint8Array([0]);
|
||||
let start = 0;
|
||||
for (let i = 0; i <= buffer.length - delimiter.length; i++) {
|
||||
let match = true;
|
||||
for (let j = 0; j < delimiter.length; j++) {
|
||||
if (buffer[i + j] !== delimiter[j]) {
|
||||
match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (match) {
|
||||
if (i > start) frames.push(buffer.slice(start, i));
|
||||
start = i + delimiter.length;
|
||||
i = start - 1;
|
||||
}
|
||||
}
|
||||
offset = start;
|
||||
break;
|
||||
}
|
||||
case "fixed-size": {
|
||||
const size = framing.fixedSize ?? 256;
|
||||
while (offset + size <= buffer.length) {
|
||||
frames.push(buffer.slice(offset, offset + size));
|
||||
offset += size;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "none":
|
||||
if (buffer.length > 0) {
|
||||
frames.push(buffer);
|
||||
offset = buffer.length;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return { frames, remainder: buffer.slice(offset) };
|
||||
}
|
||||
|
||||
export function addFrame(data: Uint8Array, framing: FramingConfig): Uint8Array {
|
||||
switch (framing.type) {
|
||||
case "length-prefix": {
|
||||
const prefixSize = framing.lengthPrefixSize ?? 4;
|
||||
const framed = new Uint8Array(prefixSize + data.length);
|
||||
writeLengthPrefix(framed, 0, data.length, prefixSize);
|
||||
framed.set(data, prefixSize);
|
||||
return framed;
|
||||
}
|
||||
case "delimiter": {
|
||||
const delimiter = framing.delimiter ?? new Uint8Array([0]);
|
||||
const framed = new Uint8Array(data.length + delimiter.length);
|
||||
framed.set(data);
|
||||
framed.set(delimiter, data.length);
|
||||
return framed;
|
||||
}
|
||||
case "fixed-size": {
|
||||
const size = framing.fixedSize ?? 256;
|
||||
const framed = new Uint8Array(size);
|
||||
framed.set(data.slice(0, size));
|
||||
return framed;
|
||||
}
|
||||
default:
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
function readLengthPrefix(
|
||||
buffer: Uint8Array,
|
||||
offset: number,
|
||||
size: 1 | 2 | 4
|
||||
): number {
|
||||
switch (size) {
|
||||
case 1:
|
||||
return buffer[offset]!;
|
||||
case 2:
|
||||
return (buffer[offset]! << 8) | buffer[offset + 1]!;
|
||||
case 4:
|
||||
return (
|
||||
(buffer[offset]! << 24) |
|
||||
(buffer[offset + 1]! << 16) |
|
||||
(buffer[offset + 2]! << 8) |
|
||||
buffer[offset + 3]!
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function writeLengthPrefix(
|
||||
buffer: Uint8Array,
|
||||
offset: number,
|
||||
length: number,
|
||||
size: 1 | 2 | 4
|
||||
): void {
|
||||
switch (size) {
|
||||
case 1:
|
||||
buffer[offset] = length & 0xff;
|
||||
break;
|
||||
case 2:
|
||||
buffer[offset] = (length >> 8) & 0xff;
|
||||
buffer[offset + 1] = length & 0xff;
|
||||
break;
|
||||
case 4:
|
||||
buffer[offset] = (length >> 24) & 0xff;
|
||||
buffer[offset + 1] = (length >> 16) & 0xff;
|
||||
buffer[offset + 2] = (length >> 8) & 0xff;
|
||||
buffer[offset + 3] = length & 0xff;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
## transform.types
|
||||
|
||||
**Path:** `bitpacker-streaming/src/transform/transform.types.ts`
|
||||
**Export:** See source for details.
|
||||
|
||||
### What it does
|
||||
|
||||
Part of the @suaveplan/ package.
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Bitpacker-streaming option bundles.
|
||||
*
|
||||
* The five generic streaming primitives — `TransformProgress`,
|
||||
* `FramingConfig`, `FrameParseResult`, `StreamTransformer`,
|
||||
* `PipelineStage` — live in `@suaveplan/types` so any data-pipeline
|
||||
* package can speak the same vocabulary. Only the bitpack/unpack-flavored
|
||||
* options stay here.
|
||||
*/
|
||||
|
||||
import type {
|
||||
FrameParseResult,
|
||||
FramingConfig,
|
||||
PipelineStage,
|
||||
StreamTransformer,
|
||||
TransformProgress,
|
||||
} from "@suaveplan/types";
|
||||
|
||||
export type {
|
||||
FrameParseResult,
|
||||
FramingConfig,
|
||||
PipelineStage,
|
||||
StreamTransformer,
|
||||
TransformProgress,
|
||||
};
|
||||
|
||||
/**
|
||||
* Legacy alias kept so downstream code that imports
|
||||
* `BitPackerTransformer` keeps compiling. New code should import
|
||||
* `StreamTransformer` from `@suaveplan/types` directly.
|
||||
*
|
||||
* @deprecated Use `StreamTransformer` from `@suaveplan/types`.
|
||||
*/
|
||||
export type BitPackerTransformer<TIn, TOut> = StreamTransformer<TIn, TOut>;
|
||||
|
||||
export interface PackTransformOptions {
|
||||
/** Number of items to pack per emitted chunk. Default `1`. */
|
||||
batchSize?: number;
|
||||
/** WHATWG TransformStream high-water mark. Default `16`. */
|
||||
highWaterMark?: number;
|
||||
/** Optional progress callback. */
|
||||
onProgress?: (progress: TransformProgress) => void;
|
||||
}
|
||||
|
||||
export interface UnpackTransformOptions {
|
||||
/** WHATWG TransformStream high-water mark. Default `16`. */
|
||||
highWaterMark?: number;
|
||||
/** Optional progress callback. */
|
||||
onProgress?: (progress: TransformProgress) => void;
|
||||
/**
|
||||
* Length-prefix width when `createUnpackTransformStream` infers a
|
||||
* `length-prefix` framing config. Default `4`.
|
||||
*/
|
||||
headerSize?: number;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/bitpacker",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"tslib",
|
||||
"zod",
|
||||
"@opentelemetry/api",
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/types",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user