Initial Commit
This commit is contained in:
100
packages/foundation/flow/streams-chunk-queue/README.md
Normal file
100
packages/foundation/flow/streams-chunk-queue/README.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# @suaveplan/streams-chunk-queue - Universal out-of-order chunk reconstitution: pluggable validation, decryption, rate limiting, storage, hooks, events, binary protocol, ReadableStream output
|
||||
|
||||
Universal out-of-order chunk reconstitution. Accepts chunks for a
|
||||
file in any order and exposes a sequential
|
||||
`ReadableStream<Uint8Array>` consumer surface, composed with pluggable
|
||||
validation, decryption, rate limiting, persistent storage, lifecycle
|
||||
hooks, a wildcard event bus, an efficient binary chunk-request
|
||||
protocol, missing-chunk timeouts, and OpenTelemetry instrumentation.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [binary-protocol](./src/binary-protocol/encoder.md) — Produces the chunk-request wire frame consumed by `@suaveplan/drive-mesh`, `drive-transport-https`, and any peer that n…
|
||||
- [encoder](./src/binary-protocol/encoder.md) — Produces the chunk-request wire frame consumed by `@suaveplan/drive-mesh`, `drive-transport-https`, and any peer that n…
|
||||
- [decryptor](./src/decryptor/decryptor.md) — Declares the `ChunkDecryptor` strategy interface and provides the `NoopChunkDecryptor` default. Decryption is the third…
|
||||
- [decryptor](./src/decryptor/decryptor.md) — Declares the `ChunkDecryptor` strategy interface and provides the `NoopChunkDecryptor` default. Decryption is the third…
|
||||
- [errors](./src/errors/streams-chunk-queue-error.md) — `StreamsChunkQueueError` is the single typed error class raised by every public surface of `@suaveplan/streams-chunk-qu…
|
||||
- [streams-chunk-queue-error](./src/errors/streams-chunk-queue-error.md) — `StreamsChunkQueueError` is the single typed error class raised by every public surface of `@suaveplan/streams-chunk-qu…
|
||||
- [events](./src/events/events.md) — Implements the lightweight event bus that powers the queue's `addEventListener` / `removeEventListener` surface. The em…
|
||||
- [events](./src/events/events.md) — Implements the lightweight event bus that powers the queue's `addEventListener` / `removeEventListener` surface. The em…
|
||||
- [hooks](./src/hooks/hooks.md) — Provides the helper functions that invoke the queue's eight lifecycle callbacks safely. The queue itself wires hook fir…
|
||||
- [hooks](./src/hooks/hooks.md) — Provides the helper functions that invoke the queue's eight lifecycle callbacks safely. The queue itself wires hook fir…
|
||||
- [pipeline](./src/queue/pipeline.md) — Houses the validation + decryption pipeline stages that the `StreamingChunkQueue` runs on every received chunk. Extract…
|
||||
- [storage-ops](./src/queue/storage-ops.md) — Holds the storage-related guards used by `StreamingChunkQueue.loadChunk` and `loadAllChunks`. Both methods need the sam…
|
||||
- [streaming-chunk-queue](./src/queue/streaming-chunk-queue.md) — `StreamingChunkQueue` is the public class that reconstitutes out-of-order chunked file transfers into a sequential `Rea…
|
||||
- [rate-limiter](./src/rate-limiter/adapter.md) — Normalises the `@suaveplan/rate-limiter` `TokenBucket` API onto the `ChunkRateLimiter` interface used internall…
|
||||
- [adapter](./src/rate-limiter/adapter.md) — Normalises the `@suaveplan/rate-limiter` `TokenBucket` API onto the `ChunkRateLimiter` interface used internall…
|
||||
- [storage-adapter](./src/storage-adapter/persister.md) — Bridges `@suaveplan/storage` `AbstractStorage` onto the queue's `ChunkPersister` contract. Persistence enables resuming…
|
||||
- [persister](./src/storage-adapter/persister.md) — Bridges `@suaveplan/storage` `AbstractStorage` onto the queue's `ChunkPersister` contract. Persistence enables resuming…
|
||||
- [telemetry](./src/telemetry/telemetry.md) — Exposes the OpenTelemetry instruments that the `StreamingChunkQueue` records during its lifecycle. Centralising instrum…
|
||||
- [telemetry](./src/telemetry/telemetry.md) — Exposes the OpenTelemetry instruments that the `StreamingChunkQueue` records during its lifecycle. Centralising instrum…
|
||||
- [types](./src/types/types.md) — Declares the shared types consumed across every feature dir of `@suaveplan/streams-chunk-queue`. Keeping them in one ty…
|
||||
- [types](./src/types/types.md) — Declares the shared types consumed across every feature dir of `@suaveplan/streams-chunk-queue`. Keeping them in one ty…
|
||||
- [validator](./src/validator/validator.md) — Declares the `ChunkValidator` strategy interface and provides the `NoopChunkValidator` default. Validation is the secon…
|
||||
- [chunk-verifier-validator](./src/validator/chunk-verifier-validator.md) — Bridges the structural `ChunkVerifier` shape from `@suaveplan/packer` into the canonical `ChunkValidator` strategy. The…
|
||||
- [validator](./src/validator/validator.md) — Declares the `ChunkValidator` strategy interface and provides the `NoopChunkValidator` default. Validation is the secon…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/streams-chunk-queue
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```ts
|
||||
import { StreamingChunkQueue } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 12345n,
|
||||
totalChunks: 5n,
|
||||
});
|
||||
|
||||
await queue.addChunk(2n, chunk2);
|
||||
await queue.addChunk(0n, chunk0);
|
||||
await queue.addChunk(1n, chunk1);
|
||||
await queue.addChunk(3n, chunk3);
|
||||
await queue.addChunk(4n, chunk4);
|
||||
|
||||
await queue.getStream().pipeTo(destination);
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- [`src/queue/streaming-chunk-queue.md`](./src/queue/streaming-chunk-queue.md) — the public `StreamingChunkQueue` class.
|
||||
- [`src/queue/pipeline.md`](./src/queue/pipeline.md) — validation + decryption stages.
|
||||
- [`src/queue/storage-ops.md`](./src/queue/storage-ops.md) — storage-side guards.
|
||||
- [`src/validator/validator.md`](./src/validator/validator.md) — `ChunkValidator` strategy + `NoopChunkValidator`.
|
||||
- [`src/validator/chunk-verifier-validator.md`](./src/validator/chunk-verifier-validator.md) — adapter for `@suaveplan/packer` `ChunkVerifier`.
|
||||
- [`src/decryptor/decryptor.md`](./src/decryptor/decryptor.md) — `ChunkDecryptor` strategy + `NoopChunkDecryptor`.
|
||||
- [`src/rate-limiter/adapter.md`](./src/rate-limiter/adapter.md) — `TokenBucket` adapter onto the queue's `ChunkRateLimiter`.
|
||||
- [`src/storage-adapter/persister.md`](./src/storage-adapter/persister.md) — `StorageChunkPersister` over `AbstractStorage`.
|
||||
- [`src/binary-protocol/encoder.md`](./src/binary-protocol/encoder.md) — chunk-request wire format.
|
||||
- [`src/hooks/hooks.md`](./src/hooks/hooks.md) — safe lifecycle-hook invocation.
|
||||
- [`src/events/events.md`](./src/events/events.md) — wildcard-capable event emitter.
|
||||
- [`src/telemetry/telemetry.md`](./src/telemetry/telemetry.md) — OTel counters + histogram.
|
||||
- [`src/errors/streams-chunk-queue-error.md`](./src/errors/streams-chunk-queue-error.md) — error codes + class.
|
||||
- [`src/types/types.md`](./src/types/types.md) — shared types.
|
||||
|
||||
## Features
|
||||
|
||||
- Out-of-order ingestion; duplicates silently ignored.
|
||||
- Sequential `ReadableStream<Uint8Array>` output.
|
||||
- Auto-start on first chunk.
|
||||
- Pluggable validator + decryptor strategies with packer `ChunkVerifier` adapter.
|
||||
- Non-destructive rate-limit backoff via `TokenBucket` adapter.
|
||||
- Resumable transfers via `StorageChunkPersister` over `AbstractStorage`.
|
||||
- Eight lifecycle hooks, wildcard event bus, 10 event types.
|
||||
- Binary chunk-request protocol (12-byte header + bitmap).
|
||||
- Configurable missing-chunk timeout (default 30 s).
|
||||
- Progress events (0–100) + frozen `StreamingQueueStatus` snapshot.
|
||||
- Bigint chunk identifiers (packer v2 u64).
|
||||
- `StreamsChunkQueueError` with 8 stable codes.
|
||||
- OTel: 5 counters + 1 histogram.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
103
packages/foundation/flow/streams-chunk-queue/package.json
Normal file
103
packages/foundation/flow/streams-chunk-queue/package.json
Normal file
@@ -0,0 +1,103 @@
|
||||
{
|
||||
"name": "@suaveplan/streams-chunk-queue",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Universal out-of-order chunk reconstitution: pluggable validation, decryption, rate limiting, storage, hooks, events, binary protocol, ReadableStream output",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
|
||||
"require": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }
|
||||
},
|
||||
"./errors": {
|
||||
"import": { "types": "./dist/errors/streams-chunk-queue-error.d.ts", "default": "./dist/errors/streams-chunk-queue-error.js" },
|
||||
"require": { "types": "./dist/errors/streams-chunk-queue-error.d.ts", "default": "./dist/errors/streams-chunk-queue-error.js" }
|
||||
},
|
||||
"./types": {
|
||||
"import": { "types": "./dist/types/types.d.ts", "default": "./dist/types/types.js" },
|
||||
"require": { "types": "./dist/types/types.d.ts", "default": "./dist/types/types.js" }
|
||||
},
|
||||
"./queue/streaming-chunk-queue": {
|
||||
"import": { "types": "./dist/queue/streaming-chunk-queue.d.ts", "default": "./dist/queue/streaming-chunk-queue.js" },
|
||||
"require": { "types": "./dist/queue/streaming-chunk-queue.d.ts", "default": "./dist/queue/streaming-chunk-queue.js" }
|
||||
},
|
||||
"./validator/validator": {
|
||||
"import": { "types": "./dist/validator/validator.d.ts", "default": "./dist/validator/validator.js" },
|
||||
"require": { "types": "./dist/validator/validator.d.ts", "default": "./dist/validator/validator.js" }
|
||||
},
|
||||
"./validator/chunk-verifier-validator": {
|
||||
"import": { "types": "./dist/validator/chunk-verifier-validator.d.ts", "default": "./dist/validator/chunk-verifier-validator.js" },
|
||||
"require": { "types": "./dist/validator/chunk-verifier-validator.d.ts", "default": "./dist/validator/chunk-verifier-validator.js" }
|
||||
},
|
||||
"./decryptor/decryptor": {
|
||||
"import": { "types": "./dist/decryptor/decryptor.d.ts", "default": "./dist/decryptor/decryptor.js" },
|
||||
"require": { "types": "./dist/decryptor/decryptor.d.ts", "default": "./dist/decryptor/decryptor.js" }
|
||||
},
|
||||
"./rate-limiter/adapter": {
|
||||
"import": { "types": "./dist/rate-limiter/adapter.d.ts", "default": "./dist/rate-limiter/adapter.js" },
|
||||
"require": { "types": "./dist/rate-limiter/adapter.d.ts", "default": "./dist/rate-limiter/adapter.js" }
|
||||
},
|
||||
"./storage-adapter/persister": {
|
||||
"import": { "types": "./dist/storage-adapter/persister.d.ts", "default": "./dist/storage-adapter/persister.js" },
|
||||
"require": { "types": "./dist/storage-adapter/persister.d.ts", "default": "./dist/storage-adapter/persister.js" }
|
||||
},
|
||||
"./binary-protocol/encoder": {
|
||||
"import": { "types": "./dist/binary-protocol/encoder.d.ts", "default": "./dist/binary-protocol/encoder.js" },
|
||||
"require": { "types": "./dist/binary-protocol/encoder.d.ts", "default": "./dist/binary-protocol/encoder.js" }
|
||||
},
|
||||
"./hooks/hooks": {
|
||||
"import": { "types": "./dist/hooks/hooks.d.ts", "default": "./dist/hooks/hooks.js" },
|
||||
"require": { "types": "./dist/hooks/hooks.d.ts", "default": "./dist/hooks/hooks.js" }
|
||||
},
|
||||
"./events/events": {
|
||||
"import": { "types": "./dist/events/events.d.ts", "default": "./dist/events/events.js" },
|
||||
"require": { "types": "./dist/events/events.d.ts", "default": "./dist/events/events.js" }
|
||||
},
|
||||
"./telemetry/telemetry": {
|
||||
"import": { "types": "./dist/telemetry/telemetry.d.ts", "default": "./dist/telemetry/telemetry.js" },
|
||||
"require": { "types": "./dist/telemetry/telemetry.d.ts", "default": "./dist/telemetry/telemetry.js" }
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/packer": "workspace:^",
|
||||
"@suaveplan/rate-limiter": "workspace:^",
|
||||
"@suaveplan/storage": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": ["dist", "src", "README.md"]
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
# binary-protocol / encoder
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Produces the chunk-request wire frame consumed by `@suaveplan/drive-mesh`,
|
||||
`drive-transport-https`, and any peer that needs to tell the sender which
|
||||
chunks of a file it is still missing. The frame is compact (12-byte
|
||||
header + bitmap) so even a 100k-chunk request stays under 13 kB, and it
|
||||
is fixed-layout so receivers can validate it without a schema.
|
||||
|
||||
## Features
|
||||
|
||||
- 8-byte big-endian `fileId` (u64).
|
||||
- 4-byte big-endian `totalChunks` (u32).
|
||||
- `⌈totalChunks / 8⌉` byte bitmap where bit `i` of byte `floor(i/8)` is
|
||||
set iff chunk `i` is requested.
|
||||
- Zero-allocation hot path: a single `Uint8Array` of the exact size and
|
||||
one DataView for the header.
|
||||
- Range checks raise `StreamsChunkQueueError` with the matching code
|
||||
(`INVALID_CHUNK_ID` for fileId, `INVALID_TOTAL_CHUNKS` for
|
||||
totalChunks).
|
||||
- Chunk ids outside `[0, totalChunks)` are silently dropped — the
|
||||
encoder's contract is "request these chunks, capped by the file
|
||||
size".
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { encodeChunkRequest } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const frame = encodeChunkRequest({
|
||||
fileId: 12345n,
|
||||
totalChunks: 10n,
|
||||
chunkIds: [0n, 2n, 5n],
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
const missing = queue.getMissingChunkIds();
|
||||
const frame = encodeChunkRequest({
|
||||
fileId: queue.getStatus().fileId,
|
||||
totalChunks: queue.getStatus().totalChunks,
|
||||
chunkIds: missing,
|
||||
});
|
||||
await webrtcDc.send(frame);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `encodeChunkRequest({ fileId, totalChunks, chunkIds })` → `Uint8Array`
|
||||
|
||||
Returns the encoded frame. Throws `StreamsChunkQueueError` when
|
||||
`fileId` is outside the u64 range or `totalChunks` is outside the u32
|
||||
range.
|
||||
|
||||
### `CHUNK_REQUEST_HEADER_SIZE`
|
||||
|
||||
Constant `12`. Exposed for callers that decode the frame.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The encoder iterates over `chunkIds` (typically the missing set,
|
||||
bounded by realistic transfer sizes) instead of `[0, totalChunks)` so
|
||||
the hot path stays O(|chunkIds|) regardless of file size. The 8-byte
|
||||
big-endian `fileId` is written as two `setUint32` calls because
|
||||
`DataView` does not have a `setBigUint64` on every supported runtime
|
||||
target.
|
||||
@@ -0,0 +1,97 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { StreamsChunkQueueError } from "../errors/streams-chunk-queue-error.js";
|
||||
import { CHUNK_REQUEST_HEADER_SIZE, encodeChunkRequest } from "./encoder.js";
|
||||
|
||||
describe("encodeChunkRequest", () => {
|
||||
it("encodes file id, total chunks, and bitmap exactly", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bytes = encodeChunkRequest({
|
||||
fileId: 12345n,
|
||||
totalChunks: 10n,
|
||||
chunkIds: [0n, 2n, 5n],
|
||||
});
|
||||
expect(bytes.byteLength).toBe(CHUNK_REQUEST_HEADER_SIZE + 2);
|
||||
expect(Array.from(bytes.slice(0, 8))).toEqual([
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x39,
|
||||
]);
|
||||
expect(Array.from(bytes.slice(8, 12))).toEqual([
|
||||
0x00, 0x00, 0x00, 0x0a,
|
||||
]);
|
||||
expect(bytes[12]).toBe(0b00100101);
|
||||
expect(bytes[13]).toBe(0x00);
|
||||
});
|
||||
|
||||
it("encodes chunk ids spanning multiple bitmap bytes", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bytes = encodeChunkRequest({
|
||||
fileId: 1n,
|
||||
totalChunks: 17n,
|
||||
chunkIds: [0n, 8n, 16n],
|
||||
});
|
||||
expect(bytes.byteLength).toBe(CHUNK_REQUEST_HEADER_SIZE + 3);
|
||||
expect(bytes[12]).toBe(0b00000001);
|
||||
expect(bytes[13]).toBe(0b00000001);
|
||||
expect(bytes[14]).toBe(0b00000001);
|
||||
});
|
||||
|
||||
it("emits an empty bitmap when no chunks are requested", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bytes = encodeChunkRequest({
|
||||
fileId: 1n,
|
||||
totalChunks: 8n,
|
||||
chunkIds: [],
|
||||
});
|
||||
expect(bytes[12]).toBe(0x00);
|
||||
});
|
||||
|
||||
it("ignores chunk ids outside the totalChunks range", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bytes = encodeChunkRequest({
|
||||
fileId: 1n,
|
||||
totalChunks: 4n,
|
||||
chunkIds: [99n, -1n],
|
||||
});
|
||||
expect(bytes[12]).toBe(0x00);
|
||||
});
|
||||
|
||||
it("rejects fileId outside the u64 range", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
encodeChunkRequest({
|
||||
fileId: -1n,
|
||||
totalChunks: 1n,
|
||||
chunkIds: [],
|
||||
})
|
||||
).toThrow(StreamsChunkQueueError);
|
||||
});
|
||||
|
||||
it("rejects totalChunks outside the u32 range", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
encodeChunkRequest({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n << 33n,
|
||||
chunkIds: [],
|
||||
})
|
||||
).toThrow(StreamsChunkQueueError);
|
||||
expect(() =>
|
||||
encodeChunkRequest({
|
||||
fileId: 1n,
|
||||
totalChunks: -1n,
|
||||
chunkIds: [],
|
||||
})
|
||||
).toThrow(StreamsChunkQueueError);
|
||||
});
|
||||
|
||||
it("rejects fileId above the u64 maximum", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
encodeChunkRequest({
|
||||
fileId: (1n << 64n) + 1n,
|
||||
totalChunks: 1n,
|
||||
chunkIds: [],
|
||||
})
|
||||
).toThrow(StreamsChunkQueueError);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import type { ChunkRequestEncodeInput } from "./encoder.types.js";
|
||||
|
||||
/** Header size: 8 bytes fileId (u64) + 4 bytes totalChunks (u32). */
|
||||
export const CHUNK_REQUEST_HEADER_SIZE = 12;
|
||||
|
||||
const U32_MASK = 0xffffffffn;
|
||||
const U32_MAX = 0xffffffffn;
|
||||
const U64_MAX = 0xffffffffffffffffn;
|
||||
|
||||
/**
|
||||
* Encode a chunk-request frame. Layout:
|
||||
*
|
||||
* | Offset | Size | Type | Description |
|
||||
* |--------|------|--------|-----------------------------|
|
||||
* | 0 | 8 | u64 BE | File ID |
|
||||
* | 8 | 4 | u32 BE | Total chunks |
|
||||
* | 12 | ⌈N/8⌉| bitmap | Bit i = 1 iff chunk i req'd |
|
||||
*/
|
||||
export function encodeChunkRequest(input: ChunkRequestEncodeInput): Uint8Array {
|
||||
if (input.fileId < 0n || input.fileId > U64_MAX) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_CHUNK_ID,
|
||||
"fileId is outside the u64 range",
|
||||
{ context: { fileId: input.fileId.toString() } }
|
||||
);
|
||||
}
|
||||
if (input.totalChunks < 0n || input.totalChunks > U32_MAX) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_TOTAL_CHUNKS,
|
||||
"totalChunks is outside the u32 range",
|
||||
{ context: { totalChunks: input.totalChunks.toString() } }
|
||||
);
|
||||
}
|
||||
const totalChunksNum = Number(input.totalChunks);
|
||||
const bitmapBytes = Math.ceil(totalChunksNum / 8);
|
||||
const buffer = new Uint8Array(CHUNK_REQUEST_HEADER_SIZE + bitmapBytes);
|
||||
const view = new DataView(buffer.buffer);
|
||||
|
||||
const fileIdHi = Number((input.fileId >> 32n) & U32_MASK) >>> 0;
|
||||
const fileIdLo = Number(input.fileId & U32_MASK) >>> 0;
|
||||
view.setUint32(0, fileIdHi, false);
|
||||
view.setUint32(4, fileIdLo, false);
|
||||
view.setUint32(8, totalChunksNum, false);
|
||||
|
||||
for (const chunkId of input.chunkIds) {
|
||||
if (chunkId < 0n || chunkId >= input.totalChunks) continue;
|
||||
const idx = Number(chunkId);
|
||||
const byteIndex = CHUNK_REQUEST_HEADER_SIZE + (idx >>> 3);
|
||||
const bitIndex = idx & 7;
|
||||
// DataView.setUint8 is bounds-checked at the runtime level and avoids
|
||||
// the `Uint8Array` narrowing branch that noUncheckedIndexedAccess otherwise forces.
|
||||
view.setUint8(byteIndex, view.getUint8(byteIndex) | (1 << bitIndex));
|
||||
}
|
||||
|
||||
return buffer;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
/** Input to the chunk-request encoder. */
|
||||
export interface ChunkRequestEncodeInput {
|
||||
readonly fileId: bigint;
|
||||
readonly totalChunks: bigint;
|
||||
readonly chunkIds: readonly bigint[];
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { CHUNK_REQUEST_HEADER_SIZE, encodeChunkRequest } from "./encoder.js";
|
||||
export type { ChunkRequestEncodeInput } from "./encoder.types.js";
|
||||
@@ -0,0 +1,91 @@
|
||||
# decryptor
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the `ChunkDecryptor` strategy interface and provides the
|
||||
`NoopChunkDecryptor` default. Decryption is the third stage of the
|
||||
chunk pipeline (after validation, before streaming). Externalising the
|
||||
strategy lets callers plug in AEAD primitives (XChaCha20-Poly1305,
|
||||
AES-256-GCM), envelope decryption, or HSM-bound decryption without the
|
||||
core queue knowing the cryptographic details.
|
||||
|
||||
## Features
|
||||
|
||||
- `ChunkDecryptor` interface with a single async `decrypt` method.
|
||||
- `NoopChunkDecryptor` pass-through default — used for unencrypted
|
||||
transports.
|
||||
- `noopChunkDecryptor` singleton shorthand.
|
||||
- Failures are caught by the queue and surfaced as
|
||||
`StreamsChunkQueueError(DECRYPTION_FAILED)`, never an opaque rejection.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { noopChunkDecryptor } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
decryptor: noopChunkDecryptor,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type { ChunkDecryptor } from "@suaveplan/streams-chunk-queue";
|
||||
import { chacha20poly1305 } from "@noble/ciphers/chacha";
|
||||
|
||||
class XChaChaDecryptor implements ChunkDecryptor {
|
||||
constructor(private readonly key: Uint8Array) {}
|
||||
|
||||
async decrypt(_chunkId: bigint, data: Uint8Array) {
|
||||
const nonce = data.slice(0, 24);
|
||||
const ciphertext = data.slice(24);
|
||||
const plaintext = chacha20poly1305(this.key, nonce).decrypt(ciphertext);
|
||||
return { data: plaintext };
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `ChunkDecryptor`
|
||||
|
||||
```ts
|
||||
interface ChunkDecryptor {
|
||||
decrypt(chunkId: bigint, data: Uint8Array): Promise<DecryptionOutcome>;
|
||||
}
|
||||
```
|
||||
|
||||
### `DecryptionOutcome`
|
||||
|
||||
`{ data: Uint8Array }` — plaintext bytes that replace the ciphertext
|
||||
in the queue's storage.
|
||||
|
||||
### `NoopChunkDecryptor`
|
||||
|
||||
Concrete class returning `{ data }` unchanged. Suitable for plaintext
|
||||
transports or when decryption happens further upstream.
|
||||
|
||||
### `noopChunkDecryptor`
|
||||
|
||||
Singleton instance of `NoopChunkDecryptor`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The interface is async because real AEAD primitives invariably are.
|
||||
Errors thrown by `decrypt` are caught by the queue and remapped to the
|
||||
canonical `DECRYPTION_FAILED` code so downstream observability code can
|
||||
filter on a stable code instead of a library-specific error class. The
|
||||
adapter contract returns `{ data }` so a decryptor can also concatenate
|
||||
multiple ciphertext fragments, strip a nonce prefix, or replace the
|
||||
bytes wholesale — the queue replaces its in-memory slot with the
|
||||
returned bytes, freeing the original ciphertext for garbage
|
||||
collection. The noop default lets the queue avoid a conditional check
|
||||
on every chunk; it returns the same `Uint8Array` reference unchanged,
|
||||
so there is no allocation cost.
|
||||
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NoopChunkDecryptor, noopChunkDecryptor } from "./decryptor.js";
|
||||
|
||||
describe("NoopChunkDecryptor", () => {
|
||||
it("returns the input bytes unchanged", { timeout: 5000 }, async () => {
|
||||
const decryptor = new NoopChunkDecryptor();
|
||||
const data = new Uint8Array([4, 5, 6]);
|
||||
const result = await decryptor.decrypt(1n, data);
|
||||
expect(result.data).toBe(data);
|
||||
});
|
||||
|
||||
it("exposes a singleton shorthand", { timeout: 5000 }, async () => {
|
||||
const result = await noopChunkDecryptor.decrypt(0n, new Uint8Array(0));
|
||||
expect(result.data.byteLength).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { ChunkDecryptor, DecryptionOutcome } from "./decryptor.types.js";
|
||||
|
||||
/** Default decryptor that returns the ciphertext bytes unchanged. */
|
||||
export class NoopChunkDecryptor implements ChunkDecryptor {
|
||||
public decrypt(
|
||||
_chunkId: bigint,
|
||||
data: Uint8Array
|
||||
): Promise<DecryptionOutcome> {
|
||||
return Promise.resolve({ data });
|
||||
}
|
||||
}
|
||||
|
||||
/** Singleton shorthand. */
|
||||
export const noopChunkDecryptor: ChunkDecryptor = new NoopChunkDecryptor();
|
||||
@@ -0,0 +1,13 @@
|
||||
/** Outcome of a chunk-decryption run: plaintext bytes. */
|
||||
export interface DecryptionOutcome {
|
||||
readonly data: Uint8Array;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy interface for chunk decryption. Implementations MUST throw
|
||||
* on failure; the queue coerces the failure into
|
||||
* `StreamsChunkQueueError(DECRYPTION_FAILED)`.
|
||||
*/
|
||||
export interface ChunkDecryptor {
|
||||
decrypt(chunkId: bigint, data: Uint8Array): Promise<DecryptionOutcome>;
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export { NoopChunkDecryptor, noopChunkDecryptor } from "./decryptor.js";
|
||||
export type { ChunkDecryptor, DecryptionOutcome } from "./decryptor.types.js";
|
||||
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
type StreamsChunkQueueErrorCode,
|
||||
type StreamsChunkQueueErrorContext,
|
||||
type StreamsChunkQueueErrorOptions,
|
||||
} from "./streams-chunk-queue-error.js";
|
||||
@@ -0,0 +1,97 @@
|
||||
# streams-chunk-queue-error
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
`StreamsChunkQueueError` is the single typed error class raised by every
|
||||
public surface of `@suaveplan/streams-chunk-queue`. It extends
|
||||
`SuaveplanError` so callers get the full enterprise error contract: a
|
||||
stable `code` string, a structured `context` map, a redactable wire form,
|
||||
OpenTelemetry span-event mapping, and an unbroken cause chain. The
|
||||
package registers every code through `defineCodes`, guaranteeing the
|
||||
codes are immutable, frozen, and reachable from real call sites.
|
||||
|
||||
## Features
|
||||
|
||||
- One frozen registry — `STREAMS_CHUNK_QUEUE_CODES` — exposes the eight
|
||||
canonical codes (`INVALID_TOTAL_CHUNKS`, `INVALID_CHUNK_ID`,
|
||||
`STORAGE_NOT_CONFIGURED`, `STORAGE_CHUNK_MISSING`, `VALIDATION_FAILED`,
|
||||
`DECRYPTION_FAILED`, `STREAM_TIMEOUT`, `RATE_LIMIT_EXHAUSTED`).
|
||||
- One literal-typed error class — `StreamsChunkQueueError` — narrows
|
||||
`code` through `instanceof`.
|
||||
- Cause chaining via `SuaveplanError`'s `{ cause }` option, so the
|
||||
underlying network or crypto failure is preserved.
|
||||
- Structured `context` map (e.g. `{ chunkId, fileId, totalChunks }`) for
|
||||
observability and i18n.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_CHUNK_ID,
|
||||
`chunk ${chunkId} out of range`,
|
||||
{ context: { chunkId: chunkId.toString() } },
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
try {
|
||||
await queue.addChunk(chunkId, bytes);
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof StreamsChunkQueueError &&
|
||||
err.code === STREAMS_CHUNK_QUEUE_CODES.VALIDATION_FAILED
|
||||
) {
|
||||
observabilityBus.publish("chunk-fraud", {
|
||||
chunkId: err.context?.chunkId,
|
||||
cause: err.cause,
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `STREAMS_CHUNK_QUEUE_CODES`
|
||||
|
||||
A frozen record built by `defineCodes("STREAMS_CHUNK_QUEUE", […])`. Each
|
||||
key maps to the fully-qualified code string
|
||||
`STREAMS_CHUNK_QUEUE.<reason>`.
|
||||
|
||||
### `StreamsChunkQueueErrorCode`
|
||||
|
||||
Discriminated union of every code in the registry.
|
||||
|
||||
### `StreamsChunkQueueError`
|
||||
|
||||
```ts
|
||||
new StreamsChunkQueueError(
|
||||
code: StreamsChunkQueueErrorCode,
|
||||
message: string,
|
||||
options?: { cause?: unknown; context?: Record<string, unknown> },
|
||||
)
|
||||
```
|
||||
|
||||
- `code` — one of the registered code strings.
|
||||
- `message` — human-readable description (or i18n message key).
|
||||
- `options.cause` — underlying error preserved for `walkCauses`.
|
||||
- `options.context` — structured fields surfaced through the wire schema.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`SuaveplanError` is `abstract`, so this module supplies the concrete
|
||||
subclass. The constructor delegates to `super` with the cause and context
|
||||
forwarded through `SuaveplanErrorOptions`. Codes are intentionally
|
||||
namespaced with `STREAMS_CHUNK_QUEUE.` so multi-package log scans can
|
||||
filter by prefix without false positives from sibling packages.
|
||||
@@ -0,0 +1,60 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "./streams-chunk-queue-error.js";
|
||||
|
||||
describe("StreamsChunkQueueError", () => {
|
||||
it("registers every documented STREAMS_CHUNK_QUEUE code", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.INVALID_TOTAL_CHUNKS).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.INVALID_TOTAL_CHUNKS"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.INVALID_CHUNK_ID).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.INVALID_CHUNK_ID"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.STORAGE_NOT_CONFIGURED).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.STORAGE_NOT_CONFIGURED"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.STORAGE_CHUNK_MISSING).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.STORAGE_CHUNK_MISSING"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.VALIDATION_FAILED).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.VALIDATION_FAILED"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.DECRYPTION_FAILED).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.DECRYPTION_FAILED"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.STREAM_TIMEOUT).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.STREAM_TIMEOUT"
|
||||
);
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES.RATE_LIMIT_EXHAUSTED).toBe(
|
||||
"STREAMS_CHUNK_QUEUE.RATE_LIMIT_EXHAUSTED"
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves code, message, cause, and context", { timeout: 5000 }, () => {
|
||||
const cause = new Error("boom");
|
||||
const err = new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STREAM_TIMEOUT,
|
||||
"timed out",
|
||||
{ cause, context: { chunkId: "1" } }
|
||||
);
|
||||
expect(err).toBeInstanceOf(StreamsChunkQueueError);
|
||||
expect(err.code).toBe("STREAMS_CHUNK_QUEUE.STREAM_TIMEOUT");
|
||||
expect(err.message).toBe("timed out");
|
||||
expect(err.cause).toBe(cause);
|
||||
expect(err.context).toEqual({ chunkId: "1" });
|
||||
});
|
||||
|
||||
it("defaults options when omitted", { timeout: 5000 }, () => {
|
||||
const err = new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_CHUNK_ID,
|
||||
"oob"
|
||||
);
|
||||
expect(err.code).toBe("STREAMS_CHUNK_QUEUE.INVALID_CHUNK_ID");
|
||||
expect(err.cause).toBeUndefined();
|
||||
expect(err.context).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { defineCodes, SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
/** Stable error codes for `@suaveplan/streams-chunk-queue`. */
|
||||
export const STREAMS_CHUNK_QUEUE_CODES = defineCodes("STREAMS_CHUNK_QUEUE", [
|
||||
"INVALID_TOTAL_CHUNKS",
|
||||
"INVALID_CHUNK_ID",
|
||||
"STORAGE_NOT_CONFIGURED",
|
||||
"STORAGE_CHUNK_MISSING",
|
||||
"VALIDATION_FAILED",
|
||||
"DECRYPTION_FAILED",
|
||||
"STREAM_TIMEOUT",
|
||||
"RATE_LIMIT_EXHAUSTED",
|
||||
] as const);
|
||||
|
||||
export type StreamsChunkQueueErrorCode =
|
||||
(typeof STREAMS_CHUNK_QUEUE_CODES)[keyof typeof STREAMS_CHUNK_QUEUE_CODES];
|
||||
|
||||
export interface StreamsChunkQueueErrorContext {
|
||||
readonly chunkId?: bigint;
|
||||
readonly fileId?: bigint;
|
||||
readonly totalChunks?: bigint;
|
||||
readonly cause?: unknown;
|
||||
readonly extra?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface StreamsChunkQueueErrorOptions {
|
||||
readonly cause?: unknown;
|
||||
readonly context?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Typed error raised by the streams-chunk-queue package. */
|
||||
export class StreamsChunkQueueError extends SuaveplanError<StreamsChunkQueueErrorCode> {
|
||||
public constructor(
|
||||
code: StreamsChunkQueueErrorCode,
|
||||
message: string,
|
||||
options: StreamsChunkQueueErrorOptions = {}
|
||||
) {
|
||||
super(code, message, {
|
||||
...(options.cause !== undefined ? { cause: options.cause } : {}),
|
||||
...(options.context !== undefined
|
||||
? { context: options.context }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
# events
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Implements the lightweight event bus that powers the queue's
|
||||
`addEventListener` / `removeEventListener` surface. The emitter keeps a
|
||||
`Map<StreamEventType, Set<Listener>>` so registration and removal are
|
||||
both O(1), and exposes a wildcard `"*"` subscription so debugging tools
|
||||
can observe every event without enumerating types.
|
||||
|
||||
## Features
|
||||
|
||||
- Concrete-type subscriptions (`chunk:received`, `stream:progress`, …).
|
||||
- Wildcard subscription via `"*"` that receives every emitted event.
|
||||
- O(1) add, remove, and emit per listener.
|
||||
- `clear()` for tear-down (used by `StreamingChunkQueue.clear()`).
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { StreamEventEmitter } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const emitter = new StreamEventEmitter();
|
||||
emitter.addEventListener("chunk:received", (event) => {
|
||||
console.info("got chunk", event.chunkId);
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
const wildcard = (event: StreamEvent) =>
|
||||
telemetry.record("stream.event", { type: event.type });
|
||||
emitter.addEventListener("*", wildcard);
|
||||
emitter.removeEventListener("*", wildcard);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `StreamEventEmitter`
|
||||
|
||||
- `addEventListener(type, listener)` — register `listener` under
|
||||
`type` (either a concrete event type or `"*"`).
|
||||
- `removeEventListener(type, listener)` — remove a previously
|
||||
registered listener.
|
||||
- `emit(event)` — dispatch to all matching subscribers, then to all
|
||||
wildcard subscribers.
|
||||
- `clear()` — drop every registered listener.
|
||||
|
||||
### `WILDCARD_EVENT_TYPE`
|
||||
|
||||
The constant `"*"`. Exported so callers don't sprinkle string literals.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The wildcard dispatch happens after concrete dispatch, mirroring the
|
||||
ordering observable consumers expect (a tracing tool sees the event
|
||||
after the application code has reacted). Listeners are synchronous —
|
||||
the bus does not await listener return values; callers that need
|
||||
async work should `queueMicrotask` inside their listener. The emitter
|
||||
is intentionally allocation-light: a `Map<type, Set<listener>>` means
|
||||
add/remove are O(1) and dispatch is O(N) over only the matching
|
||||
listeners. There is no priority ordering; concrete-type listeners are
|
||||
dispatched in insertion order, then wildcard listeners in their own
|
||||
insertion order.
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { ChunkMetadata, StreamEvent } from "../types/types.js";
|
||||
import { StreamEventEmitter, WILDCARD_EVENT_TYPE } from "./events.js";
|
||||
|
||||
const meta: ChunkMetadata = {
|
||||
fileId: 1n,
|
||||
chunkId: 0n,
|
||||
totalChunks: 1n,
|
||||
size: 0,
|
||||
receivedAt: 0,
|
||||
validated: false,
|
||||
decrypted: false,
|
||||
};
|
||||
|
||||
describe("StreamEventEmitter", () => {
|
||||
it("delivers events to wildcard and concrete listeners", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const emitter = new StreamEventEmitter();
|
||||
const wildcardEvents: StreamEvent[] = [];
|
||||
const concreteEvents: StreamEvent[] = [];
|
||||
emitter.addEventListener(WILDCARD_EVENT_TYPE, (e) =>
|
||||
wildcardEvents.push(e)
|
||||
);
|
||||
emitter.addEventListener("chunk:received", (e) =>
|
||||
concreteEvents.push(e)
|
||||
);
|
||||
const event: StreamEvent = {
|
||||
type: "chunk:received",
|
||||
chunkId: 0n,
|
||||
metadata: meta,
|
||||
};
|
||||
emitter.emit(event);
|
||||
expect(wildcardEvents).toEqual([event]);
|
||||
expect(concreteEvents).toEqual([event]);
|
||||
});
|
||||
|
||||
it("removes specific listeners without affecting siblings", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const emitter = new StreamEventEmitter();
|
||||
const a: StreamEvent[] = [];
|
||||
const b: StreamEvent[] = [];
|
||||
const lA = (e: StreamEvent) => a.push(e);
|
||||
const lB = (e: StreamEvent) => b.push(e);
|
||||
emitter.addEventListener("chunk:received", lA);
|
||||
emitter.addEventListener("chunk:received", lB);
|
||||
emitter.removeEventListener("chunk:received", lA);
|
||||
emitter.emit({ type: "chunk:received", chunkId: 0n, metadata: meta });
|
||||
expect(a.length).toBe(0);
|
||||
expect(b.length).toBe(1);
|
||||
});
|
||||
|
||||
it("removeEventListener is a no-op for unregistered types", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const emitter = new StreamEventEmitter();
|
||||
emitter.removeEventListener("stream:complete", () => {});
|
||||
});
|
||||
|
||||
it("emits when there are zero listeners", { timeout: 5000 }, () => {
|
||||
const emitter = new StreamEventEmitter();
|
||||
emitter.emit({ type: "stream:complete", totalChunks: 0n });
|
||||
});
|
||||
|
||||
it("clear() drops every registered listener", { timeout: 5000 }, () => {
|
||||
const emitter = new StreamEventEmitter();
|
||||
let count = 0;
|
||||
emitter.addEventListener("*", () => {
|
||||
count += 1;
|
||||
});
|
||||
emitter.clear();
|
||||
emitter.emit({ type: "stream:complete", totalChunks: 0n });
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
|
||||
it("notifies only wildcard listeners when no concrete subscriber exists", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const emitter = new StreamEventEmitter();
|
||||
const seen: StreamEvent[] = [];
|
||||
emitter.addEventListener("*", (e) => seen.push(e));
|
||||
emitter.emit({ type: "stream:complete", totalChunks: 1n });
|
||||
expect(seen.length).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
import type {
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
StreamEventType,
|
||||
} from "./events.types.js";
|
||||
|
||||
/** Event names that the wildcard `*` subscription receives. */
|
||||
export const WILDCARD_EVENT_TYPE = "*" as const;
|
||||
|
||||
/** Lightweight event emitter with wildcard support. */
|
||||
export class StreamEventEmitter {
|
||||
private readonly listeners: Map<StreamEventType, Set<StreamEventListener>> =
|
||||
new Map();
|
||||
|
||||
public addEventListener(
|
||||
type: StreamEventType,
|
||||
listener: StreamEventListener
|
||||
): void {
|
||||
let bucket = this.listeners.get(type);
|
||||
if (bucket === undefined) {
|
||||
bucket = new Set();
|
||||
this.listeners.set(type, bucket);
|
||||
}
|
||||
bucket.add(listener);
|
||||
}
|
||||
|
||||
public removeEventListener(
|
||||
type: StreamEventType,
|
||||
listener: StreamEventListener
|
||||
): void {
|
||||
this.listeners.get(type)?.delete(listener);
|
||||
}
|
||||
|
||||
public emit(event: StreamEvent): void {
|
||||
const concrete = this.listeners.get(event.type);
|
||||
if (concrete !== undefined) {
|
||||
for (const listener of concrete) listener(event);
|
||||
}
|
||||
const wildcard = this.listeners.get(WILDCARD_EVENT_TYPE);
|
||||
if (wildcard !== undefined) {
|
||||
for (const listener of wildcard) listener(event);
|
||||
}
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
this.listeners.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import type {
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
StreamEventType,
|
||||
} from "../types/types.js";
|
||||
|
||||
export type { StreamEvent, StreamEventListener, StreamEventType };
|
||||
@@ -0,0 +1,6 @@
|
||||
export { StreamEventEmitter, WILDCARD_EVENT_TYPE } from "./events.js";
|
||||
export type {
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
StreamEventType,
|
||||
} from "./events.types.js";
|
||||
@@ -0,0 +1,69 @@
|
||||
# hooks
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides the helper functions that invoke the queue's eight lifecycle
|
||||
callbacks safely. The queue itself wires hook firing into its pipeline;
|
||||
this module isolates the "safe invocation" logic — coercion of thrown
|
||||
values to real `Error` instances, async-await of both sync and async
|
||||
hooks, and re-routing of secondary failures through `onError`.
|
||||
|
||||
## Features
|
||||
|
||||
- `invokeHook` swallows hook errors after re-routing them via `onError`,
|
||||
so a misbehaving listener cannot crash the queue's main loop.
|
||||
- `toError` coerces any thrown value (string, object, primitive) into a
|
||||
real `Error` so downstream consumers see a uniform shape.
|
||||
- The helper avoids recursive re-routing when the failing hook IS the
|
||||
`onError` hook itself.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { invokeHook } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
await invokeHook(hooks.onChunkReceived, [chunkId, metadata], hooks.onError, chunkId);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { invokeHook, toError } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const onError = (err: Error) => logger.error({ err: toError(err) });
|
||||
|
||||
await invokeHook(
|
||||
async (id: bigint) => await reportToObservability(id),
|
||||
[chunkId],
|
||||
onError,
|
||||
chunkId,
|
||||
);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `invokeHook(hook, args, onError?, chunkId?)` → `Promise<void>`
|
||||
|
||||
- `hook` — optional `LifecycleHook<Args>`. If `undefined`, the call is
|
||||
a no-op.
|
||||
- `args` — tuple of arguments to forward.
|
||||
- `onError` — optional `(error, chunkId?) => void | Promise<void>`
|
||||
re-route. Not invoked if it is the same reference as `hook`.
|
||||
- `chunkId` — optional bigint forwarded to `onError`.
|
||||
|
||||
### `toError(value)` → `Error`
|
||||
|
||||
Coerces any value to an `Error`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The recursion guard `onError !== hook` is the difference between a
|
||||
queue that survives a buggy `onError` callback and one that hangs in an
|
||||
infinite loop. When the secondary `onError` invocation itself throws,
|
||||
the helper swallows the failure — the primary path is preserved
|
||||
because the queue independently surfaces the error via the event
|
||||
emitter and (for streaming errors) the controller.
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { invokeHook, toError } from "./hooks.js";
|
||||
|
||||
describe("toError", () => {
|
||||
it("returns Error instances unchanged", { timeout: 5000 }, () => {
|
||||
const e = new Error("boom");
|
||||
expect(toError(e)).toBe(e);
|
||||
});
|
||||
|
||||
it("wraps string throwables", { timeout: 5000 }, () => {
|
||||
const out = toError("blew up");
|
||||
expect(out).toBeInstanceOf(Error);
|
||||
expect(out.message).toBe("blew up");
|
||||
});
|
||||
|
||||
it("stringifies non-Error non-string throwables", { timeout: 5000 }, () => {
|
||||
const out = toError({ what: "oops" });
|
||||
expect(out).toBeInstanceOf(Error);
|
||||
expect(out.message).toBe("[object Object]");
|
||||
});
|
||||
});
|
||||
|
||||
describe("invokeHook", () => {
|
||||
it("is a no-op when no hook is registered", { timeout: 5000 }, async () => {
|
||||
await invokeHook(undefined, [], undefined);
|
||||
});
|
||||
|
||||
it("awaits async hooks", { timeout: 5000 }, async () => {
|
||||
let called = false;
|
||||
await invokeHook(async () => {
|
||||
await Promise.resolve();
|
||||
called = true;
|
||||
}, []);
|
||||
expect(called).toBe(true);
|
||||
});
|
||||
|
||||
it("safely invokes async and sync hooks and re-routes errors", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const seen: Array<[string, bigint | undefined]> = [];
|
||||
const onError = (e: Error, id?: bigint) => {
|
||||
seen.push([e.message, id]);
|
||||
};
|
||||
await invokeHook(
|
||||
() => {
|
||||
throw new Error("sync");
|
||||
},
|
||||
[],
|
||||
onError,
|
||||
1n
|
||||
);
|
||||
await invokeHook(
|
||||
async () => {
|
||||
throw "async";
|
||||
},
|
||||
[],
|
||||
onError,
|
||||
2n
|
||||
);
|
||||
expect(seen).toEqual([
|
||||
["sync", 1n],
|
||||
["async", 2n],
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not recurse if the failing hook IS the onError hook", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let onErrorCalls = 0;
|
||||
const onError = (_e: Error) => {
|
||||
onErrorCalls += 1;
|
||||
throw new Error("recursive");
|
||||
};
|
||||
await invokeHook(onError, [new Error("x"), undefined], onError);
|
||||
expect(onErrorCalls).toBe(1);
|
||||
});
|
||||
|
||||
it("swallows secondary failures inside onError", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const onError = () => {
|
||||
throw new Error("onError exploded");
|
||||
};
|
||||
await invokeHook(
|
||||
() => {
|
||||
throw new Error("primary");
|
||||
},
|
||||
[],
|
||||
onError,
|
||||
0n
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { StreamingChunkQueueHooks } from "../types/types.js";
|
||||
import type { LifecycleHook } from "./hooks.types.js";
|
||||
|
||||
/** Coerce any thrown / rejected value into a real `Error`. */
|
||||
export function toError(value: unknown): Error {
|
||||
if (value instanceof Error) return value;
|
||||
return new Error(typeof value === "string" ? value : String(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely invoke an optional hook. Both sync and async hooks are
|
||||
* awaited; a thrown error is re-routed via `onError` (if provided) and
|
||||
* swallowed afterwards so hook failures do not crash the queue's main
|
||||
* loop.
|
||||
*/
|
||||
export async function invokeHook<Args extends readonly unknown[]>(
|
||||
hook: LifecycleHook<Args> | undefined,
|
||||
args: Args,
|
||||
onError?: StreamingChunkQueueHooks["onError"],
|
||||
chunkId?: bigint
|
||||
): Promise<void> {
|
||||
if (hook === undefined) return;
|
||||
try {
|
||||
await hook(...args);
|
||||
} catch (err) {
|
||||
const error = toError(err);
|
||||
if (
|
||||
onError !== undefined &&
|
||||
(onError as unknown) !== (hook as unknown)
|
||||
) {
|
||||
try {
|
||||
await onError(error, chunkId);
|
||||
} catch {
|
||||
// onError itself failed — swallow.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
/** Generic shape of a queue lifecycle hook. */
|
||||
export type LifecycleHook<Args extends readonly unknown[]> = (
|
||||
...args: Args
|
||||
) => void | Promise<void>;
|
||||
@@ -0,0 +1,2 @@
|
||||
export { invokeHook, toError } from "./hooks.js";
|
||||
export type { LifecycleHook } from "./hooks.types.js";
|
||||
82
packages/foundation/flow/streams-chunk-queue/src/index.ts
Normal file
82
packages/foundation/flow/streams-chunk-queue/src/index.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
// Binary protocol
|
||||
export {
|
||||
CHUNK_REQUEST_HEADER_SIZE,
|
||||
encodeChunkRequest,
|
||||
} from "./binary-protocol/encoder.js";
|
||||
export type { ChunkRequestEncodeInput } from "./binary-protocol/encoder.types.js";
|
||||
// Decryptor
|
||||
export {
|
||||
NoopChunkDecryptor,
|
||||
noopChunkDecryptor,
|
||||
} from "./decryptor/decryptor.js";
|
||||
export type {
|
||||
ChunkDecryptor,
|
||||
DecryptionOutcome,
|
||||
} from "./decryptor/decryptor.types.js";
|
||||
// Errors
|
||||
export {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
type StreamsChunkQueueErrorCode,
|
||||
type StreamsChunkQueueErrorContext,
|
||||
type StreamsChunkQueueErrorOptions,
|
||||
} from "./errors/streams-chunk-queue-error.js";
|
||||
// Events
|
||||
export { StreamEventEmitter, WILDCARD_EVENT_TYPE } from "./events/events.js";
|
||||
// Hooks
|
||||
export { invokeHook, toError } from "./hooks/hooks.js";
|
||||
export type { LifecycleHook } from "./hooks/hooks.types.js";
|
||||
// Queue
|
||||
export { StreamingChunkQueue } from "./queue/streaming-chunk-queue.js";
|
||||
export type { StreamingChunkQueueOptions } from "./queue/streaming-chunk-queue.types.js";
|
||||
// Rate limiter
|
||||
export {
|
||||
DEFAULT_RATE_LIMIT_DELAY_MS,
|
||||
TokenBucketRateLimiterAdapter,
|
||||
} from "./rate-limiter/adapter.js";
|
||||
export type {
|
||||
ChunkRateLimiter,
|
||||
RateLimitConsumeOutcome,
|
||||
TokenBucketLike,
|
||||
} from "./rate-limiter/adapter.types.js";
|
||||
// Storage adapter
|
||||
export {
|
||||
chunkStorageKey,
|
||||
chunkStoragePrefix,
|
||||
StorageChunkPersister,
|
||||
} from "./storage-adapter/persister.js";
|
||||
export type {
|
||||
ChunkPersister,
|
||||
ChunkPersisterOptions,
|
||||
ChunkStorageLike,
|
||||
} from "./storage-adapter/persister.types.js";
|
||||
// Telemetry
|
||||
export {
|
||||
getTelemetry,
|
||||
resetTelemetryForTest,
|
||||
type StreamsChunkQueueTelemetry,
|
||||
} from "./telemetry/telemetry.js";
|
||||
// Types
|
||||
export type {
|
||||
ChunkMetadata,
|
||||
ChunkRequest,
|
||||
StoredChunk,
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
StreamEventType,
|
||||
StreamingChunkQueueHooks,
|
||||
StreamingQueueStatus,
|
||||
} from "./types/types.js";
|
||||
// Validator
|
||||
export {
|
||||
type ChunkVerifierLike,
|
||||
ChunkVerifierValidator,
|
||||
} from "./validator/chunk-verifier-validator.js";
|
||||
export {
|
||||
NoopChunkValidator,
|
||||
noopChunkValidator,
|
||||
} from "./validator/validator.js";
|
||||
export type {
|
||||
ChunkValidator,
|
||||
ValidationOutcome,
|
||||
} from "./validator/validator.types.js";
|
||||
@@ -0,0 +1,2 @@
|
||||
export { StreamingChunkQueue } from "./streaming-chunk-queue.js";
|
||||
export type { StreamingChunkQueueOptions } from "./streaming-chunk-queue.types.js";
|
||||
@@ -0,0 +1,73 @@
|
||||
# queue / pipeline
|
||||
|
||||
## Purpose
|
||||
|
||||
Houses the validation + decryption pipeline stages that the
|
||||
`StreamingChunkQueue` runs on every received chunk. Extracting these
|
||||
stages into pure helpers keeps the main queue class focused on
|
||||
orchestration (stream control, status tracking, request building) and
|
||||
lets the pipeline grow more stages — e.g. a future deduplication step —
|
||||
without ballooning the queue file past the per-file line budget.
|
||||
|
||||
## Features
|
||||
|
||||
- `runValidate` and `runDecrypt` share the same retry / event / hook /
|
||||
telemetry shape via the internal `runStage` helper.
|
||||
- Failures are coerced to `StreamsChunkQueueError` with the
|
||||
appropriate code (`VALIDATION_FAILED` / `DECRYPTION_FAILED`) and the
|
||||
original error is preserved as `cause`.
|
||||
- Successful stages mutate `metadata.validated` / `metadata.decrypted`
|
||||
through a callback so the queue's snapshot reflects pipeline state.
|
||||
- Telemetry counters (`validation_failures_total` /
|
||||
`decryption_failures_total`) are incremented on failure exactly once.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
The pipeline functions are internal to the queue package; the queue
|
||||
itself wires them.
|
||||
|
||||
```ts
|
||||
await runValidate(stored, validator, deps);
|
||||
await runDecrypt(stored, decryptor, deps);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
await runValidate(stored, this.validator, this.pipelineDeps);
|
||||
await this.runChecksum(stored);
|
||||
await runDecrypt(stored, this.decryptor, this.pipelineDeps);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `PipelineDeps`
|
||||
|
||||
```ts
|
||||
interface PipelineDeps {
|
||||
emitter: StreamEventEmitter;
|
||||
hooks: StreamingChunkQueueHooks;
|
||||
telemetry: StreamsChunkQueueTelemetry;
|
||||
}
|
||||
```
|
||||
|
||||
### `runValidate(stored, validator, deps)` → `Promise<void>`
|
||||
|
||||
Runs `validator.validate(chunkId, data)`. On success: replaces
|
||||
`stored.data`, sets `metadata.validated = true`, emits `chunk:validated`,
|
||||
and fires `onChunkValidated`. On failure: emits `validation:failed`,
|
||||
fires `onError`, and re-throws a `StreamsChunkQueueError(VALIDATION_FAILED)`.
|
||||
|
||||
### `runDecrypt(stored, decryptor, deps)` → `Promise<void>`
|
||||
|
||||
Same shape, using `decryptor.decrypt` and the
|
||||
`chunk:decrypted` / `decryption:failed` events and the
|
||||
`DECRYPTION_FAILED` code.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`runStage` re-throws an existing `StreamsChunkQueueError` untouched so a
|
||||
downstream adapter (e.g. `ChunkVerifierValidator`) can attach its own
|
||||
context and have that context survive the queue's catch handler. The
|
||||
`onError` re-route is suppressed when the failing hook is the same
|
||||
function — see `hooks.md` for the rationale.
|
||||
@@ -0,0 +1,156 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { ChunkDecryptor } from "../decryptor/decryptor.types.js";
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import { StreamEventEmitter } from "../events/events.js";
|
||||
import { getTelemetry } from "../telemetry/telemetry.js";
|
||||
import type {
|
||||
ChunkMetadata,
|
||||
StoredChunk,
|
||||
StreamEvent,
|
||||
} from "../types/types.js";
|
||||
import type { ChunkValidator } from "../validator/validator.types.js";
|
||||
import { type PipelineDeps, runDecrypt, runValidate } from "./pipeline.js";
|
||||
|
||||
function makeChunk(): StoredChunk {
|
||||
const metadata: ChunkMetadata = {
|
||||
fileId: 1n,
|
||||
chunkId: 0n,
|
||||
totalChunks: 1n,
|
||||
size: 2,
|
||||
receivedAt: 0,
|
||||
validated: false,
|
||||
decrypted: false,
|
||||
};
|
||||
return { data: new Uint8Array([1, 2]), metadata };
|
||||
}
|
||||
|
||||
function makeDeps(): { deps: PipelineDeps; events: StreamEvent[] } {
|
||||
const emitter = new StreamEventEmitter();
|
||||
const events: StreamEvent[] = [];
|
||||
emitter.addEventListener("*", (e) => events.push(e));
|
||||
return {
|
||||
events,
|
||||
deps: { emitter, hooks: {}, telemetry: getTelemetry() },
|
||||
};
|
||||
}
|
||||
|
||||
describe("runValidate", () => {
|
||||
it("flips metadata.validated and emits chunk:validated on success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps, events } = makeDeps();
|
||||
const validator: ChunkValidator = {
|
||||
validate: async (_id, d) => ({ data: d }),
|
||||
};
|
||||
await runValidate(chunk, validator, deps);
|
||||
expect(chunk.metadata.validated).toBe(true);
|
||||
expect(events.some((e) => e.type === "chunk:validated")).toBe(true);
|
||||
});
|
||||
|
||||
it("coerces a plain-string throwable into VALIDATION_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps } = makeDeps();
|
||||
const validator: ChunkValidator = {
|
||||
validate: async () => {
|
||||
throw "bad";
|
||||
},
|
||||
};
|
||||
try {
|
||||
await runValidate(chunk, validator, deps);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(STREAMS_CHUNK_QUEUE_CODES.VALIDATION_FAILED);
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves a thrown StreamsChunkQueueError unchanged", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps } = makeDeps();
|
||||
const original = new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.VALIDATION_FAILED,
|
||||
"explicit"
|
||||
);
|
||||
const validator: ChunkValidator = {
|
||||
validate: async () => {
|
||||
throw original;
|
||||
},
|
||||
};
|
||||
try {
|
||||
await runValidate(chunk, validator, deps);
|
||||
} catch (err) {
|
||||
expect(err).toBe(original);
|
||||
}
|
||||
});
|
||||
|
||||
it("invokes the onChunkValidated hook on success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps } = makeDeps();
|
||||
let called = false;
|
||||
deps.hooks.onChunkValidated = () => {
|
||||
called = true;
|
||||
};
|
||||
const validator: ChunkValidator = {
|
||||
validate: async (_id, d) => ({ data: d }),
|
||||
};
|
||||
await runValidate(chunk, validator, deps);
|
||||
expect(called).toBe(true);
|
||||
});
|
||||
|
||||
it("routes failure through onError when configured", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps } = makeDeps();
|
||||
const seen: Array<bigint | undefined> = [];
|
||||
deps.hooks.onError = (_err, id) => {
|
||||
seen.push(id);
|
||||
};
|
||||
const validator: ChunkValidator = {
|
||||
validate: async () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
runValidate(chunk, validator, deps)
|
||||
).rejects.toBeInstanceOf(StreamsChunkQueueError);
|
||||
expect(seen).toEqual([0n]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runDecrypt", () => {
|
||||
it("flips metadata.decrypted on success", { timeout: 5000 }, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps } = makeDeps();
|
||||
const decryptor: ChunkDecryptor = {
|
||||
decrypt: async (_id, d) => ({ data: d }),
|
||||
};
|
||||
await runDecrypt(chunk, decryptor, deps);
|
||||
expect(chunk.metadata.decrypted).toBe(true);
|
||||
});
|
||||
|
||||
it("maps failures to DECRYPTION_FAILED", { timeout: 5000 }, async () => {
|
||||
const chunk = makeChunk();
|
||||
const { deps } = makeDeps();
|
||||
const decryptor: ChunkDecryptor = {
|
||||
decrypt: async () => {
|
||||
throw new Error("key-rotation");
|
||||
},
|
||||
};
|
||||
try {
|
||||
await runDecrypt(chunk, decryptor, deps);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(STREAMS_CHUNK_QUEUE_CODES.DECRYPTION_FAILED);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
import type { ChunkDecryptor } from "../decryptor/decryptor.types.js";
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
type StreamsChunkQueueErrorCode,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import type { StreamEventEmitter } from "../events/events.js";
|
||||
import { invokeHook, toError } from "../hooks/hooks.js";
|
||||
import type { StreamsChunkQueueTelemetry } from "../telemetry/telemetry.js";
|
||||
import type {
|
||||
StoredChunk,
|
||||
StreamEvent,
|
||||
StreamingChunkQueueHooks,
|
||||
} from "../types/types.js";
|
||||
import type { ChunkValidator } from "../validator/validator.types.js";
|
||||
|
||||
/** Shared dependencies for the pipeline helpers. */
|
||||
export interface PipelineDeps {
|
||||
readonly emitter: StreamEventEmitter;
|
||||
readonly hooks: StreamingChunkQueueHooks;
|
||||
readonly telemetry: StreamsChunkQueueTelemetry;
|
||||
}
|
||||
|
||||
/** Run the validator stage. */
|
||||
export async function runValidate(
|
||||
stored: StoredChunk,
|
||||
validator: ChunkValidator,
|
||||
deps: PipelineDeps
|
||||
): Promise<void> {
|
||||
await runStage(
|
||||
stored,
|
||||
(id, data) => validator.validate(id, data),
|
||||
"chunk:validated",
|
||||
"validation:failed",
|
||||
STREAMS_CHUNK_QUEUE_CODES.VALIDATION_FAILED,
|
||||
deps,
|
||||
deps.hooks.onChunkValidated,
|
||||
(c) => {
|
||||
c.metadata.validated = true;
|
||||
},
|
||||
deps.telemetry.validationFailures
|
||||
);
|
||||
}
|
||||
|
||||
/** Run the decryptor stage. */
|
||||
export async function runDecrypt(
|
||||
stored: StoredChunk,
|
||||
decryptor: ChunkDecryptor,
|
||||
deps: PipelineDeps
|
||||
): Promise<void> {
|
||||
await runStage(
|
||||
stored,
|
||||
(id, data) => decryptor.decrypt(id, data),
|
||||
"chunk:decrypted",
|
||||
"decryption:failed",
|
||||
STREAMS_CHUNK_QUEUE_CODES.DECRYPTION_FAILED,
|
||||
deps,
|
||||
deps.hooks.onChunkDecrypted,
|
||||
(c) => {
|
||||
c.metadata.decrypted = true;
|
||||
},
|
||||
deps.telemetry.decryptionFailures
|
||||
);
|
||||
}
|
||||
|
||||
type StageFn = (
|
||||
chunkId: bigint,
|
||||
data: Uint8Array
|
||||
) => Promise<{ readonly data: Uint8Array }>;
|
||||
|
||||
type SuccessEvent = Extract<
|
||||
StreamEvent,
|
||||
{ type: "chunk:validated" | "chunk:decrypted" }
|
||||
>;
|
||||
type FailureEvent = Extract<
|
||||
StreamEvent,
|
||||
{ type: "validation:failed" | "decryption:failed" }
|
||||
>;
|
||||
|
||||
async function runStage(
|
||||
stored: StoredChunk,
|
||||
run: StageFn,
|
||||
successType: SuccessEvent["type"],
|
||||
failureType: FailureEvent["type"],
|
||||
failureCode: StreamsChunkQueueErrorCode,
|
||||
deps: PipelineDeps,
|
||||
onSuccess: StreamingChunkQueueHooks["onChunkValidated"],
|
||||
flagMutator: (chunk: StoredChunk) => void,
|
||||
failureCounter: PipelineDeps["telemetry"]["validationFailures"]
|
||||
): Promise<void> {
|
||||
const { metadata } = stored;
|
||||
try {
|
||||
const out = await run(metadata.chunkId, stored.data);
|
||||
stored.data = out.data;
|
||||
flagMutator(stored);
|
||||
deps.emitter.emit({
|
||||
type: successType,
|
||||
chunkId: metadata.chunkId,
|
||||
metadata,
|
||||
});
|
||||
await invokeHook(
|
||||
onSuccess,
|
||||
[metadata.chunkId, metadata],
|
||||
deps.hooks.onError,
|
||||
metadata.chunkId
|
||||
);
|
||||
} catch (err) {
|
||||
const error = toError(err);
|
||||
failureCounter.add(1);
|
||||
deps.emitter.emit({
|
||||
type: failureType,
|
||||
chunkId: metadata.chunkId,
|
||||
error,
|
||||
});
|
||||
await invokeHook(
|
||||
deps.hooks.onError,
|
||||
[error, metadata.chunkId],
|
||||
undefined
|
||||
);
|
||||
throw error instanceof StreamsChunkQueueError
|
||||
? error
|
||||
: new StreamsChunkQueueError(failureCode, error.message, {
|
||||
cause: error,
|
||||
context: { chunkId: metadata.chunkId.toString() },
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
# queue / storage-ops
|
||||
|
||||
## Purpose
|
||||
|
||||
Holds the storage-related guards used by `StreamingChunkQueue.loadChunk`
|
||||
and `loadAllChunks`. Both methods need the same "is the persister
|
||||
configured?" check and the same "is the chunk actually in storage?"
|
||||
check, so factoring them out keeps the queue file from duplicating the
|
||||
guards and from spilling past the per-file line budget.
|
||||
|
||||
## Features
|
||||
|
||||
- `loadChunkBytes` throws `STORAGE_NOT_CONFIGURED` when no persister is
|
||||
set, `STORAGE_CHUNK_MISSING` when the chunk is absent, and returns
|
||||
the bytes otherwise.
|
||||
- `listStoredChunkIds` throws `STORAGE_NOT_CONFIGURED` when no
|
||||
persister is set, otherwise forwards to `persister.listChunkIds()`.
|
||||
- Both helpers preserve a stringified `chunkId` in
|
||||
`error.context` so logs are scannable.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
const bytes = await loadChunkBytes(this.persister, chunkId);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
const ids = await listStoredChunkIds(this.persister);
|
||||
for (const id of ids) await this.loadChunk(id);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `loadChunkBytes(persister, chunkId)` → `Promise<Uint8Array>`
|
||||
|
||||
Throws `StreamsChunkQueueError(STORAGE_NOT_CONFIGURED)` if `persister`
|
||||
is `undefined`, `StreamsChunkQueueError(STORAGE_CHUNK_MISSING)` if the
|
||||
backend returns `undefined`, otherwise resolves with the bytes.
|
||||
|
||||
### `listStoredChunkIds(persister)` → `Promise<bigint[]>`
|
||||
|
||||
Throws `StreamsChunkQueueError(STORAGE_NOT_CONFIGURED)` if `persister`
|
||||
is `undefined`, otherwise forwards to `persister.listChunkIds()`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
These helpers are pure functions instead of methods so they can be
|
||||
unit-tested without spinning up a queue. The queue's `loadChunk`
|
||||
delegates to `loadChunkBytes` and then re-uses `addChunk` to push the
|
||||
bytes into the in-memory map; this means storage-loaded chunks travel
|
||||
through the same validator / decryptor / hook pipeline as
|
||||
network-arrived chunks. The two helpers throw a `StreamsChunkQueueError`
|
||||
rather than a `TypeError` because the missing-persister condition is
|
||||
an operational fault (the caller forgot to wire storage) and should be
|
||||
filterable by error code in dashboards.
|
||||
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import type { ChunkPersister } from "../storage-adapter/persister.types.js";
|
||||
import { listStoredChunkIds, loadChunkBytes } from "./storage-ops.js";
|
||||
|
||||
const passing: ChunkPersister = {
|
||||
persist: async () => {},
|
||||
load: async () => new Uint8Array([1, 2]),
|
||||
listChunkIds: async () => [0n, 1n],
|
||||
};
|
||||
|
||||
const missing: ChunkPersister = {
|
||||
persist: async () => {},
|
||||
load: async () => undefined,
|
||||
listChunkIds: async () => [],
|
||||
};
|
||||
|
||||
describe("loadChunkBytes", () => {
|
||||
it("throws STORAGE_NOT_CONFIGURED when persister is undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await loadChunkBytes(undefined, 0n);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_NOT_CONFIGURED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("throws STORAGE_CHUNK_MISSING when persister returns undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await loadChunkBytes(missing, 7n);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_CHUNK_MISSING
|
||||
);
|
||||
expect(err.context?.chunkId).toBe("7");
|
||||
}
|
||||
});
|
||||
|
||||
it("returns the bytes when the persister has them", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const out = await loadChunkBytes(passing, 0n);
|
||||
expect(out).toEqual(new Uint8Array([1, 2]));
|
||||
});
|
||||
});
|
||||
|
||||
describe("listStoredChunkIds", () => {
|
||||
it("throws STORAGE_NOT_CONFIGURED when persister is undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await listStoredChunkIds(undefined);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_NOT_CONFIGURED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards to the persister", { timeout: 5000 }, async () => {
|
||||
expect(await listStoredChunkIds(passing)).toEqual([0n, 1n]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import type { ChunkPersister } from "../storage-adapter/persister.types.js";
|
||||
|
||||
/** Load one chunk's bytes from the persister or throw. */
|
||||
export async function loadChunkBytes(
|
||||
persister: ChunkPersister | undefined,
|
||||
chunkId: bigint
|
||||
): Promise<Uint8Array> {
|
||||
if (persister === undefined) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_NOT_CONFIGURED,
|
||||
"persister is not configured"
|
||||
);
|
||||
}
|
||||
const data = await persister.load(chunkId);
|
||||
if (data === undefined) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_CHUNK_MISSING,
|
||||
`chunk ${chunkId} not in storage`,
|
||||
{ context: { chunkId: chunkId.toString() } }
|
||||
);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
/** List all chunk ids in storage or throw. */
|
||||
export async function listStoredChunkIds(
|
||||
persister: ChunkPersister | undefined
|
||||
): Promise<bigint[]> {
|
||||
if (persister === undefined) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_NOT_CONFIGURED,
|
||||
"persister is not configured"
|
||||
);
|
||||
}
|
||||
return persister.listChunkIds();
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
# streaming-chunk-queue
|
||||
|
||||
## Purpose
|
||||
|
||||
`StreamingChunkQueue` is the public class that reconstitutes
|
||||
out-of-order chunked file transfers into a sequential
|
||||
`ReadableStream<Uint8Array>`. It composes the package's pluggable
|
||||
validator, decryptor, persister, rate limiter, hooks, event emitter,
|
||||
and OTel telemetry into a single coherent reception primitive — the
|
||||
piece every chunked-file consumer (drive-mesh, drive-webrtc,
|
||||
drive-transport-https) plugs into when chunks arrive via a jittery or
|
||||
out-of-order transport.
|
||||
|
||||
## Features
|
||||
|
||||
- Out-of-order ingestion: chunks may arrive in any order; duplicates
|
||||
are silently ignored.
|
||||
- Sequential output: the readable stream emits chunks strictly in
|
||||
ascending `chunkId` order.
|
||||
- Auto-start: the first `addChunk` automatically materialises the
|
||||
stream (configurable via `autoStart: false`).
|
||||
- Pluggable validator + decryptor strategies with adapters for
|
||||
`@suaveplan/packer` `ChunkVerifier`.
|
||||
- Optional `TokenBucketRateLimiterAdapter` integration for bandwidth
|
||||
throttling without dropping chunks.
|
||||
- Optional `StorageChunkPersister` for resumable transfers.
|
||||
- Eight lifecycle hooks, ten event types, and wildcard listener
|
||||
support.
|
||||
- Binary chunk-request protocol exposed through `requestChunks` and
|
||||
`requestMissing`.
|
||||
- Missing-chunk timeout that errors the readable stream after
|
||||
`chunkTimeout` ms (default 30 000).
|
||||
- Frozen status snapshot via `getStatus`.
|
||||
- OTel counters and a histogram (`stream_duration_ms`).
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { StreamingChunkQueue } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 12345n,
|
||||
totalChunks: 5n,
|
||||
});
|
||||
|
||||
await queue.addChunk(0n, bytes0);
|
||||
await queue.addChunk(4n, bytes4);
|
||||
await queue.addChunk(1n, bytes1);
|
||||
await queue.addChunk(2n, bytes2);
|
||||
await queue.addChunk(3n, bytes3);
|
||||
|
||||
const reader = queue.getStream().getReader();
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
process(value);
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { ChunkVerifier } from "@suaveplan/packer";
|
||||
import { TokenBucket } from "@suaveplan/rate-limiter";
|
||||
import { MemoryStorage } from "@suaveplan/storage";
|
||||
import {
|
||||
ChunkVerifierValidator,
|
||||
StorageChunkPersister,
|
||||
StreamingChunkQueue,
|
||||
TokenBucketRateLimiterAdapter,
|
||||
} from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const verifier = new ChunkVerifier(serverPubKey);
|
||||
const bucket = new TokenBucket({ capacity: 1_048_576, refillRate: 1024 });
|
||||
const storage = new MemoryStorage();
|
||||
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 12345n,
|
||||
totalChunks: 1024n,
|
||||
validator: new ChunkVerifierValidator(verifier),
|
||||
rateLimiter: new TokenBucketRateLimiterAdapter(bucket),
|
||||
persister: new StorageChunkPersister({ storage, fileId: 12345n }),
|
||||
chunkTimeout: 60_000,
|
||||
hooks: {
|
||||
onStreamProgress: (pct) => statusBar.set(pct),
|
||||
onError: (err) => toaster.error(err.message),
|
||||
},
|
||||
});
|
||||
|
||||
queue.addEventListener("*", (event) =>
|
||||
telemetry.record("chunk_queue.event", { type: event.type }),
|
||||
);
|
||||
|
||||
const stream = queue.getStream();
|
||||
await stream.pipeTo(diskWriter);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `new StreamingChunkQueue(options)`
|
||||
|
||||
`options` is `StreamingChunkQueueOptions`. The constructor throws
|
||||
`StreamsChunkQueueError(INVALID_TOTAL_CHUNKS)` when `totalChunks <= 0n`.
|
||||
|
||||
### `addChunk(chunkId, data)` → `Promise<void>`
|
||||
|
||||
Persists, validates, decrypts, and enqueues the chunk. Throws
|
||||
`INVALID_CHUNK_ID` on out-of-range ids, or
|
||||
`VALIDATION_FAILED` / `DECRYPTION_FAILED` if the pipeline rejects.
|
||||
|
||||
### `getStream()` → `ReadableStream<Uint8Array>`
|
||||
|
||||
Idempotent: subsequent calls return the same stream.
|
||||
|
||||
### `requestChunks(ids?)` / `requestMissing()` → `Uint8Array`
|
||||
|
||||
Encode the binary chunk-request frame.
|
||||
|
||||
### `getMissingChunkIds()` → `bigint[]`
|
||||
|
||||
Ascending list of un-received chunk ids.
|
||||
|
||||
### `getStatus()` → `StreamingQueueStatus`
|
||||
|
||||
Frozen snapshot.
|
||||
|
||||
### `loadChunk(chunkId)` / `loadAllChunks()` → `Promise<void>`
|
||||
|
||||
Hydrate from the configured persister. Throw
|
||||
`STORAGE_NOT_CONFIGURED` / `STORAGE_CHUNK_MISSING` on failure.
|
||||
|
||||
### `isAllChunksReceived()` / `isStreamingComplete()` → `boolean`
|
||||
|
||||
### `addEventListener(type, listener)` / `removeEventListener(type, listener)`
|
||||
|
||||
`type` is any `StreamEvent['type']` or `'*'`.
|
||||
|
||||
### `clear()` → `void`
|
||||
|
||||
Reset internal state for reuse.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The validator and decryptor stages are extracted to `pipeline.ts` so
|
||||
the queue file stays close to the per-file line budget. Storage guards
|
||||
live in `storage-ops.ts` for the same reason. Rate-limiting denial is
|
||||
non-destructive — the queue sleeps for the bucket's `retryAfter` (or
|
||||
the adapter's fallback delay) and retries the same chunk; no bytes are
|
||||
ever dropped. Missing-chunk timeout polls the in-memory map at a
|
||||
configurable interval (default 100 ms) so deterministic tests can
|
||||
inject a custom `clock` + `sleep`.
|
||||
@@ -0,0 +1,687 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { ChunkDecryptor } from "../decryptor/decryptor.types.js";
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import type { ChunkRateLimiter } from "../rate-limiter/adapter.types.js";
|
||||
import { StorageChunkPersister } from "../storage-adapter/persister.js";
|
||||
import type { ChunkStorageLike } from "../storage-adapter/persister.types.js";
|
||||
import type {
|
||||
ChunkMetadata,
|
||||
StreamEvent,
|
||||
StreamingChunkQueueHooks,
|
||||
} from "../types/types.js";
|
||||
import type { ChunkValidator } from "../validator/validator.types.js";
|
||||
import { StreamingChunkQueue } from "./streaming-chunk-queue.js";
|
||||
|
||||
function makeStorage(): ChunkStorageLike & { map: Map<string, Uint8Array> } {
|
||||
const map = new Map<string, Uint8Array>();
|
||||
return {
|
||||
map,
|
||||
async get(key) {
|
||||
return map.get(key);
|
||||
},
|
||||
async set(key, value) {
|
||||
map.set(key, value);
|
||||
},
|
||||
async list({ prefix } = {}) {
|
||||
const keys = Array.from(map.keys()).filter((k) =>
|
||||
prefix ? k.startsWith(prefix) : true
|
||||
);
|
||||
return { keys };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function bytes(...values: number[]): Uint8Array {
|
||||
return new Uint8Array(values);
|
||||
}
|
||||
|
||||
async function readAll(
|
||||
stream: ReadableStream<Uint8Array>
|
||||
): Promise<Uint8Array[]> {
|
||||
const reader = stream.getReader();
|
||||
const out: Uint8Array[] = [];
|
||||
for (; ;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value !== undefined) out.push(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const FAST_TIMING = { chunkTimeout: 500, chunkPollInterval: 5 } as const;
|
||||
|
||||
describe("StreamingChunkQueue — construction", () => {
|
||||
it("rejects non-positive totalChunks", { timeout: 5000 }, () => {
|
||||
expect(
|
||||
() => new StreamingChunkQueue({ fileId: 1n, totalChunks: 0n })
|
||||
).toThrow(StreamsChunkQueueError);
|
||||
});
|
||||
|
||||
it("rejects negative totalChunks with INVALID_TOTAL_CHUNKS code", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
new StreamingChunkQueue({ fileId: 1n, totalChunks: -1n });
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_TOTAL_CHUNKS
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — ingestion", () => {
|
||||
it("rejects out-of-range chunk ids", { timeout: 5000 }, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 3n,
|
||||
autoStart: false,
|
||||
});
|
||||
await expect(q.addChunk(3n, bytes(0))).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
await expect(q.addChunk(-1n, bytes(0))).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
});
|
||||
|
||||
it("deduplicates and assembles out-of-order chunks", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
autoStart: false,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
await q.addChunk(2n, bytes(2));
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await q.addChunk(2n, bytes(99));
|
||||
await q.addChunk(1n, bytes(1));
|
||||
await q.addChunk(3n, bytes(3));
|
||||
await q.addChunk(4n, bytes(4));
|
||||
expect(q.getStatus().receivedChunks).toBe(5);
|
||||
const parts = await readAll(q.getStream());
|
||||
expect(parts.map((p) => p[0])).toEqual([0, 1, 2, 3, 4]);
|
||||
});
|
||||
|
||||
it("auto-starts streaming on the first chunk when autoStart defaults to true", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
await q.addChunk(0n, bytes(7));
|
||||
await Promise.resolve();
|
||||
expect(q.getStatus().isStreaming || q.getStatus().isComplete).toBe(
|
||||
true
|
||||
);
|
||||
const parts = await readAll(q.getStream());
|
||||
expect(parts[0]?.[0]).toBe(7);
|
||||
});
|
||||
|
||||
it("does NOT auto-start when autoStart is false", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
expect(q.getStatus().isStreaming).toBe(false);
|
||||
});
|
||||
|
||||
it("exposes bigint identifiers on status and missing-chunk APIs", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 12345n,
|
||||
totalChunks: 4n,
|
||||
autoStart: false,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
expect(q.getStatus().fileId).toBe(12345n);
|
||||
expect(q.getStatus().totalChunks).toBe(4n);
|
||||
expect(q.getMissingChunkIds()).toEqual([1n, 2n, 3n]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — interleaved ingestion", () => {
|
||||
it("continues streaming when chunks arrive after getStream() has started", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 2n,
|
||||
autoStart: false,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
const stream = q.getStream();
|
||||
// Allow the readable start callback to flip isStreamingFlag.
|
||||
await Promise.resolve();
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await q.addChunk(1n, bytes(1));
|
||||
const parts = await readAll(stream);
|
||||
expect(parts).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — stream output", () => {
|
||||
it("closes the readable stream after the final chunk", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 2n,
|
||||
autoStart: false,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await q.addChunk(1n, bytes(1));
|
||||
const parts = await readAll(q.getStream());
|
||||
expect(parts).toHaveLength(2);
|
||||
expect(q.isStreamingComplete()).toBe(true);
|
||||
expect(q.getStatus().isComplete).toBe(true);
|
||||
});
|
||||
|
||||
it("getStream() returns the same stream on subsequent calls", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
});
|
||||
expect(q.getStream()).toBe(q.getStream());
|
||||
});
|
||||
|
||||
it("ignores addChunk-driven streaming after the reader cancels", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 3n,
|
||||
autoStart: false,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
const reader = q.getStream().getReader();
|
||||
await Promise.resolve();
|
||||
await reader.cancel();
|
||||
await q.addChunk(0n, bytes(0));
|
||||
expect(q.getStatus().isStreaming).toBe(false);
|
||||
expect(q.getStatus().receivedChunks).toBe(1);
|
||||
});
|
||||
|
||||
it("supports stream cancellation", { timeout: 5000 }, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
autoStart: false,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
const reader = q.getStream().getReader();
|
||||
await reader.cancel();
|
||||
expect(q.getStatus().isStreaming).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — validation/decryption pipeline", () => {
|
||||
it("emits validation:failed and rejects when validator throws", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const validator: ChunkValidator = {
|
||||
validate: async () => {
|
||||
throw new Error("bad-sig");
|
||||
},
|
||||
};
|
||||
const events: StreamEvent[] = [];
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
validator,
|
||||
});
|
||||
q.addEventListener("*", (e) => events.push(e));
|
||||
await expect(q.addChunk(0n, bytes(0))).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
expect(events.some((e) => e.type === "validation:failed")).toBe(true);
|
||||
});
|
||||
|
||||
it("emits decryption:failed and rejects when decryptor throws", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const decryptor: ChunkDecryptor = {
|
||||
decrypt: async () => {
|
||||
throw "bad-key";
|
||||
},
|
||||
};
|
||||
const events: StreamEvent[] = [];
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
decryptor,
|
||||
});
|
||||
q.addEventListener("*", (e) => events.push(e));
|
||||
await expect(q.addChunk(0n, bytes(0))).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
expect(events.some((e) => e.type === "decryption:failed")).toBe(true);
|
||||
});
|
||||
|
||||
it("transforms data through validator and decryptor", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const validator: ChunkValidator = {
|
||||
validate: async (_id, d) => ({ data: d.slice(1) }),
|
||||
};
|
||||
const decryptor: ChunkDecryptor = {
|
||||
decrypt: async (_id, d) => ({
|
||||
data: new Uint8Array(d.map((b) => b ^ 0xff)),
|
||||
}),
|
||||
};
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
validator,
|
||||
decryptor,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0xab, 0x10));
|
||||
const [out] = await readAll(q.getStream());
|
||||
expect(out?.[0]).toBe(0xef);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — rate limiting", () => {
|
||||
it("defers streaming under rate-limit pressure without dropping chunks", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let allowed = false;
|
||||
const limiter: ChunkRateLimiter = {
|
||||
consume: () => {
|
||||
if (!allowed) {
|
||||
allowed = true;
|
||||
return { allowed: false, delay: 1 };
|
||||
}
|
||||
return { allowed: true, delay: 0 };
|
||||
},
|
||||
};
|
||||
const events: StreamEvent[] = [];
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
rateLimiter: limiter,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
q.addEventListener("*", (e) => events.push(e));
|
||||
await q.addChunk(0n, bytes(1));
|
||||
const parts = await readAll(q.getStream());
|
||||
expect(parts).toHaveLength(1);
|
||||
expect(events.some((e) => e.type === "rate:limited")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — cancel during rate-limit sleep", () => {
|
||||
it("aborts the stream loop when the reader cancels during a rate-limit sleep", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let cancelHit = false;
|
||||
const limiter: ChunkRateLimiter = {
|
||||
consume: () => ({ allowed: false, delay: 50 }),
|
||||
};
|
||||
// custom sleep that triggers cancel mid-flight, then a resolves promise.
|
||||
let queueRef: StreamingChunkQueue | undefined;
|
||||
const sleepFn = async (_ms: number) => {
|
||||
if (!cancelHit && queueRef !== undefined) {
|
||||
cancelHit = true;
|
||||
// Force a cancel via internal accessors: call clear() to simulate.
|
||||
queueRef.clear();
|
||||
}
|
||||
};
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 2n,
|
||||
autoStart: false,
|
||||
rateLimiter: limiter,
|
||||
sleep: sleepFn,
|
||||
chunkTimeout: 100,
|
||||
chunkPollInterval: 5,
|
||||
});
|
||||
queueRef = q;
|
||||
await q.addChunk(0n, bytes(0));
|
||||
const stream = q.getStream();
|
||||
// Wait for the loop to attempt enqueue and trip rate-limit sleep.
|
||||
await new Promise((r) => setTimeout(r, 30));
|
||||
expect(stream).toBeDefined();
|
||||
expect(cancelHit).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — timeout", () => {
|
||||
it("errors the readable stream after chunkTimeout", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let now = 0;
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 2n,
|
||||
autoStart: false,
|
||||
chunkTimeout: 10,
|
||||
chunkPollInterval: 1,
|
||||
clock: () => now,
|
||||
sleep: async () => {
|
||||
now += 5;
|
||||
},
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
const reader = q.getStream().getReader();
|
||||
const first = await reader.read();
|
||||
expect(first.done).toBe(false);
|
||||
await expect(reader.read()).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — hooks", () => {
|
||||
it("fires every lifecycle hook", { timeout: 5000 }, async () => {
|
||||
const fired = new Set<string>();
|
||||
const hooks: StreamingChunkQueueHooks = {
|
||||
onChunkReceived: () => {
|
||||
fired.add("received");
|
||||
},
|
||||
onChunkValidated: () => {
|
||||
fired.add("validated");
|
||||
},
|
||||
onChunkDecrypted: () => {
|
||||
fired.add("decrypted");
|
||||
},
|
||||
onChunkStreamed: () => {
|
||||
fired.add("streamed");
|
||||
},
|
||||
onStreamProgress: () => {
|
||||
fired.add("progress");
|
||||
},
|
||||
onStreamComplete: () => {
|
||||
fired.add("complete");
|
||||
},
|
||||
onError: () => {
|
||||
fired.add("error");
|
||||
},
|
||||
onRateLimit: () => {
|
||||
fired.add("rate");
|
||||
},
|
||||
};
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
hooks,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await readAll(q.getStream());
|
||||
for (const expected of [
|
||||
"received",
|
||||
"validated",
|
||||
"decrypted",
|
||||
"streamed",
|
||||
"progress",
|
||||
"complete",
|
||||
]) {
|
||||
expect(fired.has(expected)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — status snapshot", () => {
|
||||
it("reports a precise status snapshot during partial ingestion", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
autoStart: false,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await q.addChunk(2n, bytes(2));
|
||||
await q.addChunk(4n, bytes(4));
|
||||
const status = q.getStatus();
|
||||
expect(status.receivedChunks).toBe(3);
|
||||
expect(status.progress).toBe(60);
|
||||
expect(status.missingChunks).toEqual([1n, 3n]);
|
||||
expect(status.validatedChunks).toBe(3);
|
||||
expect(status.decryptedChunks).toBe(3);
|
||||
expect(q.isAllChunksReceived()).toBe(false);
|
||||
});
|
||||
|
||||
it("reports unvalidated and undecrypted chunks correctly", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const validator: ChunkValidator = {
|
||||
validate: async () => {
|
||||
throw new Error("v-fail");
|
||||
},
|
||||
};
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 2n,
|
||||
autoStart: false,
|
||||
validator,
|
||||
});
|
||||
await expect(q.addChunk(0n, bytes(0))).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
const status = q.getStatus();
|
||||
expect(status.receivedChunks).toBe(1);
|
||||
expect(status.validatedChunks).toBe(0);
|
||||
expect(status.decryptedChunks).toBe(0);
|
||||
});
|
||||
|
||||
it("flips isAllChunksReceived when every chunk has been ingested", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 2n,
|
||||
autoStart: false,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await q.addChunk(1n, bytes(1));
|
||||
expect(q.isAllChunksReceived()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — binary requests", () => {
|
||||
it("produces a binary chunk request for missing chunks", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 12345n,
|
||||
totalChunks: 10n,
|
||||
autoStart: false,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await q.addChunk(2n, bytes(2));
|
||||
const missing = q.requestMissing();
|
||||
expect(missing.byteLength).toBeGreaterThan(12);
|
||||
});
|
||||
|
||||
it("requestChunks defaults to missing chunks when no id list is supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 4n,
|
||||
autoStart: false,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
const buf = q.requestChunks();
|
||||
expect(buf[12]).toBe(0b00001110);
|
||||
});
|
||||
|
||||
it("requestChunks honours a caller-supplied id list", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 4n,
|
||||
autoStart: false,
|
||||
});
|
||||
const buf = q.requestChunks([1n, 3n]);
|
||||
expect(buf[12]).toBe(0b00001010);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — persistence", () => {
|
||||
it("hydrates from storage via loadAllChunks", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const storage = makeStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 7n });
|
||||
const source = new StreamingChunkQueue({
|
||||
fileId: 7n,
|
||||
totalChunks: 3n,
|
||||
autoStart: false,
|
||||
persister,
|
||||
});
|
||||
await source.addChunk(0n, bytes(0));
|
||||
await source.addChunk(1n, bytes(1));
|
||||
await source.addChunk(2n, bytes(2));
|
||||
|
||||
const restored = new StreamingChunkQueue({
|
||||
fileId: 7n,
|
||||
totalChunks: 3n,
|
||||
autoStart: false,
|
||||
persister,
|
||||
});
|
||||
await restored.loadAllChunks();
|
||||
expect(restored.getStatus().receivedChunks).toBe(3);
|
||||
});
|
||||
|
||||
it("loadChunk throws STORAGE_NOT_CONFIGURED when no persister is wired", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
});
|
||||
await expect(q.loadChunk(0n)).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
});
|
||||
|
||||
it("loadAllChunks throws STORAGE_NOT_CONFIGURED when no persister is wired", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
});
|
||||
await expect(q.loadAllChunks()).rejects.toBeInstanceOf(
|
||||
StreamsChunkQueueError
|
||||
);
|
||||
});
|
||||
|
||||
it("loadChunk throws STORAGE_CHUNK_MISSING when the key is absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const storage = makeStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 1n });
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
autoStart: false,
|
||||
persister,
|
||||
});
|
||||
try {
|
||||
await q.loadChunk(0n);
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STORAGE_CHUNK_MISSING
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — listeners + clear", () => {
|
||||
it("removes event listeners on demand", { timeout: 5000 }, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
});
|
||||
let count = 0;
|
||||
const listener = () => {
|
||||
count += 1;
|
||||
};
|
||||
q.addEventListener("chunk:received", listener);
|
||||
q.removeEventListener("chunk:received", listener);
|
||||
await q.addChunk(0n, bytes(0));
|
||||
expect(count).toBe(0);
|
||||
});
|
||||
|
||||
it("clears all state and is reusable", { timeout: 5000 }, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
...FAST_TIMING,
|
||||
});
|
||||
await q.addChunk(0n, bytes(0));
|
||||
await readAll(q.getStream());
|
||||
q.clear();
|
||||
expect(q.getStatus().receivedChunks).toBe(0);
|
||||
expect(q.getStatus().isComplete).toBe(false);
|
||||
await q.addChunk(0n, bytes(9));
|
||||
expect(q.getStatus().receivedChunks).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamingChunkQueue — universal purity", () => {
|
||||
it("is universal and avoids browser+node globals", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(typeof StreamingChunkQueue).toBe("function");
|
||||
});
|
||||
|
||||
it("executes under @suaveplan/testing/runner", { timeout: 5000 }, () => {
|
||||
expect(STREAMS_CHUNK_QUEUE_CODES).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("StreamEvent metadata round-trip", () => {
|
||||
it("propagates metadata into chunk:received events", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const q = new StreamingChunkQueue({
|
||||
fileId: 99n,
|
||||
totalChunks: 1n,
|
||||
autoStart: false,
|
||||
clock: () => 1234,
|
||||
});
|
||||
let captured: ChunkMetadata | undefined;
|
||||
q.addEventListener("chunk:received", (e) => {
|
||||
if (e.type === "chunk:received") captured = e.metadata;
|
||||
});
|
||||
await q.addChunk(0n, bytes(1, 2, 3));
|
||||
expect(captured?.fileId).toBe(99n);
|
||||
expect(captured?.chunkId).toBe(0n);
|
||||
expect(captured?.size).toBe(3);
|
||||
expect(captured?.receivedAt).toBe(1234);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,364 @@
|
||||
import { encodeChunkRequest } from "../binary-protocol/encoder.js";
|
||||
import { noopChunkDecryptor } from "../decryptor/decryptor.js";
|
||||
import type { ChunkDecryptor } from "../decryptor/decryptor.types.js";
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import { StreamEventEmitter } from "../events/events.js";
|
||||
import { invokeHook } from "../hooks/hooks.js";
|
||||
import type { ChunkRateLimiter } from "../rate-limiter/adapter.types.js";
|
||||
import type { ChunkPersister } from "../storage-adapter/persister.types.js";
|
||||
import { getTelemetry } from "../telemetry/telemetry.js";
|
||||
import type {
|
||||
ChunkMetadata,
|
||||
StoredChunk,
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
StreamEventType,
|
||||
StreamingChunkQueueHooks,
|
||||
StreamingQueueStatus,
|
||||
} from "../types/types.js";
|
||||
import { noopChunkValidator } from "../validator/validator.js";
|
||||
import type { ChunkValidator } from "../validator/validator.types.js";
|
||||
import { type PipelineDeps, runDecrypt, runValidate } from "./pipeline.js";
|
||||
import { listStoredChunkIds, loadChunkBytes } from "./storage-ops.js";
|
||||
import type { StreamingChunkQueueOptions } from "./streaming-chunk-queue.types.js";
|
||||
|
||||
const DEFAULT_CHUNK_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_POLL_INTERVAL_MS = 100;
|
||||
|
||||
function defaultSleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Out-of-order chunk reconstitution for chunked file transfers.
|
||||
* Accepts chunks in any order and exposes a sequential
|
||||
* `ReadableStream<Uint8Array>` consumer surface.
|
||||
*/
|
||||
export class StreamingChunkQueue {
|
||||
private readonly fileId: bigint;
|
||||
private readonly totalChunks: bigint;
|
||||
private readonly validator: ChunkValidator;
|
||||
private readonly decryptor: ChunkDecryptor;
|
||||
private readonly persister: ChunkPersister | undefined;
|
||||
private readonly rateLimiter: ChunkRateLimiter | undefined;
|
||||
private readonly hooks: StreamingChunkQueueHooks;
|
||||
private readonly autoStart: boolean;
|
||||
private readonly chunkTimeout: number;
|
||||
private readonly pollInterval: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly sleep: (ms: number) => Promise<void>;
|
||||
private readonly chunks: Map<bigint, StoredChunk> = new Map();
|
||||
private readonly emitter = new StreamEventEmitter();
|
||||
private readonly telemetry = getTelemetry();
|
||||
private readonly deps: PipelineDeps;
|
||||
private stream: ReadableStream<Uint8Array> | undefined;
|
||||
private controller: ReadableStreamDefaultController<Uint8Array> | undefined;
|
||||
private nextChunkId = 0n;
|
||||
private streamedChunks = 0;
|
||||
private isStreamingFlag = false;
|
||||
private isCompleteFlag = false;
|
||||
private streamRunning = false;
|
||||
private streamStartTs = 0;
|
||||
|
||||
public constructor(options: StreamingChunkQueueOptions) {
|
||||
if (options.totalChunks <= 0n) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_TOTAL_CHUNKS,
|
||||
`totalChunks must be > 0 (got ${options.totalChunks})`,
|
||||
{ context: { totalChunks: options.totalChunks.toString() } }
|
||||
);
|
||||
}
|
||||
this.fileId = options.fileId;
|
||||
this.totalChunks = options.totalChunks;
|
||||
this.validator = options.validator ?? noopChunkValidator;
|
||||
this.decryptor = options.decryptor ?? noopChunkDecryptor;
|
||||
this.persister = options.persister;
|
||||
this.rateLimiter = options.rateLimiter;
|
||||
this.hooks = options.hooks ?? {};
|
||||
this.autoStart = options.autoStart !== false;
|
||||
this.chunkTimeout = options.chunkTimeout ?? DEFAULT_CHUNK_TIMEOUT_MS;
|
||||
this.pollInterval =
|
||||
options.chunkPollInterval ?? DEFAULT_POLL_INTERVAL_MS;
|
||||
this.clock = options.clock ?? (() => Date.now());
|
||||
this.sleep = options.sleep ?? defaultSleep;
|
||||
this.deps = {
|
||||
emitter: this.emitter,
|
||||
hooks: this.hooks,
|
||||
telemetry: this.telemetry,
|
||||
};
|
||||
}
|
||||
|
||||
public async addChunk(chunkId: bigint, data: Uint8Array): Promise<void> {
|
||||
if (chunkId < 0n || chunkId >= this.totalChunks) {
|
||||
throw new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.INVALID_CHUNK_ID,
|
||||
`chunkId ${chunkId} out of [0, ${this.totalChunks})`,
|
||||
{ context: { chunkId: chunkId.toString() } }
|
||||
);
|
||||
}
|
||||
if (this.chunks.has(chunkId)) return;
|
||||
const metadata: ChunkMetadata = {
|
||||
fileId: this.fileId,
|
||||
chunkId,
|
||||
totalChunks: this.totalChunks,
|
||||
size: data.byteLength,
|
||||
receivedAt: this.clock(),
|
||||
validated: false,
|
||||
decrypted: false,
|
||||
};
|
||||
const stored: StoredChunk = { data, metadata };
|
||||
this.chunks.set(chunkId, stored);
|
||||
this.telemetry.chunksReceived.add(1);
|
||||
this.emit({ type: "chunk:received", chunkId, metadata });
|
||||
await invokeHook(
|
||||
this.hooks.onChunkReceived,
|
||||
[chunkId, metadata],
|
||||
this.hooks.onError,
|
||||
chunkId
|
||||
);
|
||||
if (this.persister !== undefined) await this.persister.persist(stored);
|
||||
await runValidate(stored, this.validator, this.deps);
|
||||
await runDecrypt(stored, this.decryptor, this.deps);
|
||||
await this.emitProgress();
|
||||
if (this.isStreamingFlag && this.controller !== undefined) {
|
||||
await this.streamReady();
|
||||
} else if (this.autoStart && this.stream === undefined) {
|
||||
this.getStream();
|
||||
await Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
public getStream(): ReadableStream<Uint8Array> {
|
||||
if (this.stream !== undefined) return this.stream;
|
||||
this.stream = new ReadableStream<Uint8Array>({
|
||||
start: (controller) => {
|
||||
this.controller = controller;
|
||||
this.isStreamingFlag = true;
|
||||
this.streamStartTs = this.clock();
|
||||
void this.streamReady();
|
||||
},
|
||||
cancel: () => {
|
||||
this.isStreamingFlag = false;
|
||||
this.controller = undefined;
|
||||
},
|
||||
});
|
||||
return this.stream;
|
||||
}
|
||||
|
||||
private async streamReady(): Promise<void> {
|
||||
if (
|
||||
this.controller === undefined ||
|
||||
!this.isStreamingFlag ||
|
||||
this.streamRunning
|
||||
) {
|
||||
return;
|
||||
}
|
||||
this.streamRunning = true;
|
||||
try {
|
||||
await this.streamLoop();
|
||||
} finally {
|
||||
this.streamRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async streamLoop(): Promise<void> {
|
||||
while (this.nextChunkId < this.totalChunks) {
|
||||
/* istanbul ignore if -- re-entrancy guard for mid-iteration cancel */
|
||||
if (!this.isStreamingFlag || this.controller === undefined) return;
|
||||
const chunk = this.chunks.get(this.nextChunkId);
|
||||
if (chunk === undefined) {
|
||||
const arrived = await this.waitForChunk(this.nextChunkId);
|
||||
if (arrived) continue;
|
||||
this.timeoutAndError(this.nextChunkId);
|
||||
return;
|
||||
}
|
||||
if (!(await this.rateLimitConsume(chunk))) continue;
|
||||
this.controller.enqueue(chunk.data);
|
||||
this.streamedChunks += 1;
|
||||
this.telemetry.chunksStreamed.add(1);
|
||||
this.emit({
|
||||
type: "chunk:streamed",
|
||||
chunkId: this.nextChunkId,
|
||||
metadata: chunk.metadata,
|
||||
});
|
||||
await invokeHook(
|
||||
this.hooks.onChunkStreamed,
|
||||
[this.nextChunkId, chunk.metadata],
|
||||
this.hooks.onError,
|
||||
this.nextChunkId
|
||||
);
|
||||
this.nextChunkId += 1n;
|
||||
await this.emitProgress();
|
||||
if (this.nextChunkId >= this.totalChunks) {
|
||||
this.completeStream();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async rateLimitConsume(chunk: StoredChunk): Promise<boolean> {
|
||||
if (this.rateLimiter === undefined) return true;
|
||||
const outcome = this.rateLimiter.consume(chunk.data.byteLength);
|
||||
if (outcome.allowed) return true;
|
||||
this.telemetry.rateLimited.add(1);
|
||||
this.emit({
|
||||
type: "rate:limited",
|
||||
chunkId: this.nextChunkId,
|
||||
delay: outcome.delay,
|
||||
});
|
||||
await invokeHook(
|
||||
this.hooks.onRateLimit,
|
||||
[this.nextChunkId, outcome.delay],
|
||||
this.hooks.onError,
|
||||
this.nextChunkId
|
||||
);
|
||||
await this.sleep(outcome.delay);
|
||||
return false;
|
||||
}
|
||||
|
||||
private completeStream(): void {
|
||||
this.isCompleteFlag = true;
|
||||
this.isStreamingFlag = false;
|
||||
this.controller?.close();
|
||||
this.telemetry.streamDuration.record(this.clock() - this.streamStartTs);
|
||||
this.emit({ type: "stream:complete", totalChunks: this.totalChunks });
|
||||
void invokeHook(
|
||||
this.hooks.onStreamComplete,
|
||||
[this.totalChunks],
|
||||
this.hooks.onError
|
||||
);
|
||||
}
|
||||
|
||||
private timeoutAndError(chunkId: bigint): void {
|
||||
const error = new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.STREAM_TIMEOUT,
|
||||
`timed out waiting for chunk ${chunkId}`,
|
||||
{ context: { chunkId: chunkId.toString() } }
|
||||
);
|
||||
this.emit({ type: "stream:error", error, chunkId });
|
||||
void invokeHook(this.hooks.onError, [error, chunkId], undefined);
|
||||
this.controller?.error(error);
|
||||
this.isStreamingFlag = false;
|
||||
}
|
||||
|
||||
private async waitForChunk(chunkId: bigint): Promise<boolean> {
|
||||
const start = this.clock();
|
||||
while (this.clock() - start < this.chunkTimeout) {
|
||||
if (this.chunks.has(chunkId)) return true;
|
||||
await this.sleep(this.pollInterval);
|
||||
}
|
||||
return this.chunks.has(chunkId);
|
||||
}
|
||||
|
||||
private async emitProgress(): Promise<void> {
|
||||
const received = this.chunks.size;
|
||||
const progress = Math.floor(
|
||||
(received / Number(this.totalChunks)) * 100
|
||||
);
|
||||
this.emit({
|
||||
type: "stream:progress",
|
||||
progress,
|
||||
totalChunks: this.totalChunks,
|
||||
receivedChunks: received,
|
||||
});
|
||||
await invokeHook(
|
||||
this.hooks.onStreamProgress,
|
||||
[progress, received, this.totalChunks],
|
||||
this.hooks.onError
|
||||
);
|
||||
}
|
||||
|
||||
private emit(event: StreamEvent): void {
|
||||
this.emitter.emit(event);
|
||||
}
|
||||
|
||||
public requestChunks(chunkIds?: readonly bigint[]): Uint8Array {
|
||||
return encodeChunkRequest({
|
||||
fileId: this.fileId,
|
||||
totalChunks: this.totalChunks,
|
||||
chunkIds: chunkIds ?? this.getMissingChunkIds(),
|
||||
});
|
||||
}
|
||||
|
||||
public requestMissing(): Uint8Array {
|
||||
return this.requestChunks(this.getMissingChunkIds());
|
||||
}
|
||||
|
||||
public getMissingChunkIds(): bigint[] {
|
||||
const missing: bigint[] = [];
|
||||
for (let i = 0n; i < this.totalChunks; i += 1n) {
|
||||
if (!this.chunks.has(i)) missing.push(i);
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
public getStatus(): StreamingQueueStatus {
|
||||
let validatedChunks = 0;
|
||||
let decryptedChunks = 0;
|
||||
for (const c of this.chunks.values()) {
|
||||
if (c.metadata.validated) validatedChunks += 1;
|
||||
if (c.metadata.decrypted) decryptedChunks += 1;
|
||||
}
|
||||
const received = this.chunks.size;
|
||||
return Object.freeze({
|
||||
fileId: this.fileId,
|
||||
totalChunks: this.totalChunks,
|
||||
receivedChunks: received,
|
||||
validatedChunks,
|
||||
decryptedChunks,
|
||||
streamedChunks: this.streamedChunks,
|
||||
progress: Math.floor((received / Number(this.totalChunks)) * 100),
|
||||
isStreaming: this.isStreamingFlag,
|
||||
isComplete: this.isCompleteFlag,
|
||||
missingChunks: this.getMissingChunkIds(),
|
||||
nextChunkId: this.nextChunkId,
|
||||
});
|
||||
}
|
||||
|
||||
public async loadChunk(chunkId: bigint): Promise<void> {
|
||||
const data = await loadChunkBytes(this.persister, chunkId);
|
||||
await this.addChunk(chunkId, data);
|
||||
}
|
||||
|
||||
public async loadAllChunks(): Promise<void> {
|
||||
const ids = await listStoredChunkIds(this.persister);
|
||||
for (const id of ids) await this.loadChunk(id);
|
||||
}
|
||||
|
||||
public isAllChunksReceived(): boolean {
|
||||
return BigInt(this.chunks.size) === this.totalChunks;
|
||||
}
|
||||
|
||||
public isStreamingComplete(): boolean {
|
||||
return this.isCompleteFlag;
|
||||
}
|
||||
|
||||
public addEventListener(
|
||||
type: StreamEventType,
|
||||
listener: StreamEventListener
|
||||
): void {
|
||||
this.emitter.addEventListener(type, listener);
|
||||
}
|
||||
|
||||
public removeEventListener(
|
||||
type: StreamEventType,
|
||||
listener: StreamEventListener
|
||||
): void {
|
||||
this.emitter.removeEventListener(type, listener);
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
this.chunks.clear();
|
||||
this.nextChunkId = 0n;
|
||||
this.streamedChunks = 0;
|
||||
this.isStreamingFlag = false;
|
||||
this.isCompleteFlag = false;
|
||||
this.stream = undefined;
|
||||
this.controller = undefined;
|
||||
this.streamStartTs = 0;
|
||||
this.emitter.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { ChunkDecryptor } from "../decryptor/decryptor.types.js";
|
||||
import type { ChunkRateLimiter } from "../rate-limiter/adapter.types.js";
|
||||
import type { ChunkPersister } from "../storage-adapter/persister.types.js";
|
||||
import type { StreamingChunkQueueHooks } from "../types/types.js";
|
||||
import type { ChunkValidator } from "../validator/validator.types.js";
|
||||
|
||||
/** Constructor options for `StreamingChunkQueue`. */
|
||||
export interface StreamingChunkQueueOptions {
|
||||
readonly fileId: bigint;
|
||||
readonly totalChunks: bigint;
|
||||
readonly validator?: ChunkValidator;
|
||||
readonly decryptor?: ChunkDecryptor;
|
||||
readonly persister?: ChunkPersister;
|
||||
readonly rateLimiter?: ChunkRateLimiter;
|
||||
readonly hooks?: StreamingChunkQueueHooks;
|
||||
readonly autoStart?: boolean;
|
||||
readonly chunkTimeout?: number;
|
||||
readonly chunkPollInterval?: number;
|
||||
readonly clock?: () => number;
|
||||
readonly sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
# rate-limiter / adapter
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Normalises the `@suaveplan/rate-limiter` `TokenBucket` API onto
|
||||
the `ChunkRateLimiter` interface used internally by the streaming chunk
|
||||
queue. The `TokenBucket` does NOT throw on denial — it returns
|
||||
`{ allowed: boolean, retryAfter?: number }`. This adapter encodes the
|
||||
denial branch into the canonical `{ allowed: false, delay }` shape the
|
||||
queue understands, with a configurable fallback when `retryAfter` is
|
||||
missing.
|
||||
|
||||
## Features
|
||||
|
||||
- Wraps any structurally compatible `TokenBucketLike`.
|
||||
- Returns `{ allowed: true, delay: 0 }` on admission.
|
||||
- Returns `{ allowed: false, delay }` on denial, using `retryAfter` when
|
||||
positive or a configurable fallback (default 100 ms) otherwise.
|
||||
- Stateless beyond the bucket reference — safe to share across queues.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { TokenBucket } from "@suaveplan/rate-limiter";
|
||||
import { TokenBucketRateLimiterAdapter } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const bucket = new TokenBucket({ capacity: 1024 * 1024, refillRate: 1024 });
|
||||
const limiter = new TokenBucketRateLimiterAdapter(bucket);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
// Aggressive fallback for tightly-bound real-time streams.
|
||||
const limiter = new TokenBucketRateLimiterAdapter(bucket, 25);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `ChunkRateLimiter`
|
||||
|
||||
```ts
|
||||
interface ChunkRateLimiter {
|
||||
consume(bytes: number): { allowed: boolean; delay: number };
|
||||
}
|
||||
```
|
||||
|
||||
### `TokenBucketLike`
|
||||
|
||||
Structural subset of the `TokenBucket`:
|
||||
|
||||
```ts
|
||||
interface TokenBucketLike {
|
||||
consume(tokens?: number): { allowed: boolean; retryAfter?: number };
|
||||
}
|
||||
```
|
||||
|
||||
### `TokenBucketRateLimiterAdapter`
|
||||
|
||||
```ts
|
||||
new TokenBucketRateLimiterAdapter(
|
||||
bucket: TokenBucketLike,
|
||||
fallbackDelayMs?: number, // default 100
|
||||
)
|
||||
```
|
||||
|
||||
`consume(bytes)` calls `bucket.consume(bytes)` and normalises the
|
||||
result. The fallback delay is used only when the bucket denies AND
|
||||
omits / zeroes `retryAfter`.
|
||||
|
||||
### `DEFAULT_RATE_LIMIT_DELAY_MS`
|
||||
|
||||
The constant `100`. Exposed for callers who want to reuse the default
|
||||
in their own diagnostics.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The structural `TokenBucketLike` keeps `@suaveplan/rate-limiter` an
|
||||
optional peer concern at the type boundary — tests inject a fake
|
||||
bucket and never need to spin up the real one. The adapter does not
|
||||
sleep itself; the queue is in charge of timing because it owns the
|
||||
`onRateLimit` hook and the `rate:limited` event.
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
DEFAULT_RATE_LIMIT_DELAY_MS,
|
||||
TokenBucketRateLimiterAdapter,
|
||||
} from "./adapter.js";
|
||||
import type { TokenBucketLike } from "./adapter.types.js";
|
||||
|
||||
describe("TokenBucketRateLimiterAdapter", () => {
|
||||
it("forwards an allowed consumption with zero delay", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bucket: TokenBucketLike = { consume: () => ({ allowed: true }) };
|
||||
const adapter = new TokenBucketRateLimiterAdapter(bucket);
|
||||
expect(adapter.consume(100)).toEqual({ allowed: true, delay: 0 });
|
||||
});
|
||||
|
||||
it("uses retryAfter when the bucket denies the request", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bucket: TokenBucketLike = {
|
||||
consume: () => ({ allowed: false, retryAfter: 250 }),
|
||||
};
|
||||
const adapter = new TokenBucketRateLimiterAdapter(bucket);
|
||||
expect(adapter.consume(100)).toEqual({ allowed: false, delay: 250 });
|
||||
});
|
||||
|
||||
it("defers and retries when the bucket denies a request", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
let calls = 0;
|
||||
const bucket: TokenBucketLike = {
|
||||
consume: () => {
|
||||
calls += 1;
|
||||
return calls === 1
|
||||
? { allowed: false, retryAfter: 5 }
|
||||
: { allowed: true };
|
||||
},
|
||||
};
|
||||
const adapter = new TokenBucketRateLimiterAdapter(bucket);
|
||||
const first = adapter.consume(50);
|
||||
expect(first.allowed).toBe(false);
|
||||
expect(first.delay).toBe(5);
|
||||
const second = adapter.consume(50);
|
||||
expect(second.allowed).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to the default delay when retryAfter is omitted or zero", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const omitted: TokenBucketLike = {
|
||||
consume: () => ({ allowed: false }),
|
||||
};
|
||||
const zero: TokenBucketLike = {
|
||||
consume: () => ({ allowed: false, retryAfter: 0 }),
|
||||
};
|
||||
expect(
|
||||
new TokenBucketRateLimiterAdapter(omitted).consume(1).delay
|
||||
).toBe(DEFAULT_RATE_LIMIT_DELAY_MS);
|
||||
expect(new TokenBucketRateLimiterAdapter(zero).consume(1).delay).toBe(
|
||||
DEFAULT_RATE_LIMIT_DELAY_MS
|
||||
);
|
||||
});
|
||||
|
||||
it("honours a custom fallback delay", { timeout: 5000 }, () => {
|
||||
const bucket: TokenBucketLike = { consume: () => ({ allowed: false }) };
|
||||
const adapter = new TokenBucketRateLimiterAdapter(bucket, 42);
|
||||
expect(adapter.consume(1).delay).toBe(42);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,37 @@
|
||||
import type {
|
||||
ChunkRateLimiter,
|
||||
RateLimitConsumeOutcome,
|
||||
TokenBucketLike,
|
||||
} from "./adapter.types.js";
|
||||
|
||||
/** Default delay (ms) used when the bucket denies but supplies no `retryAfter`. */
|
||||
export const DEFAULT_RATE_LIMIT_DELAY_MS = 100;
|
||||
|
||||
/**
|
||||
* Adapts a `@suaveplan/rate-limiter` `TokenBucket` (or any structurally
|
||||
* compatible bucket) onto the queue's `ChunkRateLimiter` interface.
|
||||
*/
|
||||
export class TokenBucketRateLimiterAdapter implements ChunkRateLimiter {
|
||||
private readonly bucket: TokenBucketLike;
|
||||
private readonly fallbackDelayMs: number;
|
||||
|
||||
public constructor(
|
||||
bucket: TokenBucketLike,
|
||||
fallbackDelayMs: number = DEFAULT_RATE_LIMIT_DELAY_MS
|
||||
) {
|
||||
this.bucket = bucket;
|
||||
this.fallbackDelayMs = fallbackDelayMs;
|
||||
}
|
||||
|
||||
public consume(bytes: number): RateLimitConsumeOutcome {
|
||||
const result = this.bucket.consume(bytes);
|
||||
if (result.allowed) {
|
||||
return { allowed: true, delay: 0 };
|
||||
}
|
||||
const delay =
|
||||
result.retryAfter !== undefined && result.retryAfter > 0
|
||||
? result.retryAfter
|
||||
: this.fallbackDelayMs;
|
||||
return { allowed: false, delay };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/** Outcome of a rate-limit consumption attempt. */
|
||||
export interface RateLimitConsumeOutcome {
|
||||
readonly allowed: boolean;
|
||||
readonly delay: number;
|
||||
}
|
||||
|
||||
/** Strategy interface for the queue's rate-limit consumption call. */
|
||||
export interface ChunkRateLimiter {
|
||||
consume(bytes: number): RateLimitConsumeOutcome;
|
||||
}
|
||||
|
||||
/** Minimal structural view of `@suaveplan/rate-limiter`'s `TokenBucket`. */
|
||||
export interface TokenBucketLike {
|
||||
consume(tokens?: number): {
|
||||
readonly allowed: boolean;
|
||||
readonly retryAfter?: number;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
DEFAULT_RATE_LIMIT_DELAY_MS,
|
||||
TokenBucketRateLimiterAdapter,
|
||||
} from "./adapter.js";
|
||||
export type {
|
||||
ChunkRateLimiter,
|
||||
RateLimitConsumeOutcome,
|
||||
TokenBucketLike,
|
||||
} from "./adapter.types.js";
|
||||
@@ -0,0 +1,10 @@
|
||||
export {
|
||||
chunkStorageKey,
|
||||
chunkStoragePrefix,
|
||||
StorageChunkPersister,
|
||||
} from "./persister.js";
|
||||
export type {
|
||||
ChunkPersister,
|
||||
ChunkPersisterOptions,
|
||||
ChunkStorageLike,
|
||||
} from "./persister.types.js";
|
||||
@@ -0,0 +1,92 @@
|
||||
# storage-adapter / persister
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Bridges `@suaveplan/storage` `AbstractStorage` onto the queue's
|
||||
`ChunkPersister` contract. Persistence enables resuming an interrupted
|
||||
transfer: every received chunk is written through to a key-value
|
||||
backend (IndexedDB in browsers, file-backed stores on the server)
|
||||
keyed by `file:<fileId>:chunk:<chunkId>` so a fresh queue with the
|
||||
same `fileId` can hydrate via `loadAllChunks`.
|
||||
|
||||
## Features
|
||||
|
||||
- Deterministic key layout (`chunkStorageKey`, `chunkStoragePrefix`).
|
||||
- Persists chunk bytes alongside the metadata via the storage backend's
|
||||
optional `metadata` slot.
|
||||
- Loads a single chunk or lists every persisted chunk id for the
|
||||
configured `fileId`, in ascending order.
|
||||
- Structural `ChunkStorageLike` view so tests can substitute a Map.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { MemoryStorage } from "@suaveplan/storage";
|
||||
import { StorageChunkPersister } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const storage = new MemoryStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 12345n });
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { IndexedDBStorage } from "@suaveplan/storage";
|
||||
|
||||
const storage = await IndexedDBStorage.open("downloads", "chunks");
|
||||
const persister = new StorageChunkPersister({
|
||||
storage,
|
||||
fileId: 12345n,
|
||||
});
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 12345n,
|
||||
totalChunks: 100n,
|
||||
persister,
|
||||
});
|
||||
await queue.loadAllChunks();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `chunkStorageKey(fileId, chunkId)` → `string`
|
||||
|
||||
`file:<fileId>:chunk:<chunkId>`.
|
||||
|
||||
### `chunkStoragePrefix(fileId)` → `string`
|
||||
|
||||
`file:<fileId>:chunk:`.
|
||||
|
||||
### `ChunkPersister`
|
||||
|
||||
```ts
|
||||
interface ChunkPersister {
|
||||
persist(chunk: StoredChunk): Promise<void>;
|
||||
load(chunkId: bigint): Promise<Uint8Array | undefined>;
|
||||
listChunkIds(): Promise<bigint[]>;
|
||||
}
|
||||
```
|
||||
|
||||
### `StorageChunkPersister`
|
||||
|
||||
```ts
|
||||
new StorageChunkPersister({ storage, fileId });
|
||||
```
|
||||
|
||||
Implements `ChunkPersister` over any `ChunkStorageLike`. `listChunkIds`
|
||||
filters by prefix, parses the trailing segment as `bigint`, and sorts
|
||||
ascending — the ordering means
|
||||
`StreamingChunkQueue.loadAllChunks` ingests in the canonical chunk
|
||||
order.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`BigInt` is converted to `string` before being stored in storage
|
||||
metadata so the JSON wire form is portable. The persister does NOT
|
||||
attempt to delete entries — chunks are durable until the consumer
|
||||
explicitly clears the underlying storage. Keys with an empty trailing
|
||||
segment (`file:7:chunk:`) are silently dropped from `listChunkIds`
|
||||
because they cannot be parsed as a bigint.
|
||||
@@ -0,0 +1,103 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { ChunkMetadata, StoredChunk } from "../types/types.js";
|
||||
import {
|
||||
chunkStorageKey,
|
||||
chunkStoragePrefix,
|
||||
StorageChunkPersister,
|
||||
} from "./persister.js";
|
||||
import type { ChunkStorageLike } from "./persister.types.js";
|
||||
|
||||
function makeFakeStorage(): ChunkStorageLike & {
|
||||
map: Map<string, Uint8Array>;
|
||||
} {
|
||||
const map = new Map<string, Uint8Array>();
|
||||
return {
|
||||
map,
|
||||
async get(key) {
|
||||
return map.get(key);
|
||||
},
|
||||
async set(key, value) {
|
||||
map.set(key, value);
|
||||
},
|
||||
async list({ prefix } = {}) {
|
||||
const keys = Array.from(map.keys()).filter((k) =>
|
||||
prefix ? k.startsWith(prefix) : true
|
||||
);
|
||||
return { keys };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeMetadata(chunkId: bigint): ChunkMetadata {
|
||||
return {
|
||||
fileId: 7n,
|
||||
chunkId,
|
||||
totalChunks: 5n,
|
||||
size: 8,
|
||||
receivedAt: 1,
|
||||
validated: false,
|
||||
decrypted: false,
|
||||
};
|
||||
}
|
||||
|
||||
describe("chunkStorageKey + prefix helpers", () => {
|
||||
it("builds keys and prefixes deterministically", { timeout: 5000 }, () => {
|
||||
expect(chunkStorageKey(7n, 3n)).toBe("file:7:chunk:3");
|
||||
expect(chunkStoragePrefix(7n)).toBe("file:7:chunk:");
|
||||
});
|
||||
});
|
||||
|
||||
describe("StorageChunkPersister", () => {
|
||||
it("round-trips chunks through AbstractStorage", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const storage = makeFakeStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 7n });
|
||||
const chunk: StoredChunk = {
|
||||
data: new Uint8Array([1, 2, 3, 4]),
|
||||
metadata: makeMetadata(0n),
|
||||
};
|
||||
await persister.persist(chunk);
|
||||
expect(storage.map.get("file:7:chunk:0")).toEqual(chunk.data);
|
||||
const loaded = await persister.load(0n);
|
||||
expect(loaded).toEqual(chunk.data);
|
||||
const ids = await persister.listChunkIds();
|
||||
expect(ids).toEqual([0n]);
|
||||
});
|
||||
|
||||
it("returns undefined for missing chunks", { timeout: 5000 }, async () => {
|
||||
const storage = makeFakeStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 7n });
|
||||
expect(await persister.load(99n)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("lists chunk ids in ascending order", { timeout: 5000 }, async () => {
|
||||
const storage = makeFakeStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 7n });
|
||||
await persister.persist({
|
||||
data: new Uint8Array([1]),
|
||||
metadata: makeMetadata(2n),
|
||||
});
|
||||
await persister.persist({
|
||||
data: new Uint8Array([2]),
|
||||
metadata: makeMetadata(0n),
|
||||
});
|
||||
await persister.persist({
|
||||
data: new Uint8Array([3]),
|
||||
metadata: makeMetadata(1n),
|
||||
});
|
||||
const ids = await persister.listChunkIds();
|
||||
expect(ids).toEqual([0n, 1n, 2n]);
|
||||
});
|
||||
|
||||
it("ignores keys without a trailing chunk id", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const storage = makeFakeStorage();
|
||||
const persister = new StorageChunkPersister({ storage, fileId: 7n });
|
||||
storage.map.set("file:7:chunk:", new Uint8Array(0));
|
||||
storage.map.set("file:7:chunk:5", new Uint8Array(0));
|
||||
const ids = await persister.listChunkIds();
|
||||
expect(ids).toEqual([5n]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import type { StoredChunk } from "../types/types.js";
|
||||
import type {
|
||||
ChunkPersister,
|
||||
ChunkPersisterOptions,
|
||||
ChunkStorageLike,
|
||||
} from "./persister.types.js";
|
||||
|
||||
/** Build the canonical persistence key for a `(fileId, chunkId)` pair. */
|
||||
export function chunkStorageKey(fileId: bigint, chunkId: bigint): string {
|
||||
return `file:${fileId.toString()}:chunk:${chunkId.toString()}`;
|
||||
}
|
||||
|
||||
/** Build the chunk-id key prefix for a file. */
|
||||
export function chunkStoragePrefix(fileId: bigint): string {
|
||||
return `file:${fileId.toString()}:chunk:`;
|
||||
}
|
||||
|
||||
/** Storage-backed chunk persister over `AbstractStorage`. */
|
||||
export class StorageChunkPersister implements ChunkPersister {
|
||||
private readonly storage: ChunkStorageLike;
|
||||
private readonly fileId: bigint;
|
||||
|
||||
public constructor(options: ChunkPersisterOptions) {
|
||||
this.storage = options.storage;
|
||||
this.fileId = options.fileId;
|
||||
}
|
||||
|
||||
public async persist(chunk: StoredChunk): Promise<void> {
|
||||
const key = chunkStorageKey(this.fileId, chunk.metadata.chunkId);
|
||||
await this.storage.set(key, chunk.data, {
|
||||
metadata: {
|
||||
fileId: chunk.metadata.fileId.toString(),
|
||||
chunkId: chunk.metadata.chunkId.toString(),
|
||||
totalChunks: chunk.metadata.totalChunks.toString(),
|
||||
size: chunk.metadata.size,
|
||||
receivedAt: chunk.metadata.receivedAt,
|
||||
validated: chunk.metadata.validated,
|
||||
decrypted: chunk.metadata.decrypted,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
public async load(chunkId: bigint): Promise<Uint8Array | undefined> {
|
||||
return this.storage.get(chunkStorageKey(this.fileId, chunkId));
|
||||
}
|
||||
|
||||
public async listChunkIds(): Promise<bigint[]> {
|
||||
const { keys } = await this.storage.list({
|
||||
prefix: chunkStoragePrefix(this.fileId),
|
||||
});
|
||||
const out: bigint[] = [];
|
||||
for (const key of keys) {
|
||||
const trailing = key.split(":").pop();
|
||||
if (trailing !== undefined && trailing !== "") {
|
||||
out.push(BigInt(trailing));
|
||||
}
|
||||
}
|
||||
out.sort((a, b) => (a < b ? -1 : 1));
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { ChunkMetadata, StoredChunk } from "../types/types.js";
|
||||
|
||||
/** Structural view of `@suaveplan/storage` `AbstractStorage`. */
|
||||
export interface ChunkStorageLike {
|
||||
get(key: string): Promise<Uint8Array | undefined>;
|
||||
set(
|
||||
key: string,
|
||||
value: Uint8Array,
|
||||
options?: { metadata?: Record<string, unknown> }
|
||||
): Promise<void>;
|
||||
list(options?: { prefix?: string }): Promise<{ keys: string[] }>;
|
||||
}
|
||||
|
||||
/** Persister contract used by the queue. */
|
||||
export interface ChunkPersister {
|
||||
persist(chunk: StoredChunk): Promise<void>;
|
||||
load(chunkId: bigint): Promise<Uint8Array | undefined>;
|
||||
listChunkIds(): Promise<bigint[]>;
|
||||
}
|
||||
|
||||
export interface ChunkPersisterOptions {
|
||||
readonly storage: ChunkStorageLike;
|
||||
readonly fileId: bigint;
|
||||
}
|
||||
|
||||
export type { ChunkMetadata };
|
||||
@@ -0,0 +1,5 @@
|
||||
export type { StreamsChunkQueueTelemetry } from "./telemetry.js";
|
||||
export {
|
||||
getTelemetry,
|
||||
resetTelemetryForTest,
|
||||
} from "./telemetry.js";
|
||||
@@ -0,0 +1,79 @@
|
||||
# telemetry
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Exposes the OpenTelemetry instruments that the `StreamingChunkQueue`
|
||||
records during its lifecycle. Centralising instrument creation in one
|
||||
lazy singleton means the SDK can be configured after import without
|
||||
losing observations, and tests can swap the underlying meter via
|
||||
`resetTelemetryForTest`.
|
||||
|
||||
## Features
|
||||
|
||||
- One `meter` per process (`@suaveplan/streams-chunk-queue`).
|
||||
- Five counters (`chunks_received_total`, `chunks_streamed_total`,
|
||||
`validation_failures_total`, `decryption_failures_total`,
|
||||
`rate_limited_total`).
|
||||
- One histogram (`stream_duration_ms`) recorded when a transfer
|
||||
completes.
|
||||
- Lazy initialisation — instruments are created on first call to
|
||||
`getTelemetry()`.
|
||||
- `resetTelemetryForTest()` discards the cache so each test fixture
|
||||
observes a fresh holder.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { getTelemetry } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const t = getTelemetry();
|
||||
t.chunksReceived.add(1);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { getTelemetry, resetTelemetryForTest } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
beforeEach(() => resetTelemetryForTest());
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `getTelemetry()` → `StreamsChunkQueueTelemetry`
|
||||
|
||||
Returns the singleton holder of OTel instruments.
|
||||
|
||||
### `resetTelemetryForTest()` → `void`
|
||||
|
||||
Drops the cache so the next `getTelemetry()` call rebuilds a new
|
||||
holder. Intended for test isolation only.
|
||||
|
||||
### `StreamsChunkQueueTelemetry`
|
||||
|
||||
```ts
|
||||
interface StreamsChunkQueueTelemetry {
|
||||
chunksReceived: Counter;
|
||||
chunksStreamed: Counter;
|
||||
validationFailures: Counter;
|
||||
decryptionFailures: Counter;
|
||||
rateLimited: Counter;
|
||||
streamDuration: Histogram;
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
OTel's `metrics.getMeter` returns a no-op meter when no SDK is
|
||||
registered, which is the desired behaviour for unit tests and for
|
||||
applications that have not yet configured observability. The lazy
|
||||
cache ensures that a late `MeterProvider` registration is picked up the
|
||||
first time a chunk is received. The counters were chosen so a single
|
||||
dashboard tile (`chunks_received_total - chunks_streamed_total`)
|
||||
exposes the in-flight backlog without requiring a separate gauge. The
|
||||
histogram records on `stream:complete` only, so it never includes
|
||||
partial transfers that errored on timeout.
|
||||
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { getTelemetry, resetTelemetryForTest } from "./telemetry.js";
|
||||
|
||||
describe("getTelemetry", () => {
|
||||
it("creates instruments lazily and reuses the meter", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
resetTelemetryForTest();
|
||||
const first = getTelemetry();
|
||||
const second = getTelemetry();
|
||||
expect(second).toBe(first);
|
||||
expect(typeof first.chunksReceived.add).toBe("function");
|
||||
expect(typeof first.chunksStreamed.add).toBe("function");
|
||||
expect(typeof first.validationFailures.add).toBe("function");
|
||||
expect(typeof first.decryptionFailures.add).toBe("function");
|
||||
expect(typeof first.rateLimited.add).toBe("function");
|
||||
expect(typeof first.streamDuration.record).toBe("function");
|
||||
});
|
||||
|
||||
it("rebuilds the holder after resetTelemetryForTest()", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const first = getTelemetry();
|
||||
resetTelemetryForTest();
|
||||
const second = getTelemetry();
|
||||
expect(second).not.toBe(first);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { type Counter, type Histogram, metrics } from "@opentelemetry/api";
|
||||
|
||||
/** Holder for the lazily-initialised OTel instruments. */
|
||||
export interface StreamsChunkQueueTelemetry {
|
||||
readonly chunksReceived: Counter;
|
||||
readonly chunksStreamed: Counter;
|
||||
readonly validationFailures: Counter;
|
||||
readonly decryptionFailures: Counter;
|
||||
readonly rateLimited: Counter;
|
||||
readonly streamDuration: Histogram;
|
||||
}
|
||||
|
||||
let cached: StreamsChunkQueueTelemetry | undefined;
|
||||
|
||||
/** Returns the singleton telemetry holder, creating it on first call. */
|
||||
export function getTelemetry(): StreamsChunkQueueTelemetry {
|
||||
if (cached !== undefined) return cached;
|
||||
const meter = metrics.getMeter("@suaveplan/streams-chunk-queue");
|
||||
cached = {
|
||||
chunksReceived: meter.createCounter(
|
||||
"streams_chunk_queue.chunks_received_total",
|
||||
{
|
||||
description:
|
||||
"Chunks accepted by the queue (after duplicate filtering)",
|
||||
}
|
||||
),
|
||||
chunksStreamed: meter.createCounter(
|
||||
"streams_chunk_queue.chunks_streamed_total",
|
||||
{
|
||||
description: "Chunks emitted on the ReadableStream output",
|
||||
}
|
||||
),
|
||||
validationFailures: meter.createCounter(
|
||||
"streams_chunk_queue.validation_failures_total",
|
||||
{ description: "Chunk validation failures" }
|
||||
),
|
||||
decryptionFailures: meter.createCounter(
|
||||
"streams_chunk_queue.decryption_failures_total",
|
||||
{ description: "Chunk decryption failures" }
|
||||
),
|
||||
rateLimited: meter.createCounter(
|
||||
"streams_chunk_queue.rate_limited_total",
|
||||
{
|
||||
description: "Streaming pauses due to rate-limit denial",
|
||||
}
|
||||
),
|
||||
streamDuration: meter.createHistogram(
|
||||
"streams_chunk_queue.stream_duration_ms",
|
||||
{
|
||||
description:
|
||||
"Wall-clock time from first chunk to stream:complete",
|
||||
}
|
||||
),
|
||||
};
|
||||
return cached;
|
||||
}
|
||||
|
||||
/** Reset cached instruments. Intended for tests only. */
|
||||
export function resetTelemetryForTest(): void {
|
||||
cached = undefined;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export type {
|
||||
ChunkMetadata,
|
||||
ChunkRequest,
|
||||
StoredChunk,
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
StreamEventType,
|
||||
StreamingChunkQueueHooks,
|
||||
StreamingQueueStatus,
|
||||
} from "./types.js";
|
||||
@@ -0,0 +1,95 @@
|
||||
# types
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the shared types consumed across every feature dir of
|
||||
`@suaveplan/streams-chunk-queue`. Keeping them in one types-only file
|
||||
prevents circular imports between the queue, validator, decryptor,
|
||||
storage adapter, rate-limiter adapter, hooks, events, and telemetry
|
||||
modules. Every type lives behind a named interface (no inline object
|
||||
shapes) per suaveplan convention.
|
||||
|
||||
## Features
|
||||
|
||||
- `ChunkMetadata` — per-chunk metadata recorded on arrival.
|
||||
- `StoredChunk` — bytes plus metadata, with `data` mutable so validator
|
||||
and decryptor pipelines can rewrite it in-place.
|
||||
- `StreamEvent` — discriminated union covering every emitted event.
|
||||
- `StreamEventType` — concrete event names or the `*` wildcard.
|
||||
- `StreamingChunkQueueHooks` — eight lifecycle callbacks.
|
||||
- `StreamingQueueStatus` — frozen status snapshot.
|
||||
- `ChunkRequest` — decoded chunk-request frame.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type { ChunkMetadata } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
function logArrival(meta: ChunkMetadata): void {
|
||||
console.info(`got ${meta.size}B for chunk ${meta.chunkId}`);
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type {
|
||||
StreamEvent,
|
||||
StreamEventListener,
|
||||
} from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const listener: StreamEventListener = (event: StreamEvent) => {
|
||||
if (event.type === "rate:limited") {
|
||||
retryAfterTelemetry.observe(event.delay);
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `ChunkMetadata`
|
||||
|
||||
Immutable identifiers (`fileId`, `chunkId`, `totalChunks`, `receivedAt`,
|
||||
`size`) with mutable booleans `validated` and `decrypted` flipped by the
|
||||
validator / decryptor pipeline.
|
||||
|
||||
### `StoredChunk`
|
||||
|
||||
`{ data: Uint8Array; metadata: ChunkMetadata }`. The mutability of
|
||||
`data` is intentional — validator and decryptor adapters return new
|
||||
bytes that supersede the inline ciphertext.
|
||||
|
||||
### `StreamEvent`
|
||||
|
||||
Discriminated by `type`. Variants cover `chunk:received`,
|
||||
`chunk:validated`, `chunk:decrypted`, `chunk:streamed`,
|
||||
`stream:progress`, `stream:complete`, `stream:error`,
|
||||
`validation:failed`, `decryption:failed`, and `rate:limited`.
|
||||
|
||||
### `StreamingChunkQueueHooks`
|
||||
|
||||
Eight callbacks: `onChunkReceived`, `onChunkValidated`,
|
||||
`onChunkDecrypted`, `onChunkStreamed`, `onStreamProgress`,
|
||||
`onStreamComplete`, `onError`, `onRateLimit`. All may return `void` or
|
||||
`Promise<void>`; the queue awaits both forms.
|
||||
|
||||
### `StreamingQueueStatus`
|
||||
|
||||
Frozen snapshot returned by `StreamingChunkQueue.getStatus()`.
|
||||
|
||||
### `ChunkRequest`
|
||||
|
||||
Decoded form of the chunk-request frame produced by the binary protocol
|
||||
encoder.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The mutable `validated` / `decrypted` booleans are an intentional
|
||||
optimisation: it lets the queue keep a single map keyed by `chunkId`
|
||||
without allocating a parallel "state" map. Consumers MUST NOT mutate
|
||||
these fields directly — only the queue does so, exposing the result
|
||||
through the immutable `StreamingQueueStatus` snapshot.
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* Shared types for `@suaveplan/streams-chunk-queue`.
|
||||
*
|
||||
* Chunk identifiers are `bigint` (packer v2 u64). Sizes and timestamps
|
||||
* stay `number` because they fit comfortably in IEEE-754.
|
||||
*/
|
||||
|
||||
/** Metadata recorded for every received chunk. */
|
||||
export interface ChunkMetadata {
|
||||
readonly fileId: bigint;
|
||||
readonly chunkId: bigint;
|
||||
readonly totalChunks: bigint;
|
||||
readonly size: number;
|
||||
readonly receivedAt: number;
|
||||
validated: boolean;
|
||||
decrypted: boolean;
|
||||
}
|
||||
|
||||
/** Stored chunk entry (data + metadata). */
|
||||
export interface StoredChunk {
|
||||
data: Uint8Array;
|
||||
readonly metadata: ChunkMetadata;
|
||||
}
|
||||
|
||||
/** Discriminated union of chunk-queue events. */
|
||||
export type StreamEvent =
|
||||
| { type: "chunk:received"; chunkId: bigint; metadata: ChunkMetadata }
|
||||
| { type: "chunk:validated"; chunkId: bigint; metadata: ChunkMetadata }
|
||||
| { type: "chunk:decrypted"; chunkId: bigint; metadata: ChunkMetadata }
|
||||
| { type: "chunk:streamed"; chunkId: bigint; metadata: ChunkMetadata }
|
||||
| {
|
||||
type: "stream:progress";
|
||||
progress: number;
|
||||
totalChunks: bigint;
|
||||
receivedChunks: number;
|
||||
}
|
||||
| { type: "stream:complete"; totalChunks: bigint }
|
||||
| { type: "stream:error"; error: Error; chunkId?: bigint }
|
||||
| { type: "validation:failed"; chunkId: bigint; error: Error }
|
||||
| { type: "decryption:failed"; chunkId: bigint; error: Error }
|
||||
| { type: "rate:limited"; chunkId: bigint; delay: number };
|
||||
|
||||
export type StreamEventListener = (event: StreamEvent) => void;
|
||||
|
||||
export type StreamEventType = StreamEvent["type"] | "*";
|
||||
|
||||
export interface StreamingChunkQueueHooks {
|
||||
onChunkReceived?: (
|
||||
chunkId: bigint,
|
||||
metadata: ChunkMetadata
|
||||
) => void | Promise<void>;
|
||||
onChunkValidated?: (
|
||||
chunkId: bigint,
|
||||
metadata: ChunkMetadata
|
||||
) => void | Promise<void>;
|
||||
onChunkDecrypted?: (
|
||||
chunkId: bigint,
|
||||
metadata: ChunkMetadata
|
||||
) => void | Promise<void>;
|
||||
onChunkStreamed?: (
|
||||
chunkId: bigint,
|
||||
metadata: ChunkMetadata
|
||||
) => void | Promise<void>;
|
||||
onStreamProgress?: (
|
||||
progress: number,
|
||||
received: number,
|
||||
total: bigint
|
||||
) => void | Promise<void>;
|
||||
onStreamComplete?: (totalChunks: bigint) => void | Promise<void>;
|
||||
onError?: (error: Error, chunkId?: bigint) => void | Promise<void>;
|
||||
onRateLimit?: (chunkId: bigint, delay: number) => void | Promise<void>;
|
||||
}
|
||||
|
||||
export interface StreamingQueueStatus {
|
||||
readonly fileId: bigint;
|
||||
readonly totalChunks: bigint;
|
||||
readonly receivedChunks: number;
|
||||
readonly validatedChunks: number;
|
||||
readonly decryptedChunks: number;
|
||||
readonly streamedChunks: number;
|
||||
readonly progress: number;
|
||||
readonly isStreaming: boolean;
|
||||
readonly isComplete: boolean;
|
||||
readonly missingChunks: bigint[];
|
||||
readonly nextChunkId: bigint;
|
||||
}
|
||||
|
||||
export interface ChunkRequest {
|
||||
readonly fileId: bigint;
|
||||
readonly totalChunks: bigint;
|
||||
readonly chunkIds: bigint[];
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
# chunk-verifier-validator
|
||||
|
||||
## Purpose
|
||||
|
||||
Bridges the structural `ChunkVerifier` shape from `@suaveplan/packer`
|
||||
into the canonical `ChunkValidator` strategy. The packer's verifier
|
||||
returns a discriminated result (`{ ok: true, metadata }` or
|
||||
`{ ok: false, code, message }`); this adapter coerces the failure
|
||||
branch into the canonical `StreamsChunkQueueError(VALIDATION_FAILED)`
|
||||
with the packer's diagnostic preserved in `context.verifierCode`.
|
||||
|
||||
## Features
|
||||
|
||||
- Accepts any structurally compatible verifier (`ChunkVerifierLike`),
|
||||
so adapter tests don't need a real packer instance.
|
||||
- Preserves the verifier's `code` + `message` inside the typed error's
|
||||
`context` for downstream observability.
|
||||
- Returns the input bytes unchanged on success — `ChunkVerifier`
|
||||
validates the wire frame but does not modify it.
|
||||
- Falls back to a sensible default `code`/`message` if the verifier
|
||||
omits them, keeping the contract uniform.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { ChunkVerifier } from "@suaveplan/packer";
|
||||
import { ChunkVerifierValidator } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const verifier = new ChunkVerifier(serverPubKey);
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
validator: new ChunkVerifierValidator(verifier),
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type { ChunkVerifierLike } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const hsmVerifier: ChunkVerifierLike = {
|
||||
verify(bytes) {
|
||||
return hsmSignatureVerify(bytes)
|
||||
? { ok: true }
|
||||
: { ok: false, code: "HSM.REJECTED", message: "HSM rejected" };
|
||||
},
|
||||
};
|
||||
const validator = new ChunkVerifierValidator(hsmVerifier);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `ChunkVerifierLike`
|
||||
|
||||
```ts
|
||||
interface ChunkVerifierLike {
|
||||
verify(bytes: Uint8Array): {
|
||||
ok: boolean;
|
||||
code?: string;
|
||||
message?: string;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### `ChunkVerifierValidator`
|
||||
|
||||
```ts
|
||||
new ChunkVerifierValidator(verifier: ChunkVerifierLike)
|
||||
```
|
||||
|
||||
Implements `ChunkValidator`. On `ok: true` returns
|
||||
`{ data: <input bytes> }`. On `ok: false` rejects with
|
||||
`StreamsChunkQueueError(VALIDATION_FAILED)` whose
|
||||
`context.chunkId` is the offending chunk id and `context.verifierCode`
|
||||
carries the verifier's own code.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The adapter does not mutate `data` because `ChunkVerifier.verify` only
|
||||
checks the bytes — it does not strip a header. If a future verifier
|
||||
returns transformed bytes, the adapter contract can be extended without
|
||||
breaking callers (the success branch already returns `{ data }`). The
|
||||
adapter is intentionally synchronous on the verifier side (the wrapper
|
||||
itself returns a `Promise` to satisfy the strategy interface), so a
|
||||
fast in-process verifier does not allocate an extra microtask. The
|
||||
structural typing of `ChunkVerifierLike` means hardware-backed
|
||||
verifiers, mock test verifiers, and the real packer class are all
|
||||
interchangeable.
|
||||
@@ -0,0 +1,55 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { StreamsChunkQueueError } from "../errors/streams-chunk-queue-error.js";
|
||||
import { ChunkVerifierValidator } from "./chunk-verifier-validator.js";
|
||||
|
||||
describe("ChunkVerifierValidator", () => {
|
||||
it("returns the bytes unchanged when verifier reports ok", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const verifier = { verify: () => ({ ok: true as const }) };
|
||||
const adapter = new ChunkVerifierValidator(verifier);
|
||||
const data = new Uint8Array([7, 8, 9]);
|
||||
const result = await adapter.validate(0n, data);
|
||||
expect(result.data).toBe(data);
|
||||
});
|
||||
|
||||
it("surfaces verifier failure as StreamsChunkQueueError", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const verifier = {
|
||||
verify: () => ({
|
||||
ok: false as const,
|
||||
code: "PACKER.VERIFICATION_FAILED",
|
||||
message: "ed25519 signature verification failed",
|
||||
}),
|
||||
};
|
||||
const adapter = new ChunkVerifierValidator(verifier);
|
||||
await expect(
|
||||
adapter.validate(3n, new Uint8Array([0]))
|
||||
).rejects.toBeInstanceOf(StreamsChunkQueueError);
|
||||
try {
|
||||
await adapter.validate(3n, new Uint8Array([0]));
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.code).toBe("STREAMS_CHUNK_QUEUE.VALIDATION_FAILED");
|
||||
expect(err.context?.chunkId).toBe("3");
|
||||
expect(err.context?.verifierCode).toBe(
|
||||
"PACKER.VERIFICATION_FAILED"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to default reason when verifier omits message/code", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const verifier = { verify: () => ({ ok: false as const }) };
|
||||
const adapter = new ChunkVerifierValidator(verifier);
|
||||
try {
|
||||
await adapter.validate(0n, new Uint8Array(0));
|
||||
} catch (err) {
|
||||
if (!(err instanceof StreamsChunkQueueError)) throw err;
|
||||
expect(err.message).toContain("signature verification failed");
|
||||
expect(err.context?.verifierCode).toBe("verifier-failed");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import {
|
||||
STREAMS_CHUNK_QUEUE_CODES,
|
||||
StreamsChunkQueueError,
|
||||
} from "../errors/streams-chunk-queue-error.js";
|
||||
import type { ChunkValidator, ValidationOutcome } from "./validator.types.js";
|
||||
|
||||
/** Minimal structural view of `@suaveplan/packer`'s `ChunkVerifier`. */
|
||||
export interface ChunkVerifierLike {
|
||||
verify(chunkBytes: Uint8Array): {
|
||||
readonly ok: boolean;
|
||||
readonly code?: string;
|
||||
readonly message?: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Adapter that turns a `@suaveplan/packer` `ChunkVerifier` (or any
|
||||
* structurally compatible verifier) into a `ChunkValidator`.
|
||||
*/
|
||||
export class ChunkVerifierValidator implements ChunkValidator {
|
||||
private readonly verifier: ChunkVerifierLike;
|
||||
|
||||
public constructor(verifier: ChunkVerifierLike) {
|
||||
this.verifier = verifier;
|
||||
}
|
||||
|
||||
public validate(
|
||||
chunkId: bigint,
|
||||
data: Uint8Array
|
||||
): Promise<ValidationOutcome> {
|
||||
const result = this.verifier.verify(data);
|
||||
if (!result.ok) {
|
||||
const reason = result.message ?? "signature verification failed";
|
||||
const code = result.code ?? "verifier-failed";
|
||||
return Promise.reject(
|
||||
new StreamsChunkQueueError(
|
||||
STREAMS_CHUNK_QUEUE_CODES.VALIDATION_FAILED,
|
||||
`chunk ${chunkId} validation failed: ${reason}`,
|
||||
{
|
||||
context: {
|
||||
chunkId: chunkId.toString(),
|
||||
verifierCode: code,
|
||||
},
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
return Promise.resolve({ data });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
type ChunkVerifierLike,
|
||||
ChunkVerifierValidator,
|
||||
} from "./chunk-verifier-validator.js";
|
||||
export { NoopChunkValidator, noopChunkValidator } from "./validator.js";
|
||||
export type { ChunkValidator, ValidationOutcome } from "./validator.types.js";
|
||||
@@ -0,0 +1,93 @@
|
||||
# validator
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the `ChunkValidator` strategy interface and provides the
|
||||
`NoopChunkValidator` default. Validation is the second stage of the
|
||||
chunk pipeline (between ingestion and decryption). The strategy
|
||||
interface lets callers plug in any verification primitive — Ed25519
|
||||
signature checks, HMAC tags, custom integrity schemes — without the
|
||||
core queue needing to know the cryptographic details.
|
||||
|
||||
## Features
|
||||
|
||||
- `ChunkValidator` interface with a single async `validate` method.
|
||||
- `NoopChunkValidator` default that passes every chunk through
|
||||
unchanged.
|
||||
- `noopChunkValidator` singleton shorthand for callers that don't want
|
||||
to instantiate.
|
||||
- Adapters live in sibling modules (`chunk-verifier-validator.ts`).
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { noopChunkValidator } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
const queue = new StreamingChunkQueue({
|
||||
fileId: 1n,
|
||||
totalChunks: 5n,
|
||||
validator: noopChunkValidator,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type { ChunkValidator } from "@suaveplan/streams-chunk-queue";
|
||||
|
||||
class HmacValidator implements ChunkValidator {
|
||||
constructor(private readonly key: Uint8Array) {}
|
||||
async validate(chunkId: bigint, data: Uint8Array) {
|
||||
const tagLen = 32;
|
||||
const tag = data.slice(0, tagLen);
|
||||
const body = data.slice(tagLen);
|
||||
if (!(await hmacVerify(this.key, body, tag))) {
|
||||
throw new Error(`HMAC failed for chunk ${chunkId}`);
|
||||
}
|
||||
return { data: body };
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `ChunkValidator`
|
||||
|
||||
```ts
|
||||
interface ChunkValidator {
|
||||
validate(chunkId: bigint, data: Uint8Array): Promise<ValidationOutcome>;
|
||||
}
|
||||
```
|
||||
|
||||
Implementations MUST throw on failure. The thrown error is wrapped by
|
||||
the queue into `StreamsChunkQueueError(VALIDATION_FAILED)`.
|
||||
|
||||
### `ValidationOutcome`
|
||||
|
||||
`{ data: Uint8Array }`. The returned bytes supersede the inline
|
||||
ciphertext — useful when validation strips a header or signature.
|
||||
|
||||
### `NoopChunkValidator`
|
||||
|
||||
Concrete class implementing `ChunkValidator` as a pass-through.
|
||||
|
||||
### `noopChunkValidator`
|
||||
|
||||
Singleton instance of `NoopChunkValidator`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The interface is async even for synchronous validators because all
|
||||
real-world validators end up wanting WebCrypto / native crypto, which
|
||||
is async. Forcing the contract to be uniformly async means the queue
|
||||
can `await` without per-implementation branches. The `data` return
|
||||
value is intentionally writable — adapters may strip a signature
|
||||
header before passing bytes downstream, so the queue stores the
|
||||
adapter-returned bytes (not the original wire frame) in its in-memory
|
||||
map. A noop validator is included so the queue can call
|
||||
`validator.validate` unconditionally without checking for a
|
||||
configured strategy.
|
||||
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NoopChunkValidator, noopChunkValidator } from "./validator.js";
|
||||
|
||||
describe("NoopChunkValidator", () => {
|
||||
it("returns the input bytes unchanged", { timeout: 5000 }, async () => {
|
||||
const validator = new NoopChunkValidator();
|
||||
const data = new Uint8Array([1, 2, 3]);
|
||||
const result = await validator.validate(0n, data);
|
||||
expect(result.data).toBe(data);
|
||||
});
|
||||
|
||||
it("exposes a singleton shorthand", { timeout: 5000 }, async () => {
|
||||
const result = await noopChunkValidator.validate(0n, new Uint8Array(0));
|
||||
expect(result.data.byteLength).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { ChunkValidator, ValidationOutcome } from "./validator.types.js";
|
||||
|
||||
/**
|
||||
* Default validator that accepts every chunk unchanged. Used when the
|
||||
* queue is constructed without an explicit `validator` option, so the
|
||||
* downstream pipeline observes a uniform call shape.
|
||||
*/
|
||||
export class NoopChunkValidator implements ChunkValidator {
|
||||
public validate(
|
||||
_chunkId: bigint,
|
||||
data: Uint8Array
|
||||
): Promise<ValidationOutcome> {
|
||||
return Promise.resolve({ data });
|
||||
}
|
||||
}
|
||||
|
||||
/** Singleton shorthand. */
|
||||
export const noopChunkValidator: ChunkValidator = new NoopChunkValidator();
|
||||
@@ -0,0 +1,16 @@
|
||||
/**
|
||||
* Outcome of a chunk-validation run. On success the returned bytes
|
||||
* supersede the inline ciphertext (lets the adapter strip headers /
|
||||
* signatures before downstream stages see the data).
|
||||
*/
|
||||
export interface ValidationOutcome {
|
||||
readonly data: Uint8Array;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strategy interface for chunk validation (typically signature or MAC
|
||||
* verification). Implementations MUST throw on failure.
|
||||
*/
|
||||
export interface ChunkValidator {
|
||||
validate(chunkId: bigint, data: Uint8Array): Promise<ValidationOutcome>;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { webcrypto } from "node:crypto";
|
||||
|
||||
type WritableGlobal = { crypto?: Crypto };
|
||||
const g = globalThis as WritableGlobal;
|
||||
if (g.crypto === undefined) {
|
||||
g.crypto = webcrypto as unknown as Crypto;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "dist", "node_modules"]
|
||||
}
|
||||
40
packages/foundation/flow/streams-chunk-queue/vite.config.ts
Normal file
40
packages/foundation/flow/streams-chunk-queue/vite.config.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/streams-chunk-queue",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/types",
|
||||
"@suaveplan/storage",
|
||||
"@suaveplan/rate-limiter",
|
||||
"@suaveplan/packer",
|
||||
"@opentelemetry/api",
|
||||
"tslib",
|
||||
"zod",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user