Initial Commit

openspec/archive/2026-04-24-enterprise-streams/proposal.md

@@ -0,0 +1,332 @@
+# OpenSpec Change: enterprise-streams
+
+- **Change ID**: enterprise-streams
+- **Package**: `@suaveplan/streams`
+- **Category**: universal
+- **Target path**: `universal/streams/streams`
+- **Status**: PROPOSED
+- **Date**: 2026-04-24
+- **Reference spec**: `/mnt/s/web/suaveplan/enterprise/streams.md`
+
+---
+
+## Summary
+
+Deliver `@suaveplan/streams` on the genesis branch from scratch at enterprise quality. This is the foundational stream-processing package: WHATWG-Streams-based primitives, backpressure-first transforms, zero-copy `Uint8Array` operations where possible, `AbortSignal` integration end-to-end, structured errors, OTel instrumentation, and property-based invariant proofs. All other Suaveplan packages that need pipelined data processing, file handling, network body processing, or event bridging build on this substrate.
+
+---
+
+## Current-state audit (genesis)
+
+The package does not exist on genesis yet — fresh build. No prior source, no test results, no coverage data.
+
+---
+
+## Available workspace deps on genesis
+
+| Package | Role |
+|---|---|
+| `@suaveplan/error` (`workspace:^`) | Base class for `StreamError`; `defineCodes` for `STREAM.*` codes |
+| `@suaveplan/types` (`workspace:^`) | `LoggerLike`, `Clock` injection interfaces |
+| `zod` (`catalog:forms`) | Zod schemas for stream configuration types |
+| `@opentelemetry/api` (`catalog:observability`) | Pipeline spans, chunk-count / byte-count histograms per stage |
+| `tslib` (`catalog:build`) | Runtime helpers |
+
+Dev deps: `vitest`, `@vitest/coverage-istanbul`, `fast-check`, `mitata` (all `catalog:testing`).
+
+---
+
+## Public API surface
+
+### Stream creation utilities
+
+```ts
+// Readable stream factories
+function fromArray<T>(items: T[], options?: ReadableOptions): ReadableStream<T>
+function fromAsyncGenerator<T>(gen: () => AsyncGenerator<T>, options?: ReadableOptions): ReadableStream<T>
+function fromBytes(data: Uint8Array, chunkSize?: number): ReadableStream<Uint8Array>
+
+// Byte stream utilities
+function concat(streams: ReadableStream<Uint8Array>[]): ReadableStream<Uint8Array>
+function slice(stream: ReadableStream<Uint8Array>, start: number, end?: number): ReadableStream<Uint8Array>
+
+interface ReadableOptions {
+  highWaterMark?: number
+  signal?: AbortSignal
+}
+```
+
+### Transform factories
+
+```ts
+// Data transforms
+function mapStream<TIn, TOut>(fn: (chunk: TIn) => TOut | Promise<TOut>): TransformStream<TIn, TOut>
+function filterStream<T>(predicate: (chunk: T) => boolean | Promise<boolean>): TransformStream<T, T>
+function flatMapStream<TIn, TOut>(fn: (chunk: TIn) => TOut[] | Promise<TOut[]>): TransformStream<TIn, TOut>
+function batchStream<T>(size: number): TransformStream<T, T[]>
+function tapStream<T>(fn: (chunk: T) => void | Promise<void>): TransformStream<T, T>
+function limitStream<T>(max: number): TransformStream<T, T>
+
+// Byte transforms
+function encodeTextStream(encoding?: string): TransformStream<string, Uint8Array>
+function decodeTextStream(encoding?: string): TransformStream<Uint8Array, string>
+function compressStream(format: "gzip" | "deflate"): TransformStream<Uint8Array, Uint8Array>
+function decompressStream(format: "gzip" | "deflate"): TransformStream<Uint8Array, Uint8Array>
+
+interface TransformOptions {
+  highWaterMark?: number
+  signal?: AbortSignal
+}
+```
+
+### Sink utilities
+
+```ts
+// Consumption
+function toArray<T>(stream: ReadableStream<T>, signal?: AbortSignal): Promise<T[]>
+function toBytes(stream: ReadableStream<Uint8Array>, signal?: AbortSignal): Promise<Uint8Array>
+function toString(stream: ReadableStream<Uint8Array>, encoding?: string): Promise<string>
+function forEach<T>(stream: ReadableStream<T>, fn: (chunk: T) => void | Promise<void>, signal?: AbortSignal): Promise<void>
+function toWritable<T>(stream: ReadableStream<T>, writable: WritableStream<T>, signal?: AbortSignal): Promise<void>
+```
+
+### Pipeline composition
+
+```ts
+// Type-safe pipeline builder
+function pipeline<T>(source: ReadableStream<T>): PipelineBuilder<T>
+
+interface PipelineBuilder<T> {
+  pipe<TNext>(transform: TransformStream<T, TNext>): PipelineBuilder<TNext>
+  pipe<TNext>(fn: (chunk: T) => TNext | Promise<TNext>): PipelineBuilder<TNext>
+  collect(signal?: AbortSignal): Promise<T[]>
+  drain(writable: WritableStream<T>, signal?: AbortSignal): Promise<void>
+  forEach(fn: (chunk: T) => void | Promise<void>, signal?: AbortSignal): Promise<void>
+}
+
+// Parallel fan-out
+function fanOut<T>(stream: ReadableStream<T>, n: number): ReadableStream<T>[]
+
+// Merge fan-in
+function merge<T>(streams: ReadableStream<T>[]): ReadableStream<T>
+```
+
+### Node.js interop (server-only adapter)
+
+```ts
+// Available under @suaveplan/streams/node subpath
+function fromNodeReadable(stream: NodeReadable): ReadableStream<Uint8Array>
+function toNodeReadable(stream: ReadableStream<Uint8Array>): NodeReadable
+function fromNodeWritable(stream: NodeWritable): WritableStream<Uint8Array>
+```
+
+### Instrumented pipeline
+
+```ts
+// Wraps any transform with OTel span + counters
+function instrumentTransform<TIn, TOut>(
+  transform: TransformStream<TIn, TOut>,
+  name: string,
+  options?: InstrumentOptions
+): TransformStream<TIn, TOut>
+
+interface InstrumentOptions {
+  tracer?: Tracer
+  meter?: Meter
+  attributes?: Record<string, string>
+}
+```
+
+### Error handling
+
+```ts
+class StreamError extends SuaveplanError<StreamErrorCode>
+
+type StreamErrorCode =
+  | "STREAM_ABORTED"
+  | "STREAM_BACKPRESSURE_EXCEEDED"
+  | "STREAM_CHUNK_INVALID"
+  | "STREAM_PIPE_FAILED"
+  | "STREAM_TRANSFORM_FAILED"
+  | "STREAM_SOURCE_EXHAUSTED"
+
+function registerStreamErrorCodes(): void
+```
+
+### Zod schemas
+
+```ts
+const ReadableOptionsSchema: ZodSchema
+const TransformOptionsSchema: ZodSchema
+const InstrumentOptionsSchema: ZodSchema
+```
+
+---
+
+## New additions for genesis
+
+### 1. Backpressure-first design
+
+All transforms use WHATWG Streams `highWaterMark` and queuing strategies. `batchStream` accumulates into a bounded internal buffer; when the buffer is full it applies downstream backpressure by not pulling from the source.
+
+### 2. Zero-copy byte path
+
+`concat`, `slice`, `compressStream`, and `decompressStream` operate on `Uint8Array` views without intermediate copying where the underlying platform supports it (Bun and modern V8).
+
+### 3. AbortSignal propagation
+
+Every async operation (`toArray`, `toBytes`, `forEach`, `drain`, `pipeline.*`) accepts an `AbortSignal`. On abort, open streams are cancelled and pending promises reject with `StreamError("STREAM_ABORTED")`.
+
+### 4. OTel instrumentation
+
+- `instrumentTransform` wraps any named transform with: span `stream.transform.<name>`, counter `stream.chunks_total` (attribute: `stream.stage`), histogram `stream.chunk_bytes` (byte streams only), histogram `stream.transform_duration_ms`
+- `pipeline` builder instruments each piped stage automatically when a tracer is provided via `PipelineOptions`.
+
+### 5. Property-based tests (fast-check)
+
+- `toArray(fromArray(items)) ≡ items` for arbitrary arrays
+- `batchStream` completeness: flattening all batches yields the original sequence
+- `mapStream` composition: `pipe(map(f), map(g)) ≡ map(x => g(f(x)))` for pure functions
+- `concat` byte order: concatenating two byte arrays via streams yields the same bytes as `Uint8Array` concat
+- AbortSignal: cancelling mid-flight halts processing and releases all locks
+
+### 6. Benchmark suite (`src/__benchmarks__/`)
+
+- `fromArray` → `mapStream` → `toArray`, 100k items
+- `fromBytes` → `batchStream(1024)` → `toArray`, 10MB
+- `compressStream` + `decompressStream` round-trip, 1MB
+
+---
+
+## File layout
+
+```text
+universal/streams/streams/
+  package.json
+  tsconfig.json
+  vitest.config.ts
+  src/
+    index.ts
+    error-codes.ts
+    constants/
+      index.ts
+    types/
+      common.ts
+      common.md
+    readable/
+      from-array/
+        from-array.ts
+        from-array.md
+        from-array.test.ts
+        index.ts
+      from-generator/
+        from-generator.ts
+        from-generator.md
+        from-generator.test.ts
+        index.ts
+      from-bytes/
+        from-bytes.ts
+        from-bytes.md
+        from-bytes.test.ts
+        index.ts
+    transforms/
+      map-stream/
+        map-stream.ts
+        map-stream.md
+        map-stream.test.ts
+        index.ts
+      filter-stream/
+        filter-stream.ts
+        filter-stream.md
+        filter-stream.test.ts
+        index.ts
+      batch-stream/
+        batch-stream.ts
+        batch-stream.md
+        batch-stream.test.ts
+        index.ts
+      flat-map-stream/
+        flat-map-stream.ts
+        flat-map-stream.md
+        flat-map-stream.test.ts
+        index.ts
+      tap-stream/
+        tap-stream.ts
+        tap-stream.md
+        tap-stream.test.ts
+        index.ts
+      limit-stream/
+        limit-stream.ts
+        limit-stream.md
+        limit-stream.test.ts
+        index.ts
+      text-codec/
+        text-codec.ts
+        text-codec.md
+        text-codec.test.ts
+        index.ts
+      compress/
+        compress.ts
+        compress.md
+        compress.test.ts
+        index.ts
+    sinks/
+      to-array/
+        to-array.ts
+        to-array.md
+        to-array.test.ts
+        index.ts
+      to-bytes/
+        to-bytes.ts
+        to-bytes.md
+        to-bytes.test.ts
+        index.ts
+      for-each/
+        for-each.ts
+        for-each.md
+        for-each.test.ts
+        index.ts
+    pipeline/
+      pipeline-builder/
+        pipeline-builder.ts
+        pipeline-builder.md
+        pipeline-builder.types.ts
+        pipeline-builder.test.ts
+        index.ts
+      fan-out/
+        fan-out.ts
+        fan-out.md
+        fan-out.test.ts
+        index.ts
+      merge/
+        merge.ts
+        merge.md
+        merge.test.ts
+        index.ts
+    instrument/
+      instrument-transform/
+        instrument-transform.ts
+        instrument-transform.md
+        instrument-transform.types.ts
+        instrument-transform.test.ts
+        index.ts
+    node/
+      node-adapters.ts
+      node-adapters.md
+      node-adapters.test.ts
+    __benchmarks__/
+      streams.bench.ts
+  tests/
+    integration.test.ts
+    property.test.ts
+```
+
+---
+
+## Impact
+
+**Downstream packages**: `@suaveplan/streams-metadata`, `@suaveplan/streams-chunk-queue`, `@suaveplan/etl`, and any future package doing pipelined data processing.
+
+**Workspace dep prerequisites**: `@suaveplan/error`, `@suaveplan/types` must be published to Verdaccio before this package can be built.
+
+**Breaking changes**: none (new package).