Initial Commit
This commit is contained in:
285
openspec/archive/2026-04-24-enterprise-etl/proposal.md
Normal file
285
openspec/archive/2026-04-24-enterprise-etl/proposal.md
Normal file
@@ -0,0 +1,285 @@
|
||||
# OpenSpec Change: enterprise-etl
|
||||
|
||||
- **Change ID**: enterprise-etl
|
||||
- **Package**: `@suaveplan/etl`
|
||||
- **Category**: universal
|
||||
- **Target path**: `universal/data/etl`
|
||||
- **Status**: PROPOSED
|
||||
- **Date**: 2026-04-24
|
||||
- **Reference spec**: `/mnt/s/web/suaveplan/enterprise/etl.md`
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Deliver `@suaveplan/etl` on the genesis branch from scratch at enterprise quality. The package provides extract-transform-load pipeline primitives: typed `Source`, `Transform`, and `Sink` stages composable into declarative pipelines with built-in backpressure, `AbortSignal` cancellation, structured errors, OTel instrumentation, retry/circuit-breaker on IO paths, and property-based correctness guarantees. Genesis builds on `@suaveplan/streams` for the underlying stream substrate and delegates serialization to `@suaveplan/serialize`.
|
||||
|
||||
---
|
||||
|
||||
## Current-state audit (genesis)
|
||||
|
||||
The package does not exist on genesis yet — fresh build. No prior source, no test results, no coverage data.
|
||||
|
||||
---
|
||||
|
||||
## Available workspace deps on genesis
|
||||
|
||||
| Package | Role |
|
||||
|---|---|
|
||||
| `@suaveplan/error` (`workspace:^`) | Base class for `EtlError`; `defineCodes` for `ETL.*` codes |
|
||||
| `@suaveplan/types` (`workspace:^`) | `LoggerLike`, `Clock`, `EventBusLike` injection interfaces |
|
||||
| `@suaveplan/streams` (`workspace:^`) | WHATWG-Streams substrate for pipeline stages |
|
||||
| `@suaveplan/serialize` (`workspace:^`) | Chunk serialization / deserialization for file and network sinks |
|
||||
| `@suaveplan/retry` (`workspace:^`) | Retry + circuit-breaker wrappers for IO source/sink adapters |
|
||||
| `@suaveplan/events` (`workspace:^`) | Progress and lifecycle event emission |
|
||||
| `zod` (`catalog:forms`) | Zod schemas for all public option types and record contracts |
|
||||
| `@opentelemetry/api` (`catalog:observability`) | Spans, counters, histograms on pipeline run, stage durations |
|
||||
| `tslib` (`catalog:build`) | Runtime helpers |
|
||||
|
||||
Dev deps: `vitest`, `@vitest/coverage-istanbul`, `fast-check`, `mitata` (all `catalog:testing`).
|
||||
|
||||
---
|
||||
|
||||
## Public API surface
|
||||
|
||||
### Core pipeline
|
||||
|
||||
```ts
|
||||
// Pipeline orchestrator
|
||||
function createPipeline<TIn, TOut>(options: PipelineOptions<TIn, TOut>): Pipeline<TIn, TOut>
|
||||
|
||||
interface Pipeline<TIn, TOut> {
|
||||
run(signal?: AbortSignal): Promise<PipelineResult<TOut>>
|
||||
pipe<TNext>(transform: Transform<TOut, TNext>): Pipeline<TIn, TNext>
|
||||
}
|
||||
|
||||
interface PipelineResult<T> {
|
||||
records: T[]
|
||||
stats: PipelineStats
|
||||
errors: EtlError[]
|
||||
}
|
||||
|
||||
interface PipelineStats {
|
||||
extracted: number
|
||||
transformed: number
|
||||
loaded: number
|
||||
skipped: number
|
||||
durationMs: number
|
||||
}
|
||||
|
||||
interface PipelineOptions<TIn, TOut> {
|
||||
source: Source<TIn>
|
||||
transforms?: Transform<TIn, unknown>[]
|
||||
sink: Sink<TOut>
|
||||
batchSize?: number
|
||||
concurrency?: number
|
||||
onError?: ErrorHandler
|
||||
logger?: LoggerLike
|
||||
clock?: Clock
|
||||
}
|
||||
```
|
||||
|
||||
### Source interface
|
||||
|
||||
```ts
|
||||
interface Source<T> {
|
||||
readonly name: string
|
||||
open(signal?: AbortSignal): Promise<void>
|
||||
read(): AsyncGenerator<T>
|
||||
close(): Promise<void>
|
||||
}
|
||||
|
||||
// Built-in sources
|
||||
function arraySource<T>(items: T[]): Source<T>
|
||||
function generatorSource<T>(gen: () => AsyncGenerator<T>, name?: string): Source<T>
|
||||
function streamSource<T>(stream: ReadableStream<T>, name?: string): Source<T>
|
||||
```
|
||||
|
||||
### Transform interface
|
||||
|
||||
```ts
|
||||
interface Transform<TIn, TOut> {
|
||||
readonly name: string
|
||||
transform(record: TIn, ctx: TransformContext): Promise<TOut | typeof SKIP>
|
||||
}
|
||||
|
||||
const SKIP: unique symbol
|
||||
|
||||
interface TransformContext {
|
||||
index: number
|
||||
signal?: AbortSignal
|
||||
logger?: LoggerLike
|
||||
}
|
||||
|
||||
// Built-in transforms
|
||||
function mapTransform<TIn, TOut>(fn: (record: TIn) => TOut | Promise<TOut>, name?: string): Transform<TIn, TOut>
|
||||
function filterTransform<T>(predicate: (record: T) => boolean | Promise<boolean>, name?: string): Transform<T, T>
|
||||
function validateTransform<T>(schema: ZodSchema<T>, name?: string): Transform<unknown, T>
|
||||
function batchTransform<T>(size: number): Transform<T, T[]>
|
||||
function flatMapTransform<TIn, TOut>(fn: (record: TIn) => TOut[] | Promise<TOut[]>, name?: string): Transform<TIn, TOut>
|
||||
```
|
||||
|
||||
### Sink interface
|
||||
|
||||
```ts
|
||||
interface Sink<T> {
|
||||
readonly name: string
|
||||
open(signal?: AbortSignal): Promise<void>
|
||||
write(record: T): Promise<void>
|
||||
flush(): Promise<void>
|
||||
close(): Promise<void>
|
||||
}
|
||||
|
||||
// Built-in sinks
|
||||
function arraySink<T>(): Sink<T> & { records: T[] }
|
||||
function nullSink<T>(): Sink<T>
|
||||
function transformSink<T>(fn: (record: T) => void | Promise<void>, name?: string): Sink<T>
|
||||
```
|
||||
|
||||
### Error handling
|
||||
|
||||
```ts
|
||||
class EtlError extends SuaveplanError<EtlErrorCode>
|
||||
|
||||
type EtlErrorCode =
|
||||
| "ETL_SOURCE_FAILED"
|
||||
| "ETL_TRANSFORM_FAILED"
|
||||
| "ETL_SINK_FAILED"
|
||||
| "ETL_PIPELINE_ABORTED"
|
||||
| "ETL_VALIDATION_FAILED"
|
||||
| "ETL_BATCH_OVERFLOW"
|
||||
|
||||
type ErrorHandler = (error: EtlError, record: unknown, stage: string) => ErrorAction
|
||||
type ErrorAction = "skip" | "abort" | "retry"
|
||||
|
||||
function registerEtlErrorCodes(): void
|
||||
```
|
||||
|
||||
### Zod option schemas
|
||||
|
||||
```ts
|
||||
const PipelineOptionsSchema: ZodSchema
|
||||
const SourceOptionsSchema: ZodSchema
|
||||
const TransformOptionsSchema: ZodSchema
|
||||
const SinkOptionsSchema: ZodSchema
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## New additions for genesis
|
||||
|
||||
### 1. Streams integration
|
||||
|
||||
Pipeline stages are backed by `@suaveplan/streams` `ReadableStream` / `TransformStream` / `WritableStream` internally. The public API remains stage-based but the execution engine uses WHATWG streams for backpressure propagation.
|
||||
|
||||
### 2. Retry/circuit-breaker on IO paths
|
||||
|
||||
`Source.open`, `Source.read`, `Sink.write`, and `Sink.flush` are wrapped by `@suaveplan/retry` strategies when a `retryOptions` field is provided in source/sink options.
|
||||
|
||||
### 3. OTel instrumentation
|
||||
|
||||
- `pipeline.run` — span `etl.pipeline.run`, attributes: `etl.source`, `etl.sink`, `etl.batch_size`
|
||||
- Per-stage spans: `etl.source.read`, `etl.transform.<name>`, `etl.sink.write`
|
||||
- Counters: `etl.records_extracted`, `etl.records_transformed`, `etl.records_loaded`, `etl.records_skipped`, `etl.errors_total`
|
||||
- Histogram: `etl.stage_duration_ms` (attribute: `etl.stage`)
|
||||
|
||||
### 4. Progress events via `@suaveplan/events`
|
||||
|
||||
When a `bus` is provided in `PipelineOptions`, the pipeline emits typed events:
|
||||
- `etl:pipeline:started` — `{ pipelineId, source, sink }`
|
||||
- `etl:stage:completed` — `{ stage, records, durationMs }`
|
||||
- `etl:pipeline:completed` — `{ stats }`
|
||||
- `etl:pipeline:failed` — `{ error }`
|
||||
|
||||
### 5. Property-based tests (fast-check)
|
||||
|
||||
- `mapTransform` composition: `pipe(f, g)(x) ≡ g(f(x))` for all pure functions
|
||||
- `filterTransform` idempotency: applying the same filter twice yields the same result
|
||||
- `batchTransform` completeness: total records across all batches equals input count
|
||||
- Pipeline abort: any mid-flight `AbortSignal` cancellation stops processing within one batch
|
||||
|
||||
### 6. Benchmark suite (`src/__benchmarks__/`)
|
||||
|
||||
- Array source → map transform → array sink, 10k records
|
||||
- Array source → validate transform → array sink, 10k records
|
||||
- Batch transform with batch size 100 vs 1000
|
||||
|
||||
---
|
||||
|
||||
## File layout
|
||||
|
||||
```text
|
||||
universal/data/etl/
|
||||
package.json
|
||||
tsconfig.json
|
||||
vitest.config.ts
|
||||
src/
|
||||
index.ts
|
||||
error-codes.ts
|
||||
constants/
|
||||
index.ts
|
||||
types/
|
||||
common.ts
|
||||
common.md
|
||||
core/
|
||||
pipeline/
|
||||
pipeline.ts
|
||||
pipeline.md
|
||||
pipeline.types.ts
|
||||
pipeline.test.ts
|
||||
index.ts
|
||||
stages/
|
||||
source/
|
||||
source.ts
|
||||
source.md
|
||||
source.types.ts
|
||||
source.test.ts
|
||||
index.ts
|
||||
transform/
|
||||
transform.ts
|
||||
transform.md
|
||||
transform.types.ts
|
||||
transform.test.ts
|
||||
index.ts
|
||||
sink/
|
||||
sink.ts
|
||||
sink.md
|
||||
sink.types.ts
|
||||
sink.test.ts
|
||||
index.ts
|
||||
adapters/
|
||||
array-source/
|
||||
array-source.ts
|
||||
array-source.md
|
||||
array-source.test.ts
|
||||
index.ts
|
||||
stream-source/
|
||||
stream-source.ts
|
||||
stream-source.md
|
||||
stream-source.test.ts
|
||||
index.ts
|
||||
array-sink/
|
||||
array-sink.ts
|
||||
array-sink.md
|
||||
array-sink.test.ts
|
||||
index.ts
|
||||
utils/
|
||||
retry-adapter.ts
|
||||
retry-adapter.md
|
||||
retry-adapter.test.ts
|
||||
__benchmarks__/
|
||||
pipeline.bench.ts
|
||||
tests/
|
||||
integration.test.ts
|
||||
property.test.ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Impact
|
||||
|
||||
**Downstream packages**: `@suaveplan/etl` is consumed by any data-migration, import/export, or batch-processing feature. No existing genesis package depends on it yet.
|
||||
|
||||
**Workspace dep prerequisites**: `@suaveplan/error`, `@suaveplan/types`, `@suaveplan/streams`, `@suaveplan/serialize`, `@suaveplan/retry`, `@suaveplan/events` must all be published to Verdaccio before this package can be built.
|
||||
|
||||
**Breaking changes**: none (new package).
|
||||
89
openspec/archive/2026-04-24-enterprise-etl/tasks.md
Normal file
89
openspec/archive/2026-04-24-enterprise-etl/tasks.md
Normal file
@@ -0,0 +1,89 @@
|
||||
# Tasks: enterprise-etl
|
||||
|
||||
- **Change ID**: enterprise-etl
|
||||
- **Package**: `@suaveplan/etl`
|
||||
- **Date**: 2026-04-24
|
||||
|
||||
## Prerequisites (must be published to Verdaccio first)
|
||||
|
||||
- [x] `@suaveplan/error`
|
||||
- [x] `@suaveplan/types`
|
||||
- [x] `@suaveplan/streams`
|
||||
- [x] `@suaveplan/serialize`
|
||||
- [x] `@suaveplan/retry`
|
||||
- [x] `@suaveplan/events`
|
||||
|
||||
## Tasks
|
||||
|
||||
### 1. Scaffold package
|
||||
|
||||
- [x] Create `universal/data/etl/` directory tree per file layout in `proposal.md`
|
||||
- [x] Write `package.json` (name `@suaveplan/etl`, version `0.1.0`, `sideEffects: false`, correct workspace deps and catalog externals)
|
||||
- [x] Write `tsconfig.json` extending `@suaveplan/typescript-config/library.json`
|
||||
- [x] Write `vitest.config.ts` using `createTestConfig()` from `@suaveplan/testing/config`
|
||||
- [x] Add package to root `turbo.json` pipeline if not already covered by glob
|
||||
|
||||
### 2. Error codes (`src/error-codes.ts`)
|
||||
|
||||
- [x] Define `EtlErrorCode` union type
|
||||
- [x] Call `defineCodes` to register all `ETL.*` codes with severity and retryable flags
|
||||
- [x] Export `registerEtlErrorCodes()` factory
|
||||
- [x] Export `EtlError extends SuaveplanError<EtlErrorCode>`
|
||||
|
||||
### 3. Types (`src/types/common.ts`)
|
||||
|
||||
- [x] Define and export all public interfaces: `Pipeline`, `PipelineOptions`, `PipelineResult`, `PipelineStats`, `Source`, `Transform`, `TransformContext`, `Sink`, `ErrorHandler`, `ErrorAction`
|
||||
- [x] Export `SKIP` unique symbol
|
||||
- [x] Write `common.md`
|
||||
|
||||
### 4. Stage implementations
|
||||
|
||||
- [x] Implement `Source` interface and built-in sources (`arraySource`, `generatorSource`, `streamSource`) in `src/stages/source/`
|
||||
- [x] Implement `Transform` interface and built-in transforms (`mapTransform`, `filterTransform`, `validateTransform`, `batchTransform`, `flatMapTransform`) in `src/stages/transform/`
|
||||
- [x] Implement `Sink` interface and built-in sinks (`arraySink`, `nullSink`, `transformSink`) in `src/stages/sink/`
|
||||
- [x] Each feature file has co-located `.md`, `.types.ts`, `.test.ts`, `index.ts` barrel
|
||||
|
||||
### 5. Pipeline orchestrator (`src/core/pipeline/`)
|
||||
|
||||
- [x] Implement `createPipeline` with stage chaining, batch processing, `AbortSignal` propagation, and configurable `ErrorHandler`
|
||||
- [x] Integrate `@suaveplan/streams` `ReadableStream`/`TransformStream`/`WritableStream` internally
|
||||
- [x] Emit progress events via `@suaveplan/events` bus when configured
|
||||
- [x] Write `pipeline.md`, `pipeline.types.ts`, `pipeline.test.ts`, `index.ts`
|
||||
|
||||
### 6. Retry adapter (`src/utils/retry-adapter.ts`)
|
||||
|
||||
- [x] Wrap source/sink IO methods with `@suaveplan/retry` when `retryOptions` is provided
|
||||
- [x] Write `retry-adapter.md` and `retry-adapter.test.ts`
|
||||
|
||||
### 7. OTel instrumentation
|
||||
|
||||
- [x] Add spans, counters, histograms per `proposal.md` §New additions §3
|
||||
|
||||
### 8. Tests
|
||||
|
||||
- [x] Unit tests co-located with each feature module (100% coverage target)
|
||||
- [x] `tests/integration.test.ts` — full pipeline: array source → validate → map → array sink
|
||||
- [x] `tests/property.test.ts` — fast-check properties per `proposal.md` §5
|
||||
|
||||
### 9. Benchmarks (`src/__benchmarks__/pipeline.bench.ts`)
|
||||
|
||||
- [x] Implement benchmark cases per `proposal.md` §6
|
||||
|
||||
### 10. Docs
|
||||
|
||||
- [x] Every `.ts` implementation file has a sibling `.md` with all required sections (≥200 words, both examples)
|
||||
- [x] Root `README.md` with feature table, usage examples, API reference links
|
||||
|
||||
### 11. Pre-publish gate
|
||||
|
||||
- [x] `bunx turbo lint typecheck test --filter=@suaveplan/etl` green
|
||||
- [x] `bunx turbo test:coverage --filter=@suaveplan/etl` ≥ 100% on touched modules
|
||||
|
||||
### 12. Publish
|
||||
|
||||
- [x] `bun scripts/publish.ts --package @suaveplan/etl`
|
||||
|
||||
### 13. Archive
|
||||
|
||||
- [x] Fold spec delta into `openspec/specs/data/spec.md`
|
||||
- [x] `mv openspec/changes/enterprise-etl openspec/archive/2026-04-24-enterprise-etl`
|
||||
Reference in New Issue
Block a user