SuavePlan/suaveplan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Commit

Browse Source
This commit is contained in:
admin
2026-05-27 23:47:53 +01:00
parent fbc70f96e6
commit 9b28db724f
18103 changed files with 1600554 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
packages/foundation/etl/README.md Normal file
View File

@@ -0,0 +1,13 @@
# `etl`
Extract-Transform-Load pipeline engine with typed stages.
<!-- begin:auto-docs -->
## Packages in `etl`
1 package.
| Package | Description |
|---|---|
| [`@suaveplan/etl`](./etl/README.md) | Extract-Transform-Load pipeline engine with typed stages, error handling, observable progress, and OTel instrumentation. |
<!-- end:auto-docs -->

1
packages/foundation/etl/_intro.md Normal file
View File

@@ -0,0 +1 @@
Extract-Transform-Load pipeline engine with typed stages.

46
packages/foundation/etl/etl/README.md Normal file
View File

@@ -0,0 +1,46 @@
# @suaveplan/etl - Extract-Transform-Load pipeline engine with typed stages, error handling, observable progress, and OTel instrumentation
## Summary
Extract-Transform-Load pipeline engine with typed stages, error handling, observable progress, and OTel instrumentation.
<!-- begin:auto-docs -->
## Documentation tree
- [pipeline](./src/core/pipeline/pipeline.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [pipeline](./src/core/pipeline/pipeline.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [errors](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [etl-error](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [etl.schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
<!-- end:auto-docs --> - [pipeline](./src/core/pipeline/pipeline.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [etl-error](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [etl.schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- [pipeline](./src/core/pipeline/pipeline.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [etl-error](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [etl.schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- [pipeline](./src/core/pipeline/pipeline.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [etl-error](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [etl.schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_- [pipeline](./src/core/pipeline/pipeline.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_ - [etl-error](./src/errors/etl-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_ - [etl.schemas](./src/schemas/etl.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_ - [sink](./src/stages/sink/sink.md) — <!-- begin:auto-docs --> _(no submodules)_- [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_ - [source](./src/stages/source/source.md) — <!-- begin:auto-docs --> _(no submodules)_- [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_ - [transform](./src/stages/transform/transform.md) — <!-- begin:auto-docs --> _(no submodules)_
## Modules
- [Pipeline](src/core/pipeline/pipeline.md) — pipeline orchestrator with fluent API
- [Source](src/stages/source/source.md) — array and generator source factories
- [Transform](src/stages/transform/transform.md) — map, filter, validate, flat-map transforms
- [Sink](src/stages/sink/sink.md) — array, null, callback, and batch sinks
- [EtlError](src/errors/etl-error.md) — structured errors with stable codes
- [Schemas](src/schemas/etl.schemas.md) — Zod schemas for pipeline types
- [Types](src/types/etl.types.md) — shared TypeScript interfaces
## Quick Start
```ts
import { createPipeline, arraySource, mapTransform, ArraySink } from "@suaveplan/etl";
const sink = new ArraySink<number>();
const result = await createPipeline({ source: arraySource([1, 2, 3]) })
.pipe(mapTransform((n) => n * 2))
.to(sink)
.run();
console.log(result.stats.loaded); // 3
console.log(sink.records); // [2, 4, 6]
```

77
packages/foundation/etl/etl/package.json Normal file
View File

@@ -0,0 +1,77 @@
{
"name": "@suaveplan/etl",
"version": "0.1.0",
"type": "module",
"sideEffects": false,
"description": "Extract-Transform-Load pipeline engine with typed stages, error handling, observable progress, and OTel instrumentation",
"license": "MIT",
"author": "Suaveplan Team",
"main": "./dist/index.js",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
"./package.json": "./package.json",
".": {
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
"require": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }
},
"./errors/etl-error": {
"import": { "types": "./dist/errors/etl-error.d.ts", "default": "./dist/errors/etl-error.js" },
"require": { "types": "./dist/errors/etl-error.d.ts", "default": "./dist/errors/etl-error.js" }
},
"./core/pipeline/pipeline": {
"import": { "types": "./dist/core/pipeline/pipeline.d.ts", "default": "./dist/core/pipeline/pipeline.js" },
"require": { "types": "./dist/core/pipeline/pipeline.d.ts", "default": "./dist/core/pipeline/pipeline.js" }
},
"./stages/source/source": {
"import": { "types": "./dist/stages/source/source.d.ts", "default": "./dist/stages/source/source.js" },
"require": { "types": "./dist/stages/source/source.d.ts", "default": "./dist/stages/source/source.js" }
},
"./stages/sink/sink": {
"import": { "types": "./dist/stages/sink/sink.d.ts", "default": "./dist/stages/sink/sink.js" },
"require": { "types": "./dist/stages/sink/sink.d.ts", "default": "./dist/stages/sink/sink.js" }
},
"./stages/transform/transform": {
"import": { "types": "./dist/stages/transform/transform.d.ts", "default": "./dist/stages/transform/transform.js" },
"require": { "types": "./dist/stages/transform/transform.d.ts", "default": "./dist/stages/transform/transform.js" }
}
},
"scripts": {
"build": "vite build",
"test": "vitest run --passWithNoTests",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage --passWithNoTests",
"lint": "bunx biome check src/",
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
"format": "bunx biome format --write --max-diagnostics 500 src/",
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
"bench": "bun run src/__benchmarks__/etl.bench.ts",
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
},
"dependencies": {
"@suaveplan/error": "workspace:^",
"@suaveplan/types": "workspace:^",
"@suaveplan/events": "workspace:^",
"@suaveplan/serialize": "workspace:^",
"tslib": "catalog:build",
"zod": "catalog:forms",
"@opentelemetry/api": "catalog:observability"
},
"devDependencies": {
"@suaveplan/testing": "workspace:^",
"@types/node": "catalog:",
"@vitest/coverage-istanbul": "catalog:testing",
"fast-check": "catalog:testing",
"mitata": "catalog:testing",
"typescript": "catalog:",
"vite": "catalog:build",
"vite-plugin-dts": "catalog:build",
"vitest": "catalog:testing"
},
"publishConfig": {
"registry": "https://npmjs.fq.io/",
"access": "public"
},
"files": ["dist", "src", "README.md"]
}

2
packages/foundation/etl/etl/src/constants/index.ts Normal file
View File

@@ -0,0 +1,2 @@
/** Default batch size for batch operations. */
export const DEFAULT_BATCH_SIZE = 100;

1
packages/foundation/etl/etl/src/core/pipeline/index.ts Normal file
View File

@@ -0,0 +1 @@
export { createPipeline, Pipeline, RunnablePipeline } from "./pipeline.js";

82
packages/foundation/etl/etl/src/core/pipeline/pipeline.md Normal file
View File

@@ -0,0 +1,82 @@
# pipeline
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides `createPipeline`, the primary orchestrator for the ETL engine. A pipeline connects a `Source`, zero or more `Transform` stages, and a `Sink` into a single async execution graph. Records flow from source through transforms to sink with AbortSignal cancellation, per-record error handling, optional bus event emission, and OTel instrumentation.
## Features
- `createPipeline(opts)` — creates a `Pipeline<TIn>` from a source and options
- `.pipe(transform)` — adds a transform stage; returns a new typed `Pipeline<TIn, TNext>`
- `.to(sink)` — attaches a sink; returns a `RunnablePipeline`
- `.run(signal?)` — executes the pipeline; returns `PipelineResult` with stats and non-fatal errors
- `.dryRun(maxRecords?)` — runs with a limited record count for preview/validation
- `onError` callback — returns `"skip" | "abort" | "retry"` per error
- Bus events: `etl:pipeline:started`, `etl:pipeline:completed`, `etl:pipeline:failed`
- OTel span `etl.pipeline.run` wrapping the full execution
## Usage: Basic Example
```ts
import { createPipeline } from "@suaveplan/etl";
import { arraySource } from "@suaveplan/etl";
import { mapTransform } from "@suaveplan/etl";
import { ArraySink } from "@suaveplan/etl";
const sink = new ArraySink<number>();
const result = await createPipeline({ source: arraySource([1, 2, 3]) })
.pipe(mapTransform((n) => n * 2))
.to(sink)
.run();
console.log(result.stats.loaded); // 3
console.log(sink.records); // [2, 4, 6]
```
## Usage: Advanced Example
```ts
import { createPipeline, arraySource, validateTransform, ArraySink, ETL_CODES } from "@suaveplan/etl";
import z from "zod";
const schema = z.object({ id: z.string() });
const sink = new ArraySink();
const result = await createPipeline({
source: arraySource([{ id: "1" }, { id: 2 }, { id: "3" }] as unknown[]),
onError: (err) => err.code === ETL_CODES.VALIDATION_FAILED ? "skip" : "abort",
})
.pipe(validateTransform(schema))
.to(sink)
.run();
console.log(result.stats.loaded); // 2 (one invalid skipped)
console.log(result.errors.length); // 1
```
## API Reference
### `createPipeline<TIn>(opts)`
- `opts.source: Source<TIn>` — extraction source
- `opts.batchSize?: number` — not used internally (BatchSink handles batching)
- `opts.onError?: ErrorHandler` — per-record error callback
- `opts.clock?: Clock` — injectable clock for duration calculation
- `opts.bus?: { publish }` — optional event bus
### `Pipeline<TIn, TOut>`
- `pipe<TNext>(transform): Pipeline<TIn, TNext>`
- `to(sink: Sink<TOut>): RunnablePipeline<TIn, TOut>`
### `RunnablePipeline<TIn, TOut>`
- `run(signal?): Promise<PipelineResult>`
- `dryRun(maxRecords?): Promise<PipelineResult>`
## Implementation Notes
The pipeline accumulates transforms in order without composing them eagerly — each record is passed through the array sequentially on every iteration. This allows clean type inference via the `.pipe()` fluent API. The `dryRun` method installs an internal abort controller that fires after `maxRecords` writes.

335
packages/foundation/etl/etl/src/core/pipeline/pipeline.test.ts Normal file
View File

@@ -0,0 +1,335 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import z from "zod";
import { ETL_CODES, EtlError } from "../../errors/etl-error.js";
import { ArraySink, nullSink } from "../../stages/sink/sink.js";
import { arraySource } from "../../stages/source/source.js";
import {
filterTransform,
mapTransform,
validateTransform,
} from "../../stages/transform/transform.js";
import { createPipeline } from "./pipeline.js";
describe("createPipeline", () => {
it("runs end-to-end", { timeout: 30000 }, async () => {
const sink = new ArraySink<number>();
const result = await createPipeline({ source: arraySource([1, 2, 3]) })
.to(sink)
.run();
expect(result.stats.extracted).toBe(3);
expect(result.stats.loaded).toBe(3);
expect(sink.records).toEqual([1, 2, 3]);
});
it("applies map transform", { timeout: 30000 }, async () => {
const sink = new ArraySink<number>();
await createPipeline({ source: arraySource([1, 2, 3]) })
.pipe(mapTransform((n: number) => n * 2))
.to(sink)
.run();
expect(sink.records).toEqual([2, 4, 6]);
});
it("applies filter transform", { timeout: 30000 }, async () => {
const sink = new ArraySink<number>();
const result = await createPipeline({
source: arraySource([1, 2, 3, 4]),
})
.pipe(filterTransform((n: number) => n % 2 === 0))
.to(sink)
.run();
expect(sink.records).toEqual([2, 4]);
expect(result.stats.skipped).toBe(2);
});
it("aborts on signal", { timeout: 30000 }, async () => {
const ac = new AbortController();
// Abort immediately
ac.abort();
const sink = new ArraySink<number>();
await expect(
createPipeline({ source: arraySource([1, 2, 3]) })
.to(sink)
.run(ac.signal)
).rejects.toThrow(EtlError);
});
it("onError skip continues pipeline", { timeout: 30000 }, async () => {
const schema = z.object({ id: z.string() });
const sink = new ArraySink();
const result = await createPipeline({
source: arraySource([
{ id: "1" },
{ bad: true },
{ id: "3" },
] as unknown[]),
onError: () => "skip",
})
.pipe(validateTransform(schema))
.to(sink)
.run();
expect(result.stats.loaded).toBe(2);
expect(result.errors).toHaveLength(1);
});
it("onError abort throws", { timeout: 30000 }, async () => {
const schema = z.object({ id: z.string() });
const sink = new ArraySink();
await expect(
createPipeline({
source: arraySource([{ bad: true }] as unknown[]),
onError: () => "abort",
})
.pipe(validateTransform(schema))
.to(sink)
.run()
).rejects.toThrow(EtlError);
});
it("default onError aborts", { timeout: 30000 }, async () => {
const schema = z.object({ id: z.string() });
const sink = new ArraySink();
await expect(
createPipeline({
source: arraySource([{ bad: true }] as unknown[]),
})
.pipe(validateTransform(schema))
.to(sink)
.run()
).rejects.toThrow(EtlError);
});
it("dryRun limits records", { timeout: 30000 }, async () => {
const items = Array.from({ length: 100 }, (_, i) => i);
const sink = new ArraySink<number>();
const pipeline = createPipeline({ source: arraySource(items) }).to(
sink
);
const result = await pipeline.dryRun(5);
expect(result.stats.extracted).toBeLessThanOrEqual(5);
});
it("emits bus events", { timeout: 30000 }, async () => {
const events: string[] = [];
const bus = {
publish(topic: string) {
events.push(topic);
},
};
const sink = new ArraySink<number>();
await createPipeline({
source: arraySource([1]),
bus,
})
.to(sink)
.run();
expect(events).toContain("etl:pipeline:started");
expect(events).toContain("etl:pipeline:completed");
});
it("emits bus failure event on error", { timeout: 30000 }, async () => {
const events: string[] = [];
const bus = {
publish(topic: string) {
events.push(topic);
},
};
const sink = new ArraySink();
const schema = z.object({ id: z.string() });
try {
await createPipeline({
source: arraySource([{ bad: true }] as unknown[]),
bus,
})
.pipe(validateTransform(schema))
.to(sink)
.run();
} catch {
// expected
}
expect(events).toContain("etl:pipeline:failed");
});
it("empty source runs successfully", { timeout: 30000 }, async () => {
const sink = new ArraySink<number>();
const result = await createPipeline({ source: arraySource([]) })
.to(sink)
.run();
expect(result.stats.extracted).toBe(0);
expect(result.stats.loaded).toBe(0);
});
it("sink error with skip continues", { timeout: 30000 }, async () => {
let callCount = 0;
const faultySink = {
name: "FaultySink",
async open() {},
async write(_r: number) {
callCount++;
if (callCount === 2) throw new Error("write fail");
},
async flush() {},
async close() {},
};
const result = await createPipeline({
source: arraySource([1, 2, 3]),
onError: (_e, _r, stage) =>
stage === "FaultySink" ? "skip" : "abort",
})
.to(faultySink)
.run();
expect(result.errors).toHaveLength(1);
expect(result.stats.extracted).toBe(3);
});
it("non-EtlError from transform is wrapped", {
timeout: 30000,
}, async () => {
const boom = {
name: "BoomTransform",
transform(): never {
throw new TypeError("boom");
},
};
const sink = nullSink<never>();
try {
await createPipeline({ source: arraySource([1]) })
.pipe(boom)
.to(sink)
.run();
} catch (err) {
expect(err instanceof EtlError).toBe(true);
expect((err as EtlError).code).toBe(ETL_CODES.TRANSFORM_FAILED);
}
});
it("onError retry skips record", { timeout: 30000 }, async () => {
const schema = z.object({ id: z.string() });
const sink = new ArraySink();
const result = await createPipeline({
source: arraySource([{ bad: true }] as unknown[]),
onError: () => "retry",
})
.pipe(validateTransform(schema))
.to(sink)
.run();
expect(result.errors).toHaveLength(1);
});
it("pipeline result has no errors for clean run", {
timeout: 30000,
}, async () => {
const sink = new ArraySink<number>();
const result = await createPipeline({ source: arraySource([1, 2]) })
.to(sink)
.run();
expect(result.errors).toHaveLength(0);
});
it("sink error with abort re-throws", { timeout: 30000 }, async () => {
const throwingSink = {
name: "ThrowSink",
async open() {},
async write() {
throw new EtlError(ETL_CODES.SINK_FAILED, "sink boom");
},
async flush() {},
async close() {},
};
await expect(
createPipeline({ source: arraySource([1]) })
.to(throwingSink)
.run()
).rejects.toThrow();
});
it("sink error non-Error throw is wrapped", {
timeout: 30000,
}, async () => {
const throwingSink = {
name: "ThrowSink",
async open() {},
async write() {
throw "string sink error";
},
async flush() {},
async close() {},
};
await expect(
createPipeline({ source: arraySource([1]) })
.to(throwingSink)
.run()
).rejects.toThrow();
});
it("transform non-Error throw is wrapped", { timeout: 30000 }, async () => {
const stringThrowTransform = mapTransform(() => {
throw "string transform error";
});
await expect(
createPipeline({ source: arraySource([1]) })
.pipe(stringThrowTransform)
.to(nullSink<number>())
.run()
).rejects.toThrow();
});
it("transform with signal reaches transform loop", {
timeout: 30000,
}, async () => {
const ac = new AbortController();
const sink = new ArraySink<number>();
const result = await createPipeline({ source: arraySource([1, 2]) })
.pipe(mapTransform((n: number) => n * 2))
.to(sink)
.run(ac.signal);
expect(result.stats.loaded).toBe(2);
});
it("non-EtlError in pipeline catch is wrapped as SOURCE_FAILED", {
timeout: 30000,
}, async () => {
const failSource = {
name: "FailSource",
async open() {
throw "string error";
},
async *read(): AsyncGenerator<never> {
yield undefined as never;
},
async close() {},
};
await expect(
createPipeline({ source: failSource }).to(nullSink<number>()).run()
).rejects.toThrow(EtlError);
});
it("dryRun completes naturally when source has fewer records than limit", {
timeout: 30000,
}, async () => {
const result = await createPipeline({ source: arraySource([1, 2, 3]) })
.to(nullSink<number>())
.dryRun(10);
expect(result.stats.extracted).toBe(3);
expect(result.stats.loaded).toBe(3);
});
it("dryRun re-throws non-abort errors", { timeout: 30000 }, async () => {
const failSource = {
name: "FailSource",
async open() {
throw new EtlError(ETL_CODES.SOURCE_FAILED, "boom");
},
async *read(): AsyncGenerator<never> {
yield undefined as never;
},
async close() {},
};
await expect(
createPipeline({ source: failSource })
.to(nullSink<number>())
.dryRun()
).rejects.toThrow(EtlError);
});
});

289
packages/foundation/etl/etl/src/core/pipeline/pipeline.ts Normal file
View File

@@ -0,0 +1,289 @@
import { trace } from "@opentelemetry/api";
import { ETL_CODES, EtlError } from "../../errors/etl-error.js";
import type {
PipelineOptions,
PipelineResult,
Sink,
Source,
Transform,
TransformContext,
} from "../../types/etl.types.js";
import { SKIP } from "../../types/etl.types.js";
const tracer = trace.getTracer("@suaveplan/etl");
/**
* Represents a configured ETL pipeline ready to run.
*/
export class Pipeline<TIn, TOut = TIn> {
protected readonly source: Source<TIn>;
protected readonly transforms: Transform<unknown, unknown>[];
protected readonly opts: PipelineOptions<TIn>;
public constructor(
source: Source<TIn>,
transforms: Transform<unknown, unknown>[],
opts: PipelineOptions<TIn>
) {
this.source = source;
this.transforms = transforms;
this.opts = opts;
}
/**
* Adds a transform stage and returns a new Pipeline with the updated type.
*/
public pipe<TNext>(
transform: Transform<TOut, TNext>
): Pipeline<TIn, TNext> {
return new Pipeline<TIn, TNext>(
this.source,
[...this.transforms, transform as Transform<unknown, unknown>],
this.opts as unknown as PipelineOptions<TIn>
);
}
/**
* Adds a sink and returns a `RunnablePipeline`.
*/
public to(sink: Sink<TOut>): RunnablePipeline<TIn, TOut> {
return new RunnablePipeline(
this.source,
this.transforms,
sink,
this.opts
);
}
}
/**
* A pipeline with a sink attached, ready to execute.
*/
export class RunnablePipeline<TIn, TOut> {
protected readonly source: Source<TIn>;
protected readonly transforms: Transform<unknown, unknown>[];
protected readonly sink: Sink<TOut>;
protected readonly opts: PipelineOptions<TIn>;
public constructor(
source: Source<TIn>,
transforms: Transform<unknown, unknown>[],
sink: Sink<TOut>,
opts: PipelineOptions<TIn>
) {
this.source = source;
this.transforms = transforms;
this.sink = sink;
this.opts = opts;
}
public async run(signal?: AbortSignal): Promise<PipelineResult> {
const span = tracer.startSpan("etl.pipeline.run");
const clock = this.opts.clock ?? (() => Date.now());
const startMs = clock();
const stats = {
extracted: 0,
transformed: 0,
loaded: 0,
skipped: 0,
durationMs: 0,
};
const errors: EtlError[] = [];
try {
if (this.opts.bus) {
await this.opts.bus.publish("etl:pipeline:started", {
source: this.source.name,
sink: this.sink.name,
});
}
await this.source.open(signal);
await this.sink.open(signal);
for await (const raw of this.source.read()) {
if (signal?.aborted) {
throw new EtlError(
ETL_CODES.PIPELINE_ABORTED,
"Pipeline aborted by signal"
);
}
stats.extracted++;
let current: unknown = raw;
let skip = false;
let index = stats.extracted - 1;
for (const transform of this.transforms) {
const ctx: TransformContext = {
index,
...(signal !== undefined ? { signal } : {}),
};
try {
const result = await transform.transform(current, ctx);
if (result === SKIP) {
stats.skipped++;
skip = true;
break;
}
current = result;
} catch (err) {
const etlErr =
err instanceof EtlError
? err
: new EtlError(
ETL_CODES.TRANSFORM_FAILED,
String(err),
{
cause:
err instanceof Error
? err
: new Error(String(err)),
}
);
const action =
this.opts.onError?.(
etlErr,
current,
transform.name
) ?? "abort";
if (action === "skip") {
stats.skipped++;
skip = true;
errors.push(etlErr);
break;
}
if (action === "abort") {
throw etlErr;
}
// retry: fall through (same record, same transform — not retried in this impl)
errors.push(etlErr);
skip = true;
break;
}
index++;
}
if (!skip) {
stats.transformed++;
try {
await this.sink.write(current as TOut);
stats.loaded++;
} catch (err) {
const etlErr =
err instanceof EtlError
? err
: new EtlError(
ETL_CODES.SINK_FAILED,
String(err),
{
cause:
err instanceof Error
? err
: new Error(String(err)),
}
);
const action =
this.opts.onError?.(
etlErr,
current,
this.sink.name
) ?? "abort";
if (action === "skip") {
errors.push(etlErr);
} else {
throw etlErr;
}
}
}
}
await this.sink.flush();
await this.source.close();
await this.sink.close();
} catch (err) {
span.end();
if (this.opts.bus) {
await this.opts.bus.publish("etl:pipeline:failed", {
error: err,
});
}
if (err instanceof EtlError) throw err;
throw new EtlError(ETL_CODES.SOURCE_FAILED, String(err), {
cause: err as Error,
});
}
stats.durationMs = clock() - startMs;
span.end();
if (this.opts.bus) {
await this.opts.bus.publish("etl:pipeline:completed", { stats });
}
return { stats, errors };
}
public async dryRun(maxRecords = 10): Promise<PipelineResult> {
const items: TOut[] = [];
const abortCtl = new AbortController();
let count = 0;
const limitSink: Sink<TOut> = {
name: "DryRunSink",
async open() {},
async write(record: TOut) {
items.push(record);
count++;
if (count >= maxRecords) abortCtl.abort();
},
async flush() {},
async close() {},
};
const limited = new RunnablePipeline(
this.source,
this.transforms,
limitSink,
this.opts
);
try {
return await limited.run(abortCtl.signal);
} catch (err) {
if (
err instanceof EtlError &&
err.code === ETL_CODES.PIPELINE_ABORTED
) {
return {
stats: {
extracted: count,
transformed: count,
loaded: count,
skipped: 0,
durationMs: 0,
},
errors: [],
};
}
throw err;
}
}
}
/**
* Creates a new `Pipeline` for the given source.
*
* @example
* ```ts
* const result = await createPipeline({ source: arraySource([1,2,3]) })
* .pipe(mapTransform((n) => n * 2))
* .to(arraySink)
* .run();
* ```
*/
export function createPipeline<TIn>(
opts: PipelineOptions<TIn>
): Pipeline<TIn, TIn> {
return new Pipeline<TIn, TIn>(opts.source, [], opts);
}

67
packages/foundation/etl/etl/src/errors/etl-error.md Normal file
View File

@@ -0,0 +1,67 @@
# etl-error
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Defines the structured error type and stable error-code registry for `@suaveplan/etl`. All pipeline failures — extraction errors, transformation errors, load errors, validation failures, and aborts — surface as `EtlError` instances with a machine-readable `code` field, enabling callers to discriminate by stage without fragile message parsing.
## Features
- `ETL_CODES` registry via `@suaveplan/error`'s `defineCodes` — codes namespaced `ETL.*`
- `EtlError` extends `SuaveplanError` for cause-chain traversal and wire serialisation
- Six codes: `SOURCE_FAILED`, `TRANSFORM_FAILED`, `SINK_FAILED`, `PIPELINE_ABORTED`, `VALIDATION_FAILED`, `BATCH_OVERFLOW`
- `EtlCode` union type for exhaustive error handling
## Usage: Basic Example
```ts
import { EtlError, ETL_CODES } from "@suaveplan/etl";
throw new EtlError(
ETL_CODES.VALIDATION_FAILED,
"Record failed schema validation",
{ context: { field: "email", value: "not-an-email" } }
);
```
## Usage: Advanced Example
```ts
import { EtlError, ETL_CODES } from "@suaveplan/etl";
function handleError(err: unknown): void {
if (err instanceof EtlError) {
switch (err.code) {
case ETL_CODES.PIPELINE_ABORTED:
console.log("Pipeline was cancelled");
break;
case ETL_CODES.SOURCE_FAILED:
console.error("Could not read source:", err.cause);
break;
default:
throw err;
}
}
}
```
## API Reference
### `ETL_CODES`
- `ETL_CODES.SOURCE_FAILED` — extraction stage error
- `ETL_CODES.TRANSFORM_FAILED` — transformation stage error
- `ETL_CODES.SINK_FAILED` — load/sink stage error
- `ETL_CODES.PIPELINE_ABORTED` — AbortSignal fired or explicit abort
- `ETL_CODES.VALIDATION_FAILED` — Zod or custom schema validation failure
- `ETL_CODES.BATCH_OVERFLOW` — batch size exceeded configured limit
### `EtlError`
Extends `SuaveplanError<EtlCode>`. Constructor: `(code, message, options?)`.
## Implementation Notes
Error codes are prefixed with `"ETL."` by `defineCodes`, producing strings like `"ETL.SOURCE_FAILED"`. Wrap underlying errors via `options.cause` to preserve the full error chain.

43
packages/foundation/etl/etl/src/errors/etl-error.test.ts Normal file
View File

@@ -0,0 +1,43 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import { ETL_CODES, type EtlCode, EtlError } from "./etl-error.js";
describe("EtlError", () => {
it("has correct name", { timeout: 5000 }, () => {
expect(new EtlError(ETL_CODES.SOURCE_FAILED, "test").name).toBe(
"EtlError"
);
});
it("carries code", { timeout: 5000 }, () => {
const err = new EtlError(ETL_CODES.TRANSFORM_FAILED, "transform error");
expect(err.code).toBe(ETL_CODES.TRANSFORM_FAILED);
});
it("chains cause", { timeout: 5000 }, () => {
const cause = new Error("root");
const err = new EtlError(ETL_CODES.SINK_FAILED, "sink failed", {
cause,
});
expect(err.cause).toBe(cause);
});
it("all codes start with ETL.", { timeout: 5000 }, () => {
const codes: EtlCode[] = [
ETL_CODES.SOURCE_FAILED,
ETL_CODES.TRANSFORM_FAILED,
ETL_CODES.SINK_FAILED,
ETL_CODES.PIPELINE_ABORTED,
ETL_CODES.VALIDATION_FAILED,
ETL_CODES.BATCH_OVERFLOW,
];
for (const code of codes) {
expect(code).toMatch(/^ETL\./);
}
});
it("instanceof checks work", { timeout: 5000 }, () => {
const err = new EtlError(ETL_CODES.PIPELINE_ABORTED, "abort");
expect(err instanceof EtlError).toBe(true);
expect(err instanceof Error).toBe(true);
});
});

36
packages/foundation/etl/etl/src/errors/etl-error.ts Normal file
View File

@@ -0,0 +1,36 @@
import { defineCodes, SuaveplanError } from "@suaveplan/error";
/**
* Stable error-code registry for `@suaveplan/etl`.
*/
export const ETL_CODES = defineCodes("ETL", [
"SOURCE_FAILED",
"TRANSFORM_FAILED",
"SINK_FAILED",
"PIPELINE_ABORTED",
"VALIDATION_FAILED",
"BATCH_OVERFLOW",
] as const);
export type EtlCode = (typeof ETL_CODES)[keyof typeof ETL_CODES];
/**
* Typed error for all `@suaveplan/etl` failures.
*
* @example
* ```ts
* import { EtlError, ETL_CODES } from "@suaveplan/etl";
*
* throw new EtlError(ETL_CODES.SOURCE_FAILED, "CSV file not found", { cause: fsError });
* ```
*/
export class EtlError extends SuaveplanError<EtlCode> {
public constructor(
code: EtlCode,
message: string,
options?: { cause?: Error; context?: Record<string, unknown> }
) {
super(code, message, options);
this.name = "EtlError";
}
}

2
packages/foundation/etl/etl/src/errors/index.ts Normal file
View File

@@ -0,0 +1,2 @@
export type { EtlCode } from "./etl-error.js";
export { ETL_CODES, EtlError } from "./etl-error.js";

51
packages/foundation/etl/etl/src/index.ts Normal file
View File

@@ -0,0 +1,51 @@
// Errors
// Constants
export { DEFAULT_BATCH_SIZE } from "./constants/index.js";
// Core — Pipeline
export {
createPipeline,
Pipeline,
RunnablePipeline,
} from "./core/pipeline/pipeline.js";
export type { EtlCode } from "./errors/etl-error.js";
export { ETL_CODES, EtlError } from "./errors/etl-error.js";
// Schemas
export {
PipelineOptionsSchema,
PipelineStatsSchema,
} from "./schemas/etl.schemas.js";
// Stages — Sink
export {
ArraySink,
BatchSink,
callbackSink,
nullSink,
} from "./stages/sink/sink.js";
// Stages — Source
export { arraySource, generatorSource } from "./stages/source/source.js";
// Stages — Transform
export {
filterTransform,
flatMapTransform,
mapTransform,
validateTransform,
} from "./stages/transform/transform.js";
export type {
ErrorAction,
ErrorHandler,
EtlRecord,
PipelineOptions,
PipelineResult,
PipelineStats,
Sink,
SkipSymbol,
Source,
Transform,
TransformContext,
} from "./types/etl.types.js";
// Types
export { SKIP } from "./types/etl.types.js";

50
packages/foundation/etl/etl/src/schemas/etl.schemas.md Normal file
View File

@@ -0,0 +1,50 @@
# etl.schemas
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides Zod validation schemas for the public option and result types in `@suaveplan/etl`. These schemas are used at integration boundaries — for validating pipeline configuration at runtime and for serialising pipeline result statistics for audit trails or dashboards.
## Features
- `PipelineStatsSchema` — validates the `PipelineStats` result including non-negative integer counters and duration
- `PipelineOptionsSchema` — validates configurable options with passthrough for injectable dependencies
## Usage: Basic Example
```ts
import { PipelineStatsSchema } from "@suaveplan/etl";
const raw = JSON.parse(statsJson);
const stats = PipelineStatsSchema.parse(raw);
console.log(stats.loaded);
```
## Usage: Advanced Example
```ts
import { PipelineOptionsSchema } from "@suaveplan/etl";
function validateConfig(config: unknown) {
return PipelineOptionsSchema.safeParse(config);
}
const result = validateConfig({ batchSize: 500 });
if (!result.success) throw new Error("Invalid pipeline config");
```
## API Reference
### `PipelineStatsSchema`
Validates `{ extracted, transformed, loaded, skipped, durationMs }`. All counts are non-negative integers; `durationMs` is a non-negative number.
### `PipelineOptionsSchema`
Validates `{ batchSize? }`. Uses `.passthrough()` for injectable fields like `source`, `clock`, and `bus`.
## Implementation Notes
Injectable dependencies (`Source`, `Sink`, `Transform`, `Clock`, `Bus`) cannot be validated with Zod since they are runtime objects with function members. The schemas cover only the serialisable configuration fields.

49
packages/foundation/etl/etl/src/schemas/etl.schemas.test.ts Normal file
View File

@@ -0,0 +1,49 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import { PipelineOptionsSchema, PipelineStatsSchema } from "./etl.schemas.js";
describe("PipelineStatsSchema", () => {
it("parses valid stats", { timeout: 5000 }, () => {
expect(() =>
PipelineStatsSchema.parse({
extracted: 10,
transformed: 8,
loaded: 8,
skipped: 2,
durationMs: 100,
})
).not.toThrow();
});
it("rejects negative extracted", { timeout: 5000 }, () => {
expect(() =>
PipelineStatsSchema.parse({
extracted: -1,
transformed: 0,
loaded: 0,
skipped: 0,
durationMs: 0,
})
).toThrow();
});
});
describe("PipelineOptionsSchema", () => {
it("accepts empty object", { timeout: 5000 }, () => {
expect(() => PipelineOptionsSchema.parse({})).not.toThrow();
});
it("accepts batchSize", { timeout: 5000 }, () => {
expect(() =>
PipelineOptionsSchema.parse({ batchSize: 100 })
).not.toThrow();
});
it("rejects zero batchSize", { timeout: 5000 }, () => {
expect(() => PipelineOptionsSchema.parse({ batchSize: 0 })).toThrow();
});
it("passes through unknown keys", { timeout: 5000 }, () => {
const result = PipelineOptionsSchema.parse({ custom: "value" });
expect(result).toMatchObject({ custom: "value" });
});
});

21
packages/foundation/etl/etl/src/schemas/etl.schemas.ts Normal file
View File

@@ -0,0 +1,21 @@
import z from "zod";
/**
* Zod schema for `PipelineStats`.
*/
export const PipelineStatsSchema = z.object({
extracted: z.number().int().nonnegative(),
transformed: z.number().int().nonnegative(),
loaded: z.number().int().nonnegative(),
skipped: z.number().int().nonnegative(),
durationMs: z.number().nonnegative(),
});
/**
* Zod schema for pipeline options (partial — injectable objects excluded).
*/
export const PipelineOptionsSchema = z
.object({
batchSize: z.number().int().positive().optional(),
})
.passthrough();

1
packages/foundation/etl/etl/src/schemas/index.ts Normal file
View File

@@ -0,0 +1 @@
export { PipelineOptionsSchema, PipelineStatsSchema } from "./etl.schemas.js";

1
packages/foundation/etl/etl/src/stages/sink/index.ts Normal file
View File

@@ -0,0 +1 @@
export { ArraySink, BatchSink, callbackSink, nullSink } from "./sink.js";

67
packages/foundation/etl/etl/src/stages/sink/sink.md Normal file
View File

@@ -0,0 +1,67 @@
# sink
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides built-in `Sink` implementations for the ETL pipeline load stage. Sinks receive transformed records and persist or collect them. This module ships four implementations: an array collector, a null discard sink, a callback-backed sink, and a batching sink that accumulates records before flushing to an inner sink.
## Features
- `ArraySink<T>` — collects all written records into `sink.records`; ideal for tests
- `nullSink<T>()` — discards all records; useful for dry-run benchmarking
- `callbackSink<T>(fn)` — calls `fn(record)` for each written record
- `BatchSink<T>` — buffers records and flushes them as arrays to an inner `Sink<T[]>` when batch is full or `flush()` is called
- All sinks implement the full `Sink<T>` lifecycle: `open`, `write`, `flush`, `close`
## Usage: Basic Example
```ts
import { ArraySink } from "@suaveplan/etl";
const sink = new ArraySink<number>();
await sink.write(1);
await sink.write(2);
await sink.flush();
console.log(sink.records); // [1, 2]
```
## Usage: Advanced Example
```ts
import { BatchSink, ArraySink } from "@suaveplan/etl";
const inner = new ArraySink<number[]>();
const batch = new BatchSink(100, inner);
for (let i = 0; i < 250; i++) await batch.write(i);
await batch.flush(); // flushes remaining 50
console.log(inner.records.length); // 3 batches: 100 + 100 + 50
```
## API Reference
### `ArraySink<T>`
- `records: T[]` — accumulated records
- `write(record)` — appends to `records`
### `nullSink<T>(name?)`
Discards all writes. `open`, `flush`, `close` are no-ops.
### `callbackSink<T>(fn, name?)`
`fn: (record: T) => void | Promise<void>` — called for each write.
### `BatchSink<T>`
- `constructor(batchSize, inner, name?)`
- Flushes to `inner.write(batch)` when buffer reaches `batchSize`
- `flush()` — sends remaining buffer to inner sink
## Implementation Notes
`BatchSink.close()` calls `flush()` first to ensure no records are lost when the pipeline closes. The inner sink receives `T[]` batches, not individual records — callers must implement the inner sink accordingly.

117
packages/foundation/etl/etl/src/stages/sink/sink.test.ts Normal file
View File

@@ -0,0 +1,117 @@
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
import { ArraySink, BatchSink, callbackSink, nullSink } from "./sink.js";
describe("ArraySink", () => {
it("collects records", { timeout: 5000 }, async () => {
const sink = new ArraySink<number>();
await sink.write(1);
await sink.write(2);
expect(sink.records).toEqual([1, 2]);
});
it("default name", { timeout: 5000 }, () => {
expect(new ArraySink().name).toBe("ArraySink");
});
it("custom name", { timeout: 5000 }, () => {
expect(new ArraySink("MySink").name).toBe("MySink");
});
it("open and flush and close are no-ops", { timeout: 5000 }, async () => {
const sink = new ArraySink();
await expect(sink.open()).resolves.toBeUndefined();
await expect(sink.flush()).resolves.toBeUndefined();
await expect(sink.close()).resolves.toBeUndefined();
});
});
describe("nullSink", () => {
it("discards records without error", { timeout: 5000 }, async () => {
const sink = nullSink<number>();
await expect(sink.write(42)).resolves.toBeUndefined();
});
it("default name", { timeout: 5000 }, () => {
expect(nullSink().name).toBe("NullSink");
});
it("open/flush/close are no-ops", { timeout: 5000 }, async () => {
const sink = nullSink();
await expect(sink.open()).resolves.toBeUndefined();
await expect(sink.flush()).resolves.toBeUndefined();
await expect(sink.close()).resolves.toBeUndefined();
});
});
describe("callbackSink", () => {
it("calls fn for each write", { timeout: 5000 }, async () => {
const fn = vi.fn();
const sink = callbackSink<number>(fn);
await sink.write(1);
await sink.write(2);
expect(fn).toHaveBeenCalledTimes(2);
expect(fn).toHaveBeenCalledWith(1);
expect(fn).toHaveBeenCalledWith(2);
});
it("default name", { timeout: 5000 }, () => {
expect(callbackSink(() => {}).name).toBe("CallbackSink");
});
it("handles async fn", { timeout: 5000 }, async () => {
const results: number[] = [];
const sink = callbackSink<number>(async (n) => {
results.push(n);
});
await sink.write(5);
expect(results).toEqual([5]);
});
it("open/flush/close are no-ops", { timeout: 5000 }, async () => {
const sink = callbackSink(() => {});
await expect(sink.open()).resolves.toBeUndefined();
await expect(sink.flush()).resolves.toBeUndefined();
await expect(sink.close()).resolves.toBeUndefined();
});
});
describe("BatchSink", () => {
it("flushes when batch is full", { timeout: 5000 }, async () => {
const inner = new ArraySink<number[]>();
const batch = new BatchSink<number>(3, inner);
await batch.open();
await batch.write(1);
await batch.write(2);
await batch.write(3); // triggers flush
expect(inner.records).toHaveLength(1);
expect(inner.records[0]).toEqual([1, 2, 3]);
});
it("flushes remainder on flush()", { timeout: 5000 }, async () => {
const inner = new ArraySink<number[]>();
const batch = new BatchSink<number>(10, inner);
await batch.write(1);
await batch.write(2);
await batch.flush();
expect(inner.records[0]).toEqual([1, 2]);
});
it("close flushes and closes inner", { timeout: 5000 }, async () => {
const inner = new ArraySink<number[]>();
const batch = new BatchSink<number>(10, inner);
await batch.write(99);
await batch.close();
expect(inner.records[0]).toContain(99);
});
it("flush with empty buffer does nothing", { timeout: 5000 }, async () => {
const inner = new ArraySink<number[]>();
const batch = new BatchSink<number>(10, inner);
await batch.flush();
expect(inner.records).toHaveLength(0);
});
it("default name", { timeout: 5000 }, () => {
expect(new BatchSink(10, new ArraySink()).name).toBe("BatchSink");
});
});

102
packages/foundation/etl/etl/src/stages/sink/sink.ts Normal file
View File

@@ -0,0 +1,102 @@
import type { Sink } from "../../types/etl.types.js";
/**
* A sink that collects all written records into an array.
*
* Useful for testing pipelines and in-memory batch processing.
*/
export class ArraySink<T> implements Sink<T> {
public readonly name: string;
public readonly records: T[] = [];
public constructor(name = "ArraySink") {
this.name = name;
}
public async open(): Promise<void> {}
public async write(record: T): Promise<void> {
this.records.push(record);
}
public async flush(): Promise<void> {}
public async close(): Promise<void> {}
}
/**
* A sink that discards all records.
*/
export function nullSink<T>(name = "NullSink"): Sink<T> {
return {
name,
async open() {},
async write(_record: T) {},
async flush() {},
async close() {},
};
}
/**
* Creates a sink backed by a callback function.
*/
export function callbackSink<T>(
fn: (record: T) => void | Promise<void>,
name = "CallbackSink"
): Sink<T> {
return {
name,
async open() {},
async write(record: T) {
await fn(record);
},
async flush() {},
async close() {},
};
}
/**
* Creates a batching sink that accumulates records and flushes to an inner
* sink in batches of `batchSize`.
*/
export class BatchSink<T> implements Sink<T> {
public readonly name: string;
protected readonly inner: Sink<T[]>;
protected readonly batchSize: number;
protected buffer: T[] = [];
public constructor(
batchSize: number,
inner: Sink<T[]>,
name = "BatchSink"
) {
this.batchSize = batchSize;
this.inner = inner;
this.name = name;
}
public async open(signal?: AbortSignal): Promise<void> {
await this.inner.open(signal);
}
public async write(record: T): Promise<void> {
this.buffer.push(record);
if (this.buffer.length >= this.batchSize) {
await this.inner.write([...this.buffer]);
this.buffer = [];
}
}
public async flush(): Promise<void> {
if (this.buffer.length > 0) {
await this.inner.write([...this.buffer]);
this.buffer = [];
}
await this.inner.flush();
}
public async close(): Promise<void> {
await this.flush();
await this.inner.close();
}
}

1
packages/foundation/etl/etl/src/stages/source/index.ts Normal file
View File

@@ -0,0 +1 @@
export { arraySource, generatorSource } from "./source.js";

59
packages/foundation/etl/etl/src/stages/source/source.md Normal file
View File

@@ -0,0 +1,59 @@
# source
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides built-in `Source` implementations for the ETL pipeline. Sources are the extraction stage — they yield records via an `AsyncGenerator`. This module ships two general-purpose factories: `arraySource` for in-memory datasets and `generatorSource` for custom async generators.
## Features
- `arraySource(items, name?)` — wraps a plain array as a pipeline source; no I/O, zero latency
- `generatorSource(gen, name?)` — wraps an async generator factory for custom extraction logic
- Both implement the full `Source<T>` interface with `open`/`read`/`close` lifecycle
- Named for OTel span attribution
## Usage: Basic Example
```ts
import { arraySource } from "@suaveplan/etl";
const source = arraySource([{ id: "1", name: "Alice" }, { id: "2", name: "Bob" }]);
for await (const record of source.read()) {
console.log(record.name);
}
```
## Usage: Advanced Example
```ts
import { generatorSource } from "@suaveplan/etl";
const pagingSource = generatorSource(async function* () {
let cursor: string | undefined;
do {
const page = await fetchPage(cursor);
yield* page.records;
cursor = page.nextCursor;
} while (cursor);
}, "ApiPagingSource");
```
## API Reference
### `arraySource<T>(items, name?)`
- `items: T[]` — records to yield
- `name?: string` — source name for OTel (default: `"ArraySource"`)
- Returns `Source<T>`
### `generatorSource<T>(gen, name?)`
- `gen: () => AsyncGenerator<T>` — factory called each time `read()` is invoked
- `name?: string` — source name for OTel (default: `"GeneratorSource"`)
- Returns `Source<T>`
## Implementation Notes
Both factories produce no-op `open` and `close` methods. If your source requires real I/O lifecycle management (database cursors, file handles), implement the `Source<T>` interface directly rather than using these convenience factories.

68
packages/foundation/etl/etl/src/stages/source/source.test.ts Normal file
View File

@@ -0,0 +1,68 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import { arraySource, generatorSource } from "./source.js";
describe("arraySource", () => {
it("yields all items", { timeout: 5000 }, async () => {
const src = arraySource([1, 2, 3]);
const results: number[] = [];
for await (const item of src.read()) {
results.push(item);
}
expect(results).toEqual([1, 2, 3]);
});
it("yields nothing for empty array", { timeout: 5000 }, async () => {
const src = arraySource([]);
const results: unknown[] = [];
for await (const item of src.read()) {
results.push(item);
}
expect(results).toHaveLength(0);
});
it("has default name", { timeout: 5000 }, () => {
expect(arraySource([]).name).toBe("ArraySource");
});
it("accepts custom name", { timeout: 5000 }, () => {
expect(arraySource([], "MySource").name).toBe("MySource");
});
it("open and close are no-ops", { timeout: 5000 }, async () => {
const src = arraySource([1]);
await expect(src.open()).resolves.toBeUndefined();
await expect(src.close()).resolves.toBeUndefined();
});
});
describe("generatorSource", () => {
it("yields items from generator", { timeout: 5000 }, async () => {
const src = generatorSource(async function* () {
yield "a";
yield "b";
});
const results: string[] = [];
for await (const item of src.read()) {
results.push(item);
}
expect(results).toEqual(["a", "b"]);
});
it("has default name", { timeout: 5000 }, () => {
expect(generatorSource(async function* () {}).name).toBe(
"GeneratorSource"
);
});
it("accepts custom name", { timeout: 5000 }, () => {
expect(generatorSource(async function* () {}, "Custom").name).toBe(
"Custom"
);
});
it("open and close are no-ops", { timeout: 5000 }, async () => {
const src = generatorSource(async function* () {});
await expect(src.open()).resolves.toBeUndefined();
await expect(src.close()).resolves.toBeUndefined();
});
});

45
packages/foundation/etl/etl/src/stages/source/source.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { Source } from "../../types/etl.types.js";
/**
* Creates a `Source` from an array of items.
*
* @example
* ```ts
* const src = arraySource([1, 2, 3]);
* for await (const item of src.read()) { console.log(item); }
* ```
*/
export function arraySource<T>(items: T[], name = "ArraySource"): Source<T> {
return {
name,
async open() {},
async *read() {
for (const item of items) {
yield item;
}
},
async close() {},
};
}
/**
* Creates a `Source` from an async generator factory.
*
* @example
* ```ts
* const src = generatorSource(async function* () {
* yield await fetchNextRecord();
* });
* ```
*/
export function generatorSource<T>(
gen: () => AsyncGenerator<T>,
name = "GeneratorSource"
): Source<T> {
return {
name,
async open() {},
read: gen,
async close() {},
};
}

6
packages/foundation/etl/etl/src/stages/transform/index.ts Normal file
View File

@@ -0,0 +1,6 @@
export {
filterTransform,
flatMapTransform,
mapTransform,
validateTransform,
} from "./transform.js";

60
packages/foundation/etl/etl/src/stages/transform/transform.md Normal file
View File

@@ -0,0 +1,60 @@
# transform
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides built-in `Transform` implementations for the ETL pipeline. Transforms are the middle stage that convert, filter, validate, or expand records. Each transform is a pure function wrapped in the `Transform<TIn, TOut>` interface. The `SKIP` sentinel enables filtering without throwing errors.
## Features
- `mapTransform(fn, name?)` — maps each record via `fn`; return `null` to skip the record
- `filterTransform(fn, name?)` — keeps records where `fn` returns `true`
- `validateTransform(schema, name?)` — parses with a Zod-compatible schema; throws `EtlError(ETL.VALIDATION_FAILED)` on failure
- `flatMapTransform(fn, name?)` — expands a record to multiple output records
- All transforms are async-capable and accept a `TransformContext` with index and abort signal
## Usage: Basic Example
```ts
import { mapTransform, filterTransform } from "@suaveplan/etl";
const toUpperCase = mapTransform((s: string) => s.toUpperCase());
const nonEmpty = filterTransform((s: string) => s.length > 0);
```
## Usage: Advanced Example
```ts
import { validateTransform } from "@suaveplan/etl";
import z from "zod";
const UserSchema = z.object({ id: z.string(), email: z.string().email() });
const validateUser = validateTransform(UserSchema, "ValidateUser");
// In a pipeline, invalid records throw EtlError(ETL.VALIDATION_FAILED)
```
## API Reference
### `mapTransform<TIn, TOut>(fn, name?)`
`fn: (record: TIn, ctx: TransformContext) => TOut | null | Promise<TOut | null>` — return `null` to skip.
### `filterTransform<T>(fn, name?)`
`fn: (record: T, ctx: TransformContext) => boolean | Promise<boolean>` — return `false` to skip.
### `validateTransform<T>(schema, name?)`
`schema: { parse: (data: unknown) => T }` — any Zod schema or compatible object.
### `flatMapTransform<TIn, TOut>(fn, name?)`
`fn: (record: TIn, ctx: TransformContext) => TOut[] | Promise<TOut[]>` — empty array skips.
## Implementation Notes
`validateTransform` re-throws validation failures as `EtlError` to integrate with the pipeline's `onError` callback. The underlying schema error is preserved as `cause`. The `SKIP` sentinel from `etl.types` is a unique symbol, avoiding confusion with falsy return values.

120
packages/foundation/etl/etl/src/stages/transform/transform.test.ts Normal file
View File

@@ -0,0 +1,120 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import z from "zod";
import { ETL_CODES, EtlError } from "../../errors/etl-error.js";
import type { TransformContext } from "../../types/etl.types.js";
import { SKIP } from "../../types/etl.types.js";
import {
filterTransform,
flatMapTransform,
mapTransform,
validateTransform,
} from "./transform.js";
const ctx: TransformContext = { index: 0 };
describe("mapTransform", () => {
it("maps records", { timeout: 5000 }, async () => {
const t = mapTransform((n: number) => n * 2);
expect(await t.transform(5, ctx)).toBe(10);
});
it("returns SKIP for null", { timeout: 5000 }, async () => {
const t = mapTransform((_n: number) => null);
expect(await t.transform(1, ctx)).toBe(SKIP);
});
it("default name", { timeout: 5000 }, () => {
expect(mapTransform((x: number) => x).name).toBe("MapTransform");
});
it("custom name", { timeout: 5000 }, () => {
expect(mapTransform((x: number) => x, "My").name).toBe("My");
});
it("handles async fn", { timeout: 5000 }, async () => {
const t = mapTransform(async (n: number) => n + 1);
expect(await t.transform(9, ctx)).toBe(10);
});
it("passes ctx to fn", { timeout: 5000 }, async () => {
let receivedCtx: TransformContext | undefined;
const t = mapTransform((n: number, c: TransformContext) => {
receivedCtx = c;
return n;
});
await t.transform(1, ctx);
expect(receivedCtx).toBe(ctx);
});
});
describe("filterTransform", () => {
it("keeps matching records", { timeout: 5000 }, async () => {
const t = filterTransform((n: number) => n > 0);
expect(await t.transform(5, ctx)).toBe(5);
});
it("returns SKIP for non-matching", { timeout: 5000 }, async () => {
const t = filterTransform((n: number) => n > 0);
expect(await t.transform(-1, ctx)).toBe(SKIP);
});
it("default name", { timeout: 5000 }, () => {
expect(filterTransform(() => true).name).toBe("FilterTransform");
});
it("handles async predicate", { timeout: 5000 }, async () => {
const t = filterTransform(async (s: string) => s.length > 0);
expect(await t.transform("hello", ctx)).toBe("hello");
expect(await t.transform("", ctx)).toBe(SKIP);
});
});
describe("validateTransform", () => {
const schema = z.object({ id: z.string() });
it("passes valid records", { timeout: 5000 }, async () => {
const t = validateTransform(schema);
expect(await t.transform({ id: "1" }, ctx)).toEqual({ id: "1" });
});
it("throws EtlError for invalid records", { timeout: 5000 }, async () => {
const t = validateTransform(schema);
await expect(t.transform({ id: 123 }, ctx)).rejects.toThrow(EtlError);
});
it("error has VALIDATION_FAILED code", { timeout: 5000 }, async () => {
const t = validateTransform(schema);
const err = await t.transform({}, ctx).catch((e: unknown) => e);
expect((err as EtlError).code).toBe(ETL_CODES.VALIDATION_FAILED);
});
it("default name", { timeout: 5000 }, () => {
expect(validateTransform(schema).name).toBe("ValidateTransform");
});
});
describe("flatMapTransform", () => {
it("expands records", { timeout: 5000 }, async () => {
const t = flatMapTransform((n: number) => [n, n * 2]);
expect(await t.transform(3, ctx)).toEqual([3, 6]);
});
it("returns SKIP for empty array", { timeout: 5000 }, async () => {
const t = flatMapTransform((_n: number) => []);
expect(await t.transform(1, ctx)).toBe(SKIP);
});
it("default name", { timeout: 5000 }, () => {
expect(flatMapTransform((x: number) => [x]).name).toBe(
"FlatMapTransform"
);
});
it("handles async fn", { timeout: 5000 }, async () => {
const t = flatMapTransform(async (s: string) => [
s.toLowerCase(),
s.toUpperCase(),
]);
expect(await t.transform("Hi", ctx)).toEqual(["hi", "HI"]);
});
});

98
packages/foundation/etl/etl/src/stages/transform/transform.ts Normal file
View File

@@ -0,0 +1,98 @@
import { ETL_CODES, EtlError } from "../../errors/etl-error.js";
import type {
SkipSymbol,
Transform,
TransformContext,
} from "../../types/etl.types.js";
import { SKIP } from "../../types/etl.types.js";
/**
* Creates a mapping transform that applies `fn` to each record.
* Return `null` to filter the record out (equivalent to `SKIP`).
*/
export function mapTransform<TIn, TOut>(
fn: (
record: TIn,
ctx: TransformContext
) => TOut | null | Promise<TOut | null>,
name = "MapTransform"
): Transform<TIn, TOut> {
return {
name,
async transform(
record: TIn,
ctx: TransformContext
): Promise<TOut | SkipSymbol> {
const result = await fn(record, ctx);
return result === null ? SKIP : result;
},
};
}
/**
* Creates a filter transform that keeps records matching `fn`.
*/
export function filterTransform<T>(
fn: (record: T, ctx: TransformContext) => boolean | Promise<boolean>,
name = "FilterTransform"
): Transform<T, T> {
return {
name,
async transform(
record: T,
ctx: TransformContext
): Promise<T | SkipSymbol> {
const keep = await fn(record, ctx);
return keep ? record : SKIP;
},
};
}
/**
* Creates a validating transform using a Zod-compatible schema.
* Records that fail validation are either skipped or throw `EtlError`.
*/
export function validateTransform<T>(
schema: { parse: (data: unknown) => T },
name = "ValidateTransform"
): Transform<unknown, T> {
return {
name,
async transform(
record: unknown,
_ctx: TransformContext
): Promise<T | SkipSymbol> {
try {
return schema.parse(record);
} catch (err) {
throw new EtlError(
ETL_CODES.VALIDATION_FAILED,
"Record failed validation",
{
cause: err as Error,
context: { record },
}
);
}
},
};
}
/**
* Creates a flat-map transform that expands one record to many.
*/
export function flatMapTransform<TIn, TOut>(
fn: (record: TIn, ctx: TransformContext) => TOut[] | Promise<TOut[]>,
name = "FlatMapTransform"
): Transform<TIn, TOut[]> {
return {
name,
async transform(
record: TIn,
ctx: TransformContext
): Promise<TOut[] | SkipSymbol> {
const results = await fn(record, ctx);
return results.length === 0 ? SKIP : results;
},
};
}

75
packages/foundation/etl/etl/src/types/etl.types.md Normal file
View File

@@ -0,0 +1,75 @@
# etl.types
## Purpose
Shared TypeScript type definitions for the `@suaveplan/etl` pipeline engine. Defines the core interfaces (`Source`, `Transform`, `Sink`), the `SKIP` sentinel, pipeline statistics, result types, and error handling contracts. All ETL implementations depend on these types, which are kept in a separate module to avoid circular imports.
## Features
- `Source<T>` — async generator source with `open`/`read`/`close` lifecycle
- `Transform<TIn, TOut>` — transforms a single record, returns `TOut | SKIP`
- `Sink<T>` — writes records with `open`/`write`/`flush`/`close` lifecycle
- `SKIP` unique symbol — filter records from the pipeline without throwing
- `TransformContext` — carries the record index and optional abort signal
- `PipelineStats` — counters for extracted, transformed, loaded, skipped records and duration
- `PipelineResult` — combines stats with a list of non-fatal errors
- `ErrorHandler` — per-record error callback returning `"skip" | "abort" | "retry"`
- `PipelineOptions<TIn>` — full configuration including batchSize, clock, bus
## Usage: Basic Example
```ts
import type { Source, Transform, Sink } from "@suaveplan/etl";
import { SKIP } from "@suaveplan/etl";
const myTransform: Transform<string, number> = {
name: "parse-int",
transform: (s) => {
const n = parseInt(s, 10);
return isNaN(n) ? SKIP : n;
},
};
```
## Usage: Advanced Example
```ts
import type { ErrorHandler, PipelineOptions } from "@suaveplan/etl";
const opts: PipelineOptions<string> = {
source: mySource,
batchSize: 500,
onError: (err, record, stage) => {
if (err.code === "ETL.VALIDATION_FAILED") return "skip";
if (stage === "source") return "abort";
return "retry";
},
};
```
## API Reference
### `Source<T>`
- `name: string` — display name for OTel spans
- `open(signal?): Promise<void>` — open connection
- `read(): AsyncGenerator<T>` — yield records
- `close(): Promise<void>` — release resources
### `Transform<TIn, TOut>`
- `name: string`
- `transform(record, ctx): Promise<TOut | SKIP> | TOut | SKIP`
### `Sink<T>`
- `name: string`
- `open(signal?)`, `write(record)`, `flush()`, `close()`
### `SKIP`
Unique symbol. Return from a transform to drop the record from the pipeline.
## Implementation Notes
The `SKIP` sentinel is a `unique symbol` rather than `null` or `undefined` to avoid ambiguity with valid `null` payloads. All lifecycle methods (`open`, `close`, `flush`) are async even when no real I/O occurs, to keep adapters interchangeable.

95
packages/foundation/etl/etl/src/types/etl.types.ts Normal file
View File

@@ -0,0 +1,95 @@
import type { Clock } from "@suaveplan/types";
import type { EtlError } from "../errors/etl-error.js";
/**
* A record flowing through the ETL pipeline.
*/
export type EtlRecord<T = unknown> = T;
/**
* A source that yields records asynchronously.
*/
export interface Source<T> {
readonly name: string;
open(signal?: AbortSignal): Promise<void>;
read(): AsyncGenerator<T>;
close(): Promise<void>;
}
/**
* Sentinel value returned by transforms to skip/filter a record.
*/
export const SKIP: unique symbol = Symbol("SKIP");
export type SkipSymbol = typeof SKIP;
/**
* Context passed to each transform call.
*/
export interface TransformContext {
index: number;
signal?: AbortSignal;
}
/**
* A single transformation stage.
*/
export interface Transform<TIn, TOut> {
readonly name: string;
transform(
record: TIn,
ctx: TransformContext
): Promise<TOut | SkipSymbol> | TOut | SkipSymbol;
}
/**
* A sink (loader) that writes records.
*/
export interface Sink<T> {
readonly name: string;
open(signal?: AbortSignal): Promise<void>;
write(record: T): Promise<void>;
flush(): Promise<void>;
close(): Promise<void>;
}
/**
* Statistics from a completed pipeline run.
*/
export interface PipelineStats {
extracted: number;
transformed: number;
loaded: number;
skipped: number;
durationMs: number;
}
/**
* Full result of a pipeline run.
*/
export interface PipelineResult {
stats: PipelineStats;
errors: EtlError[];
}
/**
* Error handling strategy.
*/
export type ErrorAction = "skip" | "abort" | "retry";
export type ErrorHandler = (
error: EtlError,
record: unknown,
stage: string
) => ErrorAction;
/**
* Options for configuring a pipeline run.
*/
export interface PipelineOptions<TIn> {
source: Source<TIn>;
batchSize?: number;
onError?: ErrorHandler;
clock?: Clock;
bus?: {
publish(topic: string, payload: unknown): Promise<void> | void;
};
}

14
packages/foundation/etl/etl/src/types/index.ts Normal file
View File

@@ -0,0 +1,14 @@
export type {
ErrorAction,
ErrorHandler,
EtlRecord,
PipelineOptions,
PipelineResult,
PipelineStats,
Sink,
SkipSymbol,
Source,
Transform,
TransformContext,
} from "./etl.types.js";
export { SKIP } from "./etl.types.js";

13
packages/foundation/etl/etl/tests/dep-graph.test.ts Normal file
View File

@@ -0,0 +1,13 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
describe("@suaveplan/etl dep-graph", () => {
it("imports without error", { timeout: 30000 }, async () => {
const mod = await import("../src/index.js");
expect(typeof mod.createPipeline).toBe("function");
expect(typeof mod.arraySource).toBe("function");
expect(typeof mod.ArraySink).toBe("function");
expect(typeof mod.mapTransform).toBe("function");
expect(typeof mod.EtlError).toBe("function");
expect(mod.SKIP).toBeDefined();
});
});

1
packages/foundation/etl/etl/tests/setup/polyfills.ts Normal file
View File

@@ -0,0 +1 @@
// Polyfills for ETL tests

12
packages/foundation/etl/etl/tsconfig.lib.json Normal file
View File

@@ -0,0 +1,12 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "../../../../tsconfig.base.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"lib": ["ES2022"],
"types": ["node"]
},
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
}

39
packages/foundation/etl/etl/vite.config.ts Normal file
View File

@@ -0,0 +1,39 @@
import { defineConfig } from "vite";
import dts from "vite-plugin-dts";
export default defineConfig({
root: __dirname,
cacheDir: "../../../node_modules/.vite/packages/etl",
plugins: [
dts({
entryRoot: "src",
tsconfigPath: "./tsconfig.lib.json",
insertTypesEntry: true,
}),
],
build: {
outDir: "./dist",
emptyOutDir: true,
reportCompressedSize: true,
sourcemap: true,
lib: { entry: "src/index.ts", formats: ["es"] },
rollupOptions: {
treeshake: true,
output: {
preserveModules: true,
preserveModulesRoot: "src",
entryFileNames: "[name].js",
},
external: [
"@suaveplan/error",
"@suaveplan/types",
"@suaveplan/events",
"@suaveplan/serialize",
"tslib",
"zod",
"@opentelemetry/api",
/^node:/,
],
},
},
});

26
packages/foundation/etl/etl/vitest.config.ts Normal file
View File

@@ -0,0 +1,26 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
globals: false,
environment: "node",
pool: "forks",
setupFiles: ["./tests/setup/polyfills.ts"],
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
exclude: ["dist/**", "node_modules/**"],
coverage: {
provider: "istanbul",
reporter: ["text", "html", "lcov"],
include: ["src/**/*.ts"],
exclude: [
"src/**/*.test.ts",
"src/**/*.types.ts",
"src/**/index.ts",
"src/**/__benchmarks__/**",
"tests/**",
],
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
},
testTimeout: 30000,
},
});