Initial Commit
This commit is contained in:
141
packages/api-stack/api/api-plugin-import/README.md
Normal file
141
packages/api-stack/api/api-plugin-import/README.md
Normal file
@@ -0,0 +1,141 @@
|
||||
# @suaveplan/api-plugin-import - Tabular-data ingest plugin for @suaveplan/api. CSV / JSON / TSV-XLSX parsers, declarative transform pipeline (coerce + validate + computed), real-time progress tracker, async ImportStoreAdapter (in-memory + SurrealDB backends ship), canonical EventEmitter-based ImportService with owner-scoped jobs, 5 auth-gated OpenAPI Hono routes, RFC 7807 problem+json renderer, 9 IMPORT.* error codes, 6 OTel instruments. Server-tier.
|
||||
|
||||
Tabular-data ingest plugin for `@suaveplan/api`. Parses CSV / JSON /
|
||||
TSV-XLSX payloads, applies declarative column mappings + per-field
|
||||
validation + computed fields, persists owner-scoped import jobs through
|
||||
a pluggable async store, emits real-time progress via an event bus, and
|
||||
exposes five auth-gated OpenAPI routes with full Zod request/response
|
||||
schemas. Ships nine `IMPORT.*` error codes (extending
|
||||
`SuaveplanError`), four OpenTelemetry instruments (`import.jobs_total`,
|
||||
`import.rows_total`, `import.row_errors_total`, `import.duration_ms`),
|
||||
and a hand-rolled RFC 7807 problem+json renderer for the route layer.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [error-codes](./src/error-codes/error-codes.md) — Declares the nine stable `IMPORT.*` error codes used throughout `@suaveplan/api-plugin-import` and exports the `ImportE…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — Declares the nine stable `IMPORT.*` error codes used throughout `@suaveplan/api-plugin-import` and exports the `ImportE…
|
||||
- [otel](./src/otel/instruments.md) — Builds the OpenTelemetry instrument bundle emitted by the canonical `ImportService` so consumers can wire production ob…
|
||||
- [instruments](./src/otel/instruments.md) — Builds the OpenTelemetry instrument bundle emitted by the canonical `ImportService` so consumers can wire production ob…
|
||||
- [csv](./src/parsers/csv.md) — Provides the CSV-family parsers consumed by the canonical `ImportService.preview` / `startImport` pipeline and by any c…
|
||||
- [json](./src/parsers/json.md) — Parses JSON-array input documents into the same `Record<string, unknown>[]` row shape produced by the CSV / XLSX parser…
|
||||
- [xlsx](./src/parsers/xlsx.md) — Provides a lightweight tab-separated-value parser exposed under the `xlsx` format so consumers can ingest spreadsheet-s…
|
||||
- [plugin](./src/plugin/plugin.md) — Returns a typed `PluginDescriptor<ImportPluginConfig>` wired onto the `@suaveplan/api-plugin` lifecycle. Valida…
|
||||
- [plugin](./src/plugin/plugin.md) — Returns a typed `PluginDescriptor<ImportPluginConfig>` wired onto the `@suaveplan/api-plugin` lifecycle. Valida…
|
||||
- [progress](./src/progress/tracker.md) — Provides the `ImportProgressTracker` class consumed by the canonical `ImportService` (and exported for direct use by CL…
|
||||
- [tracker](./src/progress/tracker.md) — Provides the `ImportProgressTracker` class consumed by the canonical `ImportService` (and exported for direct use by CL…
|
||||
- [progress-sse](./src/routes/progress-sse/progress-sse.md) — Exposes `GET /imports/:id/progress` returning a long-lived `text/event-stream` so clients can subscribe to a job's real…
|
||||
- [progress-sse](./src/routes/progress-sse/progress-sse.md) — Exposes `GET /imports/:id/progress` returning a long-lived `text/event-stream` so clients can subscribe to a job's real…
|
||||
- [build-routes](./src/routes/build-routes.md) — Constructs the `OpenAPIHono` router mounted by `createImportPlugin` at the configured `routePrefix`. Five routes are re…
|
||||
- [problem-json](./src/routes/problem-json.md) — Bridges thrown errors from the route handlers into RFC 7807 `application/problem+json` responses. Reads the HTTP status…
|
||||
- [schemas](./src/schemas/schemas.md) — Carries the full public Zod surface used by the route handlers and any external client (CLI tool, integration test, bro…
|
||||
- [options](./src/schemas/options.md) — Declares the Zod schema for the options accepted by `createImportPlugin`. Parsing options at the factory boundary guara…
|
||||
- [schemas](./src/schemas/schemas.md) — Carries the full public Zod surface used by the route handlers and any external client (CLI tool, integration test, bro…
|
||||
- [service](./src/service/service.md) — Implements the canonical `ImportService` used by `createImportPlugin` and exported for direct programmatic use. Owns th…
|
||||
- [service](./src/service/service.md) — Implements the canonical `ImportService` used by `createImportPlugin` and exported for direct programmatic use. Owns th…
|
||||
- [services](./src/services/registry-augmentation.md) — Augments the `ServiceRegistry` interface from `@suaveplan/api-plugin/service-registry` with the `importService` slot so…
|
||||
- [registry-augmentation](./src/services/registry-augmentation.md) — Augments the `ServiceRegistry` interface from `@suaveplan/api-plugin/service-registry` with the `importService` slot so…
|
||||
- [store](./src/store/memory.md) — Declares the async `ImportStoreAdapter` interface and ships the `MemoryImportStore` implementation used by the canonica…
|
||||
- [surreal](./src/store/surreal/surreal-import-store.md) — Implements the canonical `ImportStoreAdapter` interface against a connected `SurrealAdapter` from `@suaveplan/surrealdb…
|
||||
- [surreal-import-store](./src/store/surreal/surreal-import-store.md) — Implements the canonical `ImportStoreAdapter` interface against a connected `SurrealAdapter` from `@suaveplan/surrealdb…
|
||||
- [memory](./src/store/memory.md) — Declares the async `ImportStoreAdapter` interface and ships the `MemoryImportStore` implementation used by the canonica…
|
||||
- [transform](./src/transform/pipeline.md) — Implements a three-stage row-transform pipeline that any consumer can drive without depending on the canonical `ImportS…
|
||||
- [pipeline](./src/transform/pipeline.md) — Implements a three-stage row-transform pipeline that any consumer can drive without depending on the canonical `ImportS…
|
||||
- [types](./src/types/types.md) — Defines the cross-module types and constants consumed by every other feature module in `@suaveplan/api-plugin-import`.…
|
||||
- [types](./src/types/types.md) — Defines the cross-module types and constants consumed by every other feature module in `@suaveplan/api-plugin-import`.…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/api-plugin-import
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```ts
|
||||
import { createImportPlugin } from "@suaveplan/api-plugin-import";
|
||||
|
||||
// Register with @suaveplan/api's plugin registry.
|
||||
const importPlugin = createImportPlugin({ routePrefix: "/imports" });
|
||||
|
||||
// Or use the service directly:
|
||||
import { ImportService } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const service = new ImportService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda\nBob",
|
||||
mappings: [{ sourceColumn: "name", targetField: "displayName" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
console.log(progress.successCount); // 2
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- [error-codes](./src/error-codes/error-codes.md) — `IMPORT_CODES` /
|
||||
`ImportError` registry, 9 codes.
|
||||
- [types](./src/types/types.md) — cross-module shared shapes.
|
||||
- [parsers/csv](./src/parsers/csv.md) — CSV + streaming CSV parsers.
|
||||
- [parsers/json](./src/parsers/json.md) — JSON parser.
|
||||
- [parsers/xlsx](./src/parsers/xlsx.md) — TSV-style XLSX parser.
|
||||
- [transform/pipeline](./src/transform/pipeline.md) — coerce / validate
|
||||
/ computed-field pipeline.
|
||||
- [progress/tracker](./src/progress/tracker.md) — `ImportProgressTracker`.
|
||||
- [store/memory](./src/store/memory.md) — `ImportStoreAdapter` +
|
||||
`MemoryImportStore`.
|
||||
- [store/surreal/surreal-import-store](./src/store/surreal/surreal-import-store.md)
|
||||
— durable `SurrealImportStore` (SurrealDB-backed). Migration shipped at
|
||||
[`surql/tables/import_job.surql`](./surql/tables/import_job.surql).
|
||||
- [service/service](./src/service/service.md) — canonical `ImportService`.
|
||||
- [routes/problem-json](./src/routes/problem-json.md) — RFC 7807
|
||||
renderer.
|
||||
- [routes/build-routes](./src/routes/build-routes.md) — 5 auth-gated
|
||||
OpenAPI routes.
|
||||
- [otel/instruments](./src/otel/instruments.md) — OTel bundle.
|
||||
- [schemas/options](./src/schemas/options.md) — plugin options schema.
|
||||
- [schemas/schemas](./src/schemas/schemas.md) — public request /
|
||||
response Zod surface.
|
||||
- [services/registry-augmentation](./src/services/registry-augmentation.md)
|
||||
— `ServiceRegistry["importService"]` slot.
|
||||
- [plugin/plugin](./src/plugin/plugin.md) — `createImportPlugin`
|
||||
factory.
|
||||
|
||||
## Error codes
|
||||
|
||||
Every failure path raises an `ImportError` carrying one of nine
|
||||
`IMPORT.*` codes — `NOT_FOUND`, `FORBIDDEN`, `VALIDATION`,
|
||||
`FILE_TOO_LARGE`, `UNSUPPORTED_FORMAT`, `PROCESSING_FAILED`,
|
||||
`STORAGE_ERROR`, `TIMEOUT`, `INVALID_OPTIONS`. Anti-enumeration:
|
||||
foreign-owner lookups return `NOT_FOUND`, never `FORBIDDEN`.
|
||||
|
||||
## SurrealDB store
|
||||
|
||||
`SurrealImportStore` is the durable alternative to `MemoryImportStore`,
|
||||
implementing the same `ImportStoreAdapter` interface against a connected
|
||||
`@suaveplan/surrealdb` `SurrealAdapter`. The matching schema migration
|
||||
ships co-located at [`surql/tables/import_job.surql`](./surql/tables/import_job.surql);
|
||||
hosts apply it through `@suaveplan/api-plugin-database`'s migration
|
||||
runner or via `adapter.query(readFileSync(...))`. The store uses
|
||||
positional `$p0,$p1,…` SurrealQL placeholders — the canonical pattern
|
||||
used by every SurrealDB repository.
|
||||
|
||||
## Observability
|
||||
|
||||
Six OpenTelemetry instruments are emitted on the production hot paths
|
||||
— call `getImportInstruments()` to reuse the cached bundle, or
|
||||
`resetImportInstruments()` in tests after installing a fresh
|
||||
`MeterProvider`. Four cover the service (`jobs_total`, `rows_total`,
|
||||
`row_errors_total`, `duration_ms`); two cover the store
|
||||
(`import.store.writes_total{outcome}` and
|
||||
`import.store.reads_total{outcome}` — emitted by `SurrealImportStore`
|
||||
on every adapter call so host dashboards can split backend health from
|
||||
upstream import success/failure).
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
|
||||
169
packages/api-stack/api/api-plugin-import/package.json
Normal file
169
packages/api-stack/api/api-plugin-import/package.json
Normal file
@@ -0,0 +1,169 @@
|
||||
{
|
||||
"name": "@suaveplan/api-plugin-import",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Tabular-data ingest plugin for @suaveplan/api. CSV / JSON / TSV-XLSX parsers, declarative transform pipeline (coerce + validate + computed), real-time progress tracker, async ImportStoreAdapter (in-memory + SurrealDB backends ship), canonical EventEmitter-based ImportService with owner-scoped jobs, 5 auth-gated OpenAPI Hono routes, RFC 7807 problem+json renderer, 9 IMPORT.* error codes, 6 OTel instruments. Server-tier.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./error-codes/error-codes": {
|
||||
"import": {
|
||||
"types": "./dist/error-codes/error-codes.d.ts",
|
||||
"default": "./dist/error-codes/error-codes.js"
|
||||
}
|
||||
},
|
||||
"./parsers/csv": {
|
||||
"import": {
|
||||
"types": "./dist/parsers/csv.d.ts",
|
||||
"default": "./dist/parsers/csv.js"
|
||||
}
|
||||
},
|
||||
"./parsers/json": {
|
||||
"import": {
|
||||
"types": "./dist/parsers/json.d.ts",
|
||||
"default": "./dist/parsers/json.js"
|
||||
}
|
||||
},
|
||||
"./parsers/xlsx": {
|
||||
"import": {
|
||||
"types": "./dist/parsers/xlsx.d.ts",
|
||||
"default": "./dist/parsers/xlsx.js"
|
||||
}
|
||||
},
|
||||
"./transform/pipeline": {
|
||||
"import": {
|
||||
"types": "./dist/transform/pipeline.d.ts",
|
||||
"default": "./dist/transform/pipeline.js"
|
||||
}
|
||||
},
|
||||
"./progress/tracker": {
|
||||
"import": {
|
||||
"types": "./dist/progress/tracker.d.ts",
|
||||
"default": "./dist/progress/tracker.js"
|
||||
}
|
||||
},
|
||||
"./store/memory": {
|
||||
"import": {
|
||||
"types": "./dist/store/memory.d.ts",
|
||||
"default": "./dist/store/memory.js"
|
||||
}
|
||||
},
|
||||
"./store/surreal/surreal-import-store": {
|
||||
"import": {
|
||||
"types": "./dist/store/surreal/surreal-import-store.d.ts",
|
||||
"default": "./dist/store/surreal/surreal-import-store.js"
|
||||
}
|
||||
},
|
||||
"./service/service": {
|
||||
"import": {
|
||||
"types": "./dist/service/service.d.ts",
|
||||
"default": "./dist/service/service.js"
|
||||
}
|
||||
},
|
||||
"./routes/problem-json": {
|
||||
"import": {
|
||||
"types": "./dist/routes/problem-json.d.ts",
|
||||
"default": "./dist/routes/problem-json.js"
|
||||
}
|
||||
},
|
||||
"./routes/build-routes": {
|
||||
"import": {
|
||||
"types": "./dist/routes/build-routes.d.ts",
|
||||
"default": "./dist/routes/build-routes.js"
|
||||
}
|
||||
},
|
||||
"./routes/progress-sse/progress-sse": {
|
||||
"import": {
|
||||
"types": "./dist/routes/progress-sse/progress-sse.d.ts",
|
||||
"default": "./dist/routes/progress-sse/progress-sse.js"
|
||||
}
|
||||
},
|
||||
"./otel/instruments": {
|
||||
"import": {
|
||||
"types": "./dist/otel/instruments.d.ts",
|
||||
"default": "./dist/otel/instruments.js"
|
||||
}
|
||||
},
|
||||
"./schemas/options": {
|
||||
"import": {
|
||||
"types": "./dist/schemas/options.d.ts",
|
||||
"default": "./dist/schemas/options.js"
|
||||
}
|
||||
},
|
||||
"./schemas/schemas": {
|
||||
"import": {
|
||||
"types": "./dist/schemas/schemas.d.ts",
|
||||
"default": "./dist/schemas/schemas.js"
|
||||
}
|
||||
},
|
||||
"./plugin/plugin": {
|
||||
"import": {
|
||||
"types": "./dist/plugin/plugin.d.ts",
|
||||
"default": "./dist/plugin/plugin.js"
|
||||
}
|
||||
},
|
||||
"./types/types": {
|
||||
"import": {
|
||||
"types": "./dist/types/types.d.ts",
|
||||
"default": "./dist/types/types.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"test:integration": "vitest run --config ./vitest.integration.config.ts --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/zod-openapi": "catalog:server",
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/api-plugin": "workspace:^",
|
||||
"@suaveplan/api-plugin-auth": "workspace:^",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/events": "workspace:^",
|
||||
"@suaveplan/surrealdb": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"hono": "catalog:",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@opentelemetry/sdk-metrics": "1.30.0",
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@suaveplan/testing-integration-surrealdb": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": ["dist", "src", "surql", "README.md"]
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
# `error-codes` — IMPORT.* registry
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the nine stable `IMPORT.*` error codes used throughout
|
||||
`@suaveplan/api-plugin-import` and exports the `ImportError` class that
|
||||
extends `SuaveplanError` so `instanceof ImportError` narrows the `code`
|
||||
property to the literal union. Every failure path in the plugin — parser
|
||||
errors, mapping errors, owner-mismatch lookups, plugin-config validation
|
||||
failures, store I/O errors, processing timeouts — raises an
|
||||
`ImportError` carrying one of these codes, so consumers can branch on
|
||||
the string value without depending on private types.
|
||||
|
||||
The registry is built via `defineCodes("IMPORT", […])` from
|
||||
`@suaveplan/error`, which guarantees every code is namespaced
|
||||
(`IMPORT.NOT_FOUND`, …) and is mapped 1:1 to a `keyof IMPORT_CODES`
|
||||
identifier. Renaming a code is a breaking change.
|
||||
|
||||
## Features
|
||||
|
||||
- Nine canonical codes covering anti-enumeration lookups (NOT_FOUND),
|
||||
authz (FORBIDDEN), domain validation (VALIDATION), oversized payloads
|
||||
(FILE_TOO_LARGE), enum mismatches (UNSUPPORTED_FORMAT), runtime
|
||||
failures (PROCESSING_FAILED), store I/O (STORAGE_ERROR), deadlines
|
||||
(TIMEOUT), and factory-config gates (INVALID_OPTIONS).
|
||||
- `ImportError` exposes `code` (the canonical literal) plus a redundant
|
||||
`importCode` alias so instance-narrowing keeps the literal type when
|
||||
the variable is widened to `Error`.
|
||||
- Every code is reachable from a production path; the spec-conformance
|
||||
test pins the contract that none can be deleted to satisfy a coverage
|
||||
gate.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { ImportError, IMPORT_CODES } from "@suaveplan/api-plugin-import";
|
||||
|
||||
throw new ImportError(IMPORT_CODES.NOT_FOUND, `Import job "${id}" not found`, {
|
||||
context: { jobId: id, statusCode: 404 },
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ImportError, IMPORT_CODES } from "@suaveplan/api-plugin-import";
|
||||
|
||||
try {
|
||||
await service.startImport(payload, { ownerId });
|
||||
} catch (err) {
|
||||
if (err instanceof ImportError) {
|
||||
switch (err.code) {
|
||||
case IMPORT_CODES.PROCESSING_FAILED:
|
||||
metrics.increment("import.fail.parse");
|
||||
break;
|
||||
case IMPORT_CODES.FILE_TOO_LARGE:
|
||||
return c.json({ error: "payload too large" }, 413);
|
||||
default:
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `IMPORT_CODES: Readonly<{ NOT_FOUND: "IMPORT.NOT_FOUND"; … }>` — frozen
|
||||
literal map.
|
||||
- `type ImportErrorCode = (typeof IMPORT_CODES)[keyof typeof IMPORT_CODES]`
|
||||
— union of the nine literal values.
|
||||
- `class ImportError extends SuaveplanError<ImportErrorCode>` — concrete
|
||||
error; constructor `(code, message, options?: SuaveplanErrorOptions)`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Built via `defineCodes("IMPORT", […])` so every key acquires the
|
||||
`"IMPORT."` prefix automatically.
|
||||
- The class is `class` not `abstract class` because the plugin throws it
|
||||
directly (no subclass per failure mode).
|
||||
- Status-code hints travel via `options.context.statusCode` so the RFC
|
||||
7807 renderer (`routes/problem-json`) can choose the right HTTP code
|
||||
without re-importing the registry.
|
||||
@@ -0,0 +1,52 @@
|
||||
import { SuaveplanError } from "@suaveplan/error";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { IMPORT_CODES, ImportError } from "./error-codes.js";
|
||||
|
||||
describe("IMPORT_CODES", () => {
|
||||
it("declares all nine IMPORT codes", { timeout: 5000 }, () => {
|
||||
expect(Object.keys(IMPORT_CODES).sort()).toEqual(
|
||||
[
|
||||
"FILE_TOO_LARGE",
|
||||
"FORBIDDEN",
|
||||
"INVALID_OPTIONS",
|
||||
"NOT_FOUND",
|
||||
"PROCESSING_FAILED",
|
||||
"STORAGE_ERROR",
|
||||
"TIMEOUT",
|
||||
"UNSUPPORTED_FORMAT",
|
||||
"VALIDATION",
|
||||
].sort()
|
||||
);
|
||||
});
|
||||
|
||||
it("namespaces every code under IMPORT.*", { timeout: 5000 }, () => {
|
||||
for (const value of Object.values(IMPORT_CODES)) {
|
||||
expect(value.startsWith("IMPORT.")).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("every IMPORT code is reachable from a production path", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// The conformance test enumerates each code; here we assert each
|
||||
// is constructible and `instanceof ImportError` narrows to the
|
||||
// literal union.
|
||||
for (const value of Object.values(IMPORT_CODES)) {
|
||||
const err = new ImportError(value, "msg");
|
||||
expect(err).toBeInstanceOf(ImportError);
|
||||
expect(err).toBeInstanceOf(SuaveplanError);
|
||||
expect(err.code).toBe(value);
|
||||
expect(err.importCode).toBe(value);
|
||||
}
|
||||
});
|
||||
|
||||
it("carries the supplied context and cause", { timeout: 5000 }, () => {
|
||||
const cause = new Error("boom");
|
||||
const err = new ImportError(IMPORT_CODES.STORAGE_ERROR, "msg", {
|
||||
cause,
|
||||
context: { jobId: "j1", statusCode: 502 },
|
||||
});
|
||||
expect(err.cause).toBe(cause);
|
||||
expect(err.context).toEqual({ jobId: "j1", statusCode: 502 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import {
|
||||
defineCodes,
|
||||
SuaveplanError,
|
||||
type SuaveplanErrorOptions,
|
||||
} from "@suaveplan/error";
|
||||
|
||||
/**
|
||||
* Stable error-code registry for `@suaveplan/api-plugin-import`.
|
||||
*
|
||||
* Every failure path in the import plugin raises an {@link ImportError}
|
||||
* carrying one of these nine `IMPORT.*` codes. Consumers branch on the
|
||||
* string value (e.g. `error.code === IMPORT_CODES.NOT_FOUND`).
|
||||
*
|
||||
* - `NOT_FOUND` — `getJob` / `getResults` / `deleteJob` referenced a
|
||||
* missing or foreign-owner job id. Anti-enumeration: foreign-owner
|
||||
* reads map to NOT_FOUND, never `FORBIDDEN`. HTTP 404.
|
||||
* - `FORBIDDEN` — a write the caller is not permitted to perform (e.g.
|
||||
* a route that requires explicit ownership grant). HTTP 403.
|
||||
* - `VALIDATION` — input failed a domain rule (`startImport` with no
|
||||
* mappings, schema-validation outside the Zod boundary). HTTP 400.
|
||||
* - `FILE_TOO_LARGE` — payload byte-length exceeds the configured
|
||||
* `maxFileSizeBytes`. HTTP 413.
|
||||
* - `UNSUPPORTED_FORMAT` — `format` is outside the documented enum.
|
||||
* HTTP 415.
|
||||
* - `PROCESSING_FAILED` — a downstream step failed during the run
|
||||
* (parse error, mapping error, transform error). HTTP 422.
|
||||
* - `STORAGE_ERROR` — an `ImportStoreAdapter` call failed. HTTP 502
|
||||
* with retryable-exponential metadata.
|
||||
* - `TIMEOUT` — an import exceeded its allowed processing time. HTTP
|
||||
* 504 with retryable-linear metadata.
|
||||
* - `INVALID_OPTIONS` — `createImportPlugin` was called with options
|
||||
* failing `ImportPluginOptionsSchema`. `cause` holds the ZodError.
|
||||
* HTTP 500 (factory-time misconfiguration).
|
||||
*/
|
||||
export const IMPORT_CODES = defineCodes("IMPORT", [
|
||||
"NOT_FOUND",
|
||||
"FORBIDDEN",
|
||||
"VALIDATION",
|
||||
"FILE_TOO_LARGE",
|
||||
"UNSUPPORTED_FORMAT",
|
||||
"PROCESSING_FAILED",
|
||||
"STORAGE_ERROR",
|
||||
"TIMEOUT",
|
||||
"INVALID_OPTIONS",
|
||||
] as const);
|
||||
|
||||
/** Union of all nine `IMPORT.*` string-literal codes. */
|
||||
export type ImportErrorCode = (typeof IMPORT_CODES)[keyof typeof IMPORT_CODES];
|
||||
|
||||
/**
|
||||
* Concrete error class for the api-plugin-import package. Extends
|
||||
* {@link SuaveplanError} so `instanceof ImportError` narrows `code` to
|
||||
* {@link ImportErrorCode}.
|
||||
*/
|
||||
export class ImportError extends SuaveplanError<ImportErrorCode> {
|
||||
public readonly importCode: ImportErrorCode;
|
||||
|
||||
public constructor(
|
||||
code: ImportErrorCode,
|
||||
message: string,
|
||||
options?: SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.importCode = code;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export type { ImportErrorCode } from "./error-codes.js";
|
||||
export {
|
||||
IMPORT_CODES,
|
||||
ImportError,
|
||||
} from "./error-codes.js";
|
||||
142
packages/api-stack/api/api-plugin-import/src/index.ts
Normal file
142
packages/api-stack/api/api-plugin-import/src/index.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
// Side-effect import: register the importService slot on the shared
|
||||
// `ServiceRegistry` interface so downstream plugins can resolve it
|
||||
// without a cast.
|
||||
import "./services/registry-augmentation.js";
|
||||
|
||||
// Error codes
|
||||
export {
|
||||
IMPORT_CODES,
|
||||
ImportError,
|
||||
type ImportErrorCode,
|
||||
} from "./error-codes/error-codes.js";
|
||||
// OTel
|
||||
export {
|
||||
getImportInstruments,
|
||||
type ImportInstruments,
|
||||
resetImportInstruments,
|
||||
} from "./otel/instruments.js";
|
||||
// Parsers
|
||||
export {
|
||||
applyColumnMappings,
|
||||
type CsvParseResult,
|
||||
detectColumns,
|
||||
detectHeaders,
|
||||
parseCsv,
|
||||
parseCsvLine,
|
||||
parseCsvWithHeaders,
|
||||
parseJson,
|
||||
parseJsonWithHeaders,
|
||||
parseStreamLine,
|
||||
parseXlsx,
|
||||
type StreamParseResult,
|
||||
type StreamParserConfig,
|
||||
type StreamRowError,
|
||||
streamParseCsv,
|
||||
} from "./parsers/index.js";
|
||||
// Plugin
|
||||
export {
|
||||
type CreateImportPluginOptions,
|
||||
createImportPlugin,
|
||||
} from "./plugin/plugin.js";
|
||||
// Progress
|
||||
export {
|
||||
ImportProgressTracker,
|
||||
type ImportProgressTrackerOptions,
|
||||
type ProgressCallback,
|
||||
type ProgressSnapshot,
|
||||
} from "./progress/tracker.js";
|
||||
// Routes
|
||||
export {
|
||||
createImportRoutes,
|
||||
type ImportRoutesDeps,
|
||||
} from "./routes/build-routes.js";
|
||||
export {
|
||||
type HonoLike,
|
||||
PROBLEM_JSON_CONTENT_TYPE,
|
||||
type ProblemDetails,
|
||||
renderProblem,
|
||||
respondProblemJson,
|
||||
} from "./routes/problem-json.js";
|
||||
export {
|
||||
createProgressSseHandler,
|
||||
formatProgressFrame,
|
||||
type ProgressSseDeps,
|
||||
type ProgressSseEventKind,
|
||||
} from "./routes/progress-sse/progress-sse.js";
|
||||
// Schemas
|
||||
export {
|
||||
type ImportPluginConfig,
|
||||
type ImportPluginOptionsInput,
|
||||
ImportPluginOptionsSchema,
|
||||
} from "./schemas/options.js";
|
||||
export {
|
||||
type ColumnMappingInput,
|
||||
ColumnMappingSchema,
|
||||
type CreateImportJobInput,
|
||||
CreateImportJobSchema,
|
||||
type ImportFormatInput,
|
||||
ImportFormatSchema,
|
||||
type ImportJobListQueryInput,
|
||||
ImportJobListQuerySchema,
|
||||
ImportJobResponseSchema,
|
||||
type ImportJobResponseShape,
|
||||
type ImportStatusInput,
|
||||
ImportStatusSchema,
|
||||
type PreviewRequestInput,
|
||||
PreviewRequestSchema,
|
||||
PreviewResponseSchema,
|
||||
type PreviewResponseShape,
|
||||
RowErrorSchema,
|
||||
} from "./schemas/schemas.js";
|
||||
// Service
|
||||
export {
|
||||
asProcessingFailed,
|
||||
ImportService,
|
||||
type ImportServiceListFilter,
|
||||
type ImportServiceOptions,
|
||||
} from "./service/service.js";
|
||||
// Store
|
||||
export {
|
||||
type ImportStoreAdapter,
|
||||
type ImportStoreListFilter,
|
||||
MemoryImportStore,
|
||||
type MemoryImportStoreOptions,
|
||||
} from "./store/memory.js";
|
||||
export {
|
||||
DEFAULT_IMPORT_JOB_TABLE,
|
||||
SurrealImportStore,
|
||||
type SurrealImportStoreConfig,
|
||||
SurrealImportStoreConfigSchema,
|
||||
} from "./store/surreal/index.js";
|
||||
// Transform pipeline
|
||||
export {
|
||||
type CoerceResult,
|
||||
type ComputedField,
|
||||
coerceType,
|
||||
type FieldValidation,
|
||||
type TransformError,
|
||||
type TransformMapping,
|
||||
type TransformPipelineConfig,
|
||||
type TransformRowResult,
|
||||
toRowError,
|
||||
transformRow,
|
||||
transformRows,
|
||||
validateField,
|
||||
} from "./transform/pipeline.js";
|
||||
// Types
|
||||
export {
|
||||
type ColumnMapping,
|
||||
DEFAULT_BATCH_SIZE,
|
||||
type ImportEventMap,
|
||||
type ImportFormat,
|
||||
type ImportJobConfig,
|
||||
type ImportJobRecord,
|
||||
type ImportJobStatus,
|
||||
type ImportOwnerContext,
|
||||
type ImportPreview,
|
||||
type ImportProgress,
|
||||
type ImportRowProcessedEvent,
|
||||
type ImportStatus,
|
||||
type RowError,
|
||||
VALID_FORMATS,
|
||||
} from "./types/types.js";
|
||||
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
getImportInstruments,
|
||||
type ImportInstruments,
|
||||
resetImportInstruments,
|
||||
} from "./instruments.js";
|
||||
@@ -0,0 +1,91 @@
|
||||
# `otel/instruments` — OpenTelemetry instruments
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Builds the OpenTelemetry instrument bundle emitted by the canonical
|
||||
`ImportService` so consumers can wire production observability without
|
||||
hand-rolling counter names. The bundle is a single cached object keyed
|
||||
against the global `MeterProvider` — the first `getImportInstruments()`
|
||||
call resolves the meter; subsequent calls return the same reference.
|
||||
Tests rebind by installing their own `MeterProvider`, calling
|
||||
`resetImportInstruments()`, and reading back via an `InMemoryMetricExporter`.
|
||||
|
||||
## Features
|
||||
|
||||
- Nine instruments: `jobsTotal`, `rowsTotal`, `rowErrorsTotal`,
|
||||
`durationMs` (histogram), `storeWritesTotal`, `storeReadsTotal`,
|
||||
`sseConnected`, `sseEventsSent`, `sseDisconnected`.
|
||||
- `storeWritesTotal{outcome}` and `storeReadsTotal{outcome}` are emitted
|
||||
by `SurrealImportStore` on every adapter call so dashboards can split
|
||||
backend health from upstream import success/failure. The in-memory
|
||||
adapter (`MemoryImportStore`) does not emit; durable backends do.
|
||||
- `sseConnected`, `sseEventsSent{kind}`, and `sseDisconnected{reason}`
|
||||
are emitted by `GET /imports/:id/progress` so dashboards can chart
|
||||
open SSE connections, frame throughput by event kind, and teardown
|
||||
causes (terminal `complete` / `failed` vs client abort).
|
||||
- Cached bundle — production code does not re-resolve the meter on
|
||||
every emit.
|
||||
- `resetImportInstruments()` drops the cache so a freshly-installed
|
||||
`MeterProvider` rebinds on the next call.
|
||||
- Asserted by `instruments.test.ts` against a real in-memory exporter
|
||||
(no mocks).
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { getImportInstruments } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const instruments = getImportInstruments();
|
||||
instruments.jobsTotal.add(1, { outcome: "success" });
|
||||
instruments.rowsTotal.add(42, { outcome: "success" });
|
||||
instruments.durationMs.record(123);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { metrics } from "@opentelemetry/api";
|
||||
import {
|
||||
AggregationTemporality,
|
||||
InMemoryMetricExporter,
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import { getImportInstruments, resetImportInstruments } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);
|
||||
const reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
metrics.setGlobalMeterProvider(provider);
|
||||
resetImportInstruments();
|
||||
|
||||
const instruments = getImportInstruments();
|
||||
instruments.jobsTotal.add(1, { outcome: "success" });
|
||||
await provider.forceFlush();
|
||||
const collected = exporter.getMetrics();
|
||||
// collected[0].scopeMetrics[0].metrics — read back the counter datapoint
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface ImportInstruments { jobsTotal: Counter; rowsTotal: Counter; rowErrorsTotal: Counter; durationMs: Histogram; storeWritesTotal: Counter; storeReadsTotal: Counter; sseConnected: Counter; sseEventsSent: Counter; sseDisconnected: Counter }`
|
||||
- `getImportInstruments(): ImportInstruments`
|
||||
- `resetImportInstruments(): void`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The meter is named `@suaveplan/api-plugin-import` — matching the
|
||||
package name keeps cross-package metric reports readable.
|
||||
- Counters carry the `outcome` label so dashboards can split jobs/rows
|
||||
by success vs error without re-aggregating.
|
||||
- The histogram unit is set to `"ms"` so OTel collectors that respect
|
||||
unit annotation produce labelled charts; consumers MUST emit
|
||||
millisecond values.
|
||||
- The cache is module-scoped — every consumer of the package shares the
|
||||
same instrument set. This is the desired behaviour because the meter
|
||||
resolves against the global `MeterProvider`, which is a process-wide
|
||||
singleton.
|
||||
@@ -0,0 +1,75 @@
|
||||
import { metrics } from "@opentelemetry/api";
|
||||
import {
|
||||
AggregationTemporality,
|
||||
InMemoryMetricExporter,
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { getImportInstruments, resetImportInstruments } from "./instruments.js";
|
||||
|
||||
describe("import instruments", () => {
|
||||
it("getImportInstruments returns a cached bundle", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
resetImportInstruments();
|
||||
const a = getImportInstruments();
|
||||
const b = getImportInstruments();
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("resetImportInstruments rebinds against a fresh MeterProvider", {
|
||||
timeout: 10_000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const first = getImportInstruments();
|
||||
|
||||
const exporter = new InMemoryMetricExporter(
|
||||
AggregationTemporality.CUMULATIVE
|
||||
);
|
||||
const reader = new PeriodicExportingMetricReader({
|
||||
exporter,
|
||||
exportIntervalMillis: 60_000,
|
||||
});
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
metrics.setGlobalMeterProvider(provider);
|
||||
resetImportInstruments();
|
||||
|
||||
const second = getImportInstruments();
|
||||
expect(second).not.toBe(first);
|
||||
second.jobsTotal.add(1, { outcome: "success" });
|
||||
second.rowsTotal.add(5, { outcome: "success" });
|
||||
second.rowErrorsTotal.add(1);
|
||||
second.durationMs.record(123);
|
||||
second.storeWritesTotal.add(1, { outcome: "success" });
|
||||
second.storeReadsTotal.add(1, { outcome: "success" });
|
||||
second.sseConnected.add(1);
|
||||
second.sseEventsSent.add(1, { kind: "progress" });
|
||||
second.sseDisconnected.add(1, { reason: "complete" });
|
||||
await provider.forceFlush();
|
||||
const collected = exporter.getMetrics();
|
||||
expect(collected.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("bundle exposes the store write and read counters", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
resetImportInstruments();
|
||||
const bundle = getImportInstruments();
|
||||
expect(bundle.storeWritesTotal).toBeDefined();
|
||||
expect(bundle.storeReadsTotal).toBeDefined();
|
||||
expect(typeof bundle.storeWritesTotal.add).toBe("function");
|
||||
expect(typeof bundle.storeReadsTotal.add).toBe("function");
|
||||
});
|
||||
|
||||
it("bundle exposes the SSE counters", { timeout: 5000 }, () => {
|
||||
resetImportInstruments();
|
||||
const bundle = getImportInstruments();
|
||||
expect(bundle.sseConnected).toBeDefined();
|
||||
expect(bundle.sseEventsSent).toBeDefined();
|
||||
expect(bundle.sseDisconnected).toBeDefined();
|
||||
expect(typeof bundle.sseConnected.add).toBe("function");
|
||||
expect(typeof bundle.sseEventsSent.add).toBe("function");
|
||||
expect(typeof bundle.sseDisconnected.add).toBe("function");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import { type Counter, type Histogram, metrics } from "@opentelemetry/api";
|
||||
|
||||
/**
|
||||
* OpenTelemetry instruments emitted by the import plugin.
|
||||
*
|
||||
* - `jobsTotal{outcome}` — incremented on every `startImport` call.
|
||||
* `outcome ∈ "success" | "error"`.
|
||||
* - `rowsTotal{outcome}` — incremented per row processed.
|
||||
* `outcome ∈ "success" | "error"`.
|
||||
* - `rowErrorsTotal` — incremented per row-level validation/transform
|
||||
* error.
|
||||
* - `durationMs` — histogram of `startImport` wall-clock duration in
|
||||
* milliseconds.
|
||||
* - `storeWritesTotal{outcome}` — incremented per `ImportStoreAdapter`
|
||||
* write call (`create`, `update`, `delete`). `outcome ∈ "success" | "error"`.
|
||||
* - `storeReadsTotal{outcome}` — incremented per `ImportStoreAdapter`
|
||||
* read call (`get`, `list`). `outcome ∈ "success" | "error"`. A `get`
|
||||
* that returns `undefined` is still a successful read (the backend
|
||||
* answered correctly that no record exists).
|
||||
* - `sseConnected` — incremented once per accepted
|
||||
* `GET /imports/:id/progress` SSE connection.
|
||||
* - `sseEventsSent{kind}` — incremented per SSE frame written to a
|
||||
* connected client. `kind ∈ "progress" | "complete" | "failed"`.
|
||||
* - `sseDisconnected{reason}` — incremented once per SSE connection
|
||||
* teardown. `reason ∈ "complete" | "failed" | "client"` (terminal
|
||||
* complete event, terminal failed event, or client-side abort).
|
||||
*/
|
||||
export interface ImportInstruments {
|
||||
readonly jobsTotal: Counter;
|
||||
readonly rowsTotal: Counter;
|
||||
readonly rowErrorsTotal: Counter;
|
||||
readonly durationMs: Histogram;
|
||||
readonly storeWritesTotal: Counter;
|
||||
readonly storeReadsTotal: Counter;
|
||||
readonly sseConnected: Counter;
|
||||
readonly sseEventsSent: Counter;
|
||||
readonly sseDisconnected: Counter;
|
||||
}
|
||||
|
||||
let cached: ImportInstruments | undefined;
|
||||
|
||||
/**
|
||||
* Lazily build (and cache) the import instruments bundle against the
|
||||
* current global `MeterProvider`. Call {@link resetImportInstruments}
|
||||
* in test setup to force a re-bind against a freshly-installed
|
||||
* provider.
|
||||
*/
|
||||
export function getImportInstruments(): ImportInstruments {
|
||||
if (cached !== undefined) return cached;
|
||||
const meter = metrics.getMeter("@suaveplan/api-plugin-import");
|
||||
cached = {
|
||||
jobsTotal: meter.createCounter("import.jobs_total", {
|
||||
description: "Total import jobs run, partitioned by outcome label",
|
||||
}),
|
||||
rowsTotal: meter.createCounter("import.rows_total", {
|
||||
description:
|
||||
"Total rows processed across import jobs, partitioned by outcome",
|
||||
}),
|
||||
rowErrorsTotal: meter.createCounter("import.row_errors_total", {
|
||||
description:
|
||||
"Total per-row validation/transform errors across import jobs",
|
||||
}),
|
||||
durationMs: meter.createHistogram("import.duration_ms", {
|
||||
description: "Wall-clock duration of startImport calls in ms",
|
||||
unit: "ms",
|
||||
}),
|
||||
storeWritesTotal: meter.createCounter("import.store.writes_total", {
|
||||
description:
|
||||
"Total ImportStoreAdapter write calls (create/update/delete), partitioned by outcome",
|
||||
}),
|
||||
storeReadsTotal: meter.createCounter("import.store.reads_total", {
|
||||
description:
|
||||
"Total ImportStoreAdapter read calls (get/list), partitioned by outcome",
|
||||
}),
|
||||
sseConnected: meter.createCounter("import.sse.connected", {
|
||||
description:
|
||||
"Total accepted GET /imports/:id/progress SSE connections",
|
||||
}),
|
||||
sseEventsSent: meter.createCounter("import.sse.events_sent", {
|
||||
description:
|
||||
"Total SSE frames written to connected progress clients, partitioned by kind",
|
||||
}),
|
||||
sseDisconnected: meter.createCounter("import.sse.disconnected", {
|
||||
description:
|
||||
"Total SSE connection teardowns, partitioned by reason (complete/failed/client)",
|
||||
}),
|
||||
};
|
||||
return cached;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop the cached instrument bundle so the next
|
||||
* {@link getImportInstruments} call rebinds against the global
|
||||
* `MeterProvider` in effect at that point.
|
||||
*/
|
||||
export function resetImportInstruments(): void {
|
||||
cached = undefined;
|
||||
}
|
||||
86
packages/api-stack/api/api-plugin-import/src/parsers/csv.md
Normal file
86
packages/api-stack/api/api-plugin-import/src/parsers/csv.md
Normal file
@@ -0,0 +1,86 @@
|
||||
# `parsers/csv` — CSV parsers
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides the CSV-family parsers consumed by the canonical
|
||||
`ImportService.preview` / `startImport` pipeline and by any caller that
|
||||
needs to drive imports outside the plugin (CLI tools, batch workers).
|
||||
Ships both a high-level `parseCsv` returning row objects and a
|
||||
streaming variant (`streamParseCsv`) that surfaces per-row validation
|
||||
errors without aborting the run — important when the caller needs to
|
||||
collect every failed row in one pass rather than restart after each
|
||||
exception.
|
||||
|
||||
The kernel (`parseCsvLine`, `parseStreamLine`) handles double-quote
|
||||
escaping (`""`), custom delimiters, embedded newlines inside quoted
|
||||
fields, and trims field whitespace. Empty input or header-only input
|
||||
returns an empty rows array — not an exception — so route handlers can
|
||||
distinguish "no data" from "parse failed."
|
||||
|
||||
## Features
|
||||
|
||||
- `parseCsv` / `parseCsvLine` — happy-path one-shot parsing with
|
||||
default `,` delimiter.
|
||||
- `parseStreamLine` / `detectHeaders` — primitives used by
|
||||
`streamParseCsv`; exported for hosts needing custom-delimited
|
||||
variants (semicolons, pipes).
|
||||
- `streamParseCsv` — line-by-line streaming with optional `onRow`,
|
||||
`onError`, and `validate` callbacks. Validation failures are
|
||||
collected, not thrown.
|
||||
- `parseCsvWithHeaders` — header-aware result shape
|
||||
(`{ headers, rows, totalRows }`) matching `parseJson` and `parseXlsx`
|
||||
so routes can branch on `format` once.
|
||||
- `applyColumnMappings` — maps `sourceColumn` → `targetField` with
|
||||
optional per-mapping transform.
|
||||
- `detectColumns` — union of keys across heterogeneous rows.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { parseCsv } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const rows = parseCsv("name,city\nAda,London\nBob,Paris");
|
||||
// [{ name: "Ada", city: "London" }, { name: "Bob", city: "Paris" }]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { streamParseCsv } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const result = streamParseCsv("a,b\n1,2\n,3", {
|
||||
delimiter: ",",
|
||||
validate: (row, idx) => (row["a"] === "" ? `row ${idx}: a is required` : undefined),
|
||||
onError: (err) => logger.warn({ row: err.row, msg: err.message }, "drop"),
|
||||
});
|
||||
// result.errors → [{ row: 1, line: ",3", message: "row 1: a is required" }]
|
||||
// result.rows → [{ a: "1", b: "2" }]
|
||||
// result.validRows === 1
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `parseCsv(data: string): Record<string, string>[]`
|
||||
- `parseCsvLine(line: string): string[]`
|
||||
- `detectColumns(rows: Record<string, unknown>[]): string[]`
|
||||
- `parseStreamLine(line, delimiter, quoteChar): string[]`
|
||||
- `detectHeaders(firstLine, config: StreamParserConfig): string[]`
|
||||
- `streamParseCsv(data, config?): StreamParseResult`
|
||||
- `parseCsvWithHeaders(content, options?: { delimiter?; hasHeaders? }): CsvParseResult`
|
||||
- `applyColumnMappings(rows, mappings: ColumnMapping[]): Record<string, unknown>[]`
|
||||
- `interface StreamParserConfig { delimiter?; quoteChar?; hasHeaders?; onRow?; onError?; validate? }`
|
||||
- `interface StreamParseResult { headers; totalRows; validRows; errors; rows }`
|
||||
- `interface StreamRowError { row; line; message }`
|
||||
- `interface CsvParseResult { headers; rows; totalRows }`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Quote handling is single-state-machine (no regex backtracking) — O(n)
|
||||
in input length, no exponential pathological inputs.
|
||||
- `splitLines` keeps an in-quotes counter across newlines so RFC 4180
|
||||
multi-line records survive the split.
|
||||
- `applyColumnMappings` only invokes `transform` when the source value
|
||||
is a string — JSON-sourced numbers / booleans pass through unchanged.
|
||||
- The high-level `parseCsv` strips the header line by index (line 0);
|
||||
if the input contains only a header row, the function returns `[]`
|
||||
rather than `[{ header: undefined }]`.
|
||||
264
packages/api-stack/api/api-plugin-import/src/parsers/csv.test.ts
Normal file
264
packages/api-stack/api/api-plugin-import/src/parsers/csv.test.ts
Normal file
@@ -0,0 +1,264 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
applyColumnMappings,
|
||||
detectColumns,
|
||||
detectHeaders,
|
||||
parseCsv,
|
||||
parseCsvLine,
|
||||
parseCsvWithHeaders,
|
||||
parseStreamLine,
|
||||
streamParseCsv,
|
||||
} from "./csv.js";
|
||||
|
||||
describe("parseCsv", () => {
|
||||
it("parses headers + rows", { timeout: 5000 }, () => {
|
||||
expect(parseCsv("a,b\n1,2\n3,4")).toEqual([
|
||||
{ a: "1", b: "2" },
|
||||
{ a: "3", b: "4" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns an empty array for header-only input", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseCsv("a,b")).toEqual([]);
|
||||
expect(parseCsv("")).toEqual([]);
|
||||
});
|
||||
|
||||
it("parses quoted fields with embedded commas", { timeout: 5000 }, () => {
|
||||
expect(parseCsv('name,city\n"Doe, Jane",NYC')).toEqual([
|
||||
{ name: "Doe, Jane", city: "NYC" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("handles escaped double quotes inside a quoted field", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseCsvLine('"He said ""hi"""')).toEqual(['He said "hi"']);
|
||||
});
|
||||
|
||||
it("fills missing trailing values with empty strings", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseCsv("a,b,c\n1,2")).toEqual([{ a: "1", b: "2", c: "" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectColumns", () => {
|
||||
it("returns the union of keys across heterogeneous rows", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
detectColumns([
|
||||
{ a: 1, b: 2 },
|
||||
{ b: 3, c: 4 },
|
||||
])
|
||||
).toEqual(["a", "b", "c"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseStreamLine", () => {
|
||||
it("respects a custom delimiter and quote char", { timeout: 5000 }, () => {
|
||||
expect(parseStreamLine("a;b;c", ";", '"')).toEqual(["a", "b", "c"]);
|
||||
expect(parseStreamLine("'a;b';c", ";", "'")).toEqual(["a;b", "c"]);
|
||||
});
|
||||
|
||||
it("handles doubled quote-char escapes", { timeout: 5000 }, () => {
|
||||
expect(parseStreamLine('"He said ""hi"""', ",", '"')).toEqual([
|
||||
'He said "hi"',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectHeaders", () => {
|
||||
it("returns the parsed first line when hasHeaders is unset", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(detectHeaders("a,b,c", {})).toEqual(["a", "b", "c"]);
|
||||
});
|
||||
|
||||
it("synthesises column_N headers when hasHeaders is false", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
detectHeaders("x;y", { delimiter: ";", hasHeaders: false })
|
||||
).toEqual(["column_1", "column_2"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("streamParseCsv", () => {
|
||||
it("returns an empty result for empty input", { timeout: 5000 }, () => {
|
||||
const r = streamParseCsv("");
|
||||
expect(r.rows).toEqual([]);
|
||||
expect(r.totalRows).toBe(0);
|
||||
expect(r.validRows).toBe(0);
|
||||
expect(r.errors).toEqual([]);
|
||||
expect(r.headers).toEqual([]);
|
||||
});
|
||||
|
||||
it("streamParseCsv surfaces per-row validate errors", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const errs: string[] = [];
|
||||
const rows: Record<string, string>[] = [];
|
||||
const result = streamParseCsv("a,b\n,1\n2,3", {
|
||||
validate: (row, idx) =>
|
||||
row["a"] === "" ? `row ${idx}: a is required` : undefined,
|
||||
onError: (e) => errs.push(e.message),
|
||||
onRow: (r) => rows.push(r),
|
||||
});
|
||||
expect(result.errors).toEqual([
|
||||
{ row: 0, line: ",1", message: "row 0: a is required" },
|
||||
]);
|
||||
expect(result.validRows).toBe(1);
|
||||
expect(result.rows).toEqual([{ a: "2", b: "3" }]);
|
||||
expect(errs).toEqual(["row 0: a is required"]);
|
||||
expect(rows.length).toBe(1);
|
||||
});
|
||||
|
||||
it("handles quoted multiline values", { timeout: 5000 }, () => {
|
||||
// splitLines preserves the quoted newline in the line slice; the
|
||||
// value-parser then strips the surrounding quotes so the consumer
|
||||
// receives the literal embedded newline.
|
||||
const result = streamParseCsv('a,b\n"hello\nworld",2');
|
||||
expect(result.rows).toEqual([{ a: "hello\nworld", b: "2" }]);
|
||||
});
|
||||
|
||||
it("supports hasHeaders: false", { timeout: 5000 }, () => {
|
||||
const result = streamParseCsv("1,2\n3,4", { hasHeaders: false });
|
||||
expect(result.headers).toEqual(["column_1", "column_2"]);
|
||||
expect(result.rows).toEqual([
|
||||
{ column_1: "1", column_2: "2" },
|
||||
{ column_1: "3", column_2: "4" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("handles CRLF line endings", { timeout: 5000 }, () => {
|
||||
const result = streamParseCsv("a,b\r\n1,2\r\n3,4");
|
||||
expect(result.rows).toEqual([
|
||||
{ a: "1", b: "2" },
|
||||
{ a: "3", b: "4" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("fills missing trailing cells with empty strings", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = streamParseCsv("a,b,c\n1,2");
|
||||
expect(result.rows).toEqual([{ a: "1", b: "2", c: "" }]);
|
||||
});
|
||||
|
||||
it("supports an onError that is not provided", { timeout: 5000 }, () => {
|
||||
// Without onError, the validate-failure path still pushes to errors.
|
||||
const result = streamParseCsv("a,b\n,2", {
|
||||
validate: () => "fail",
|
||||
});
|
||||
expect(result.errors.length).toBe(1);
|
||||
});
|
||||
|
||||
it("supports an onRow that is not provided", { timeout: 5000 }, () => {
|
||||
const result = streamParseCsv("a,b\n1,2");
|
||||
expect(result.rows.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvWithHeaders", () => {
|
||||
it("returns headers + rows + totalRows", { timeout: 5000 }, () => {
|
||||
const r = parseCsvWithHeaders("a,b\n1,2");
|
||||
expect(r.headers).toEqual(["a", "b"]);
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
expect(r.totalRows).toBe(1);
|
||||
});
|
||||
|
||||
it("returns empty result for empty input", { timeout: 5000 }, () => {
|
||||
const r = parseCsvWithHeaders("");
|
||||
expect(r.headers).toEqual([]);
|
||||
expect(r.rows).toEqual([]);
|
||||
expect(r.totalRows).toBe(0);
|
||||
});
|
||||
|
||||
it("synthesises column_N headers when hasHeaders is false", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = parseCsvWithHeaders("1,2\n3,4", { hasHeaders: false });
|
||||
expect(r.headers).toEqual(["column_1", "column_2"]);
|
||||
expect(r.rows).toEqual([
|
||||
{ column_1: "1", column_2: "2" },
|
||||
{ column_1: "3", column_2: "4" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("supports a custom delimiter", { timeout: 5000 }, () => {
|
||||
const r = parseCsvWithHeaders("a;b\n1;2", { delimiter: ";" });
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCsvWithHeaders edge cases", () => {
|
||||
it("skips trailing blank lines when computing rows", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = parseCsvWithHeaders("a,b\n1,2\n\n");
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
expect(r.totalRows).toBe(1);
|
||||
});
|
||||
|
||||
it("handles trailing line without final newline", { timeout: 5000 }, () => {
|
||||
const r = parseCsvWithHeaders("a,b\n1,2");
|
||||
expect(r.rows.length).toBe(1);
|
||||
});
|
||||
|
||||
it("fills missing trailing cells with empty strings", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = parseCsvWithHeaders("a,b,c\n1,2");
|
||||
expect(r.rows[0]).toEqual({ a: "1", b: "2", c: "" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("streamParseCsv edge cases", () => {
|
||||
it("trims trailing blank lines via splitLines", { timeout: 5000 }, () => {
|
||||
const r = streamParseCsv("a,b\n1,2\n\n \n");
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyColumnMappings", () => {
|
||||
it("maps source columns to target fields", { timeout: 5000 }, () => {
|
||||
const result = applyColumnMappings(
|
||||
[{ src: "alpha", junk: "x" }],
|
||||
[{ sourceColumn: "src", targetField: "tgt" }]
|
||||
);
|
||||
expect(result).toEqual([{ tgt: "alpha" }]);
|
||||
});
|
||||
|
||||
it("applies a string transform when supplied", { timeout: 5000 }, () => {
|
||||
const result = applyColumnMappings(
|
||||
[{ src: "hello" }],
|
||||
[
|
||||
{
|
||||
sourceColumn: "src",
|
||||
targetField: "tgt",
|
||||
transform: (v) => v.toUpperCase(),
|
||||
},
|
||||
]
|
||||
);
|
||||
expect(result).toEqual([{ tgt: "HELLO" }]);
|
||||
});
|
||||
|
||||
it("passes non-string values through without invoking transform", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = applyColumnMappings(
|
||||
[{ src: 42 }],
|
||||
[
|
||||
{
|
||||
sourceColumn: "src",
|
||||
targetField: "tgt",
|
||||
transform: (v) => `transformed-${String(v)}`,
|
||||
},
|
||||
]
|
||||
);
|
||||
expect(result).toEqual([{ tgt: 42 }]);
|
||||
});
|
||||
});
|
||||
309
packages/api-stack/api/api-plugin-import/src/parsers/csv.ts
Normal file
309
packages/api-stack/api/api-plugin-import/src/parsers/csv.ts
Normal file
@@ -0,0 +1,309 @@
|
||||
import type { ColumnMapping } from "../types/types.js";
|
||||
|
||||
/** Configuration for the streaming CSV parser. */
|
||||
export interface StreamParserConfig {
|
||||
readonly delimiter?: string;
|
||||
readonly quoteChar?: string;
|
||||
readonly hasHeaders?: boolean;
|
||||
readonly onRow?: (row: Record<string, string>, index: number) => void;
|
||||
readonly onError?: (error: StreamRowError) => void;
|
||||
readonly validate?: (
|
||||
row: Record<string, string>,
|
||||
index: number
|
||||
) => string | undefined;
|
||||
}
|
||||
|
||||
/** Error collected during streaming with row context. */
|
||||
export interface StreamRowError {
|
||||
readonly row: number;
|
||||
readonly line: string;
|
||||
readonly message: string;
|
||||
}
|
||||
|
||||
/** Result returned after stream parsing completes. */
|
||||
export interface StreamParseResult {
|
||||
readonly headers: string[];
|
||||
readonly totalRows: number;
|
||||
readonly validRows: number;
|
||||
readonly errors: StreamRowError[];
|
||||
readonly rows: Record<string, string>[];
|
||||
}
|
||||
|
||||
/** Result returned from the high-level `parseCsvWithHeaders` helper. */
|
||||
export interface CsvParseResult {
|
||||
readonly headers: string[];
|
||||
readonly rows: Record<string, unknown>[];
|
||||
readonly totalRows: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a CSV document into `Record<string, string>[]` rows.
|
||||
*
|
||||
* Assumes the first non-empty line is the header row. Quoted fields
|
||||
* (`"…"`), escaped quotes (`""`), commas inside quotes, and trailing
|
||||
* whitespace per field are handled. Empty input (or input with a header
|
||||
* row only) returns `[]`.
|
||||
*/
|
||||
export function parseCsv(data: string): Record<string, string>[] {
|
||||
const lines = data.trim().split("\n");
|
||||
if (lines.length < 2) return [];
|
||||
const headers = parseCsvLine(String(lines[0]));
|
||||
return lines.slice(1).map((line) => {
|
||||
const values = parseCsvLine(line);
|
||||
const row: Record<string, string> = {};
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
const header = String(headers[i]);
|
||||
row[header] = values[i] ?? "";
|
||||
}
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
/** Parse a single CSV line, respecting double-quote escaping. */
|
||||
export function parseCsvLine(line: string): string[] {
|
||||
const result: string[] = [];
|
||||
let current = "";
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (inQuotes) {
|
||||
if (ch === '"' && line[i + 1] === '"') {
|
||||
current += '"';
|
||||
i++;
|
||||
} else if (ch === '"') {
|
||||
inQuotes = false;
|
||||
} else {
|
||||
current += ch;
|
||||
}
|
||||
} else if (ch === '"') {
|
||||
inQuotes = true;
|
||||
} else if (ch === ",") {
|
||||
result.push(current.trim());
|
||||
current = "";
|
||||
} else {
|
||||
current += ch;
|
||||
}
|
||||
}
|
||||
result.push(current.trim());
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Detect the union of keys present across an array of row objects. */
|
||||
export function detectColumns(rows: Record<string, unknown>[]): string[] {
|
||||
const columns = new Set<string>();
|
||||
for (const row of rows) {
|
||||
for (const key of Object.keys(row)) {
|
||||
columns.add(key);
|
||||
}
|
||||
}
|
||||
return [...columns];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single line respecting a custom delimiter and quote character —
|
||||
* the low-level kernel used by `streamParseCsv` and any host needing a
|
||||
* custom-delimited variant (`;`-separated European CSVs, pipe-separated
|
||||
* dumps, …).
|
||||
*/
|
||||
export function parseStreamLine(
|
||||
line: string,
|
||||
delimiter: string,
|
||||
quoteChar: string
|
||||
): string[] {
|
||||
const values: string[] = [];
|
||||
let current = "";
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < line.length; i++) {
|
||||
const ch = line[i];
|
||||
if (ch === quoteChar) {
|
||||
if (inQuotes && line[i + 1] === quoteChar) {
|
||||
current += quoteChar;
|
||||
i++;
|
||||
} else {
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
} else if (ch === delimiter && !inQuotes) {
|
||||
values.push(current.trim());
|
||||
current = "";
|
||||
} else {
|
||||
current += ch;
|
||||
}
|
||||
}
|
||||
values.push(current.trim());
|
||||
return values;
|
||||
}
|
||||
|
||||
/** Detect headers from the first line; synthesise placeholders if disabled. */
|
||||
export function detectHeaders(
|
||||
firstLine: string,
|
||||
config: StreamParserConfig
|
||||
): string[] {
|
||||
const delimiter = config.delimiter ?? ",";
|
||||
const quoteChar = config.quoteChar ?? '"';
|
||||
|
||||
if (config.hasHeaders === false) {
|
||||
const count = parseStreamLine(firstLine, delimiter, quoteChar).length;
|
||||
return Array.from({ length: count }, (_, i) => `column_${i + 1}`);
|
||||
}
|
||||
return parseStreamLine(firstLine, delimiter, quoteChar);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream-style CSV parser — processes the document line-by-line without
|
||||
* allocating an intermediate dense matrix. Surfaces per-row validation
|
||||
* errors via the optional `validate` callback so callers can collect
|
||||
* failures without aborting the run.
|
||||
*/
|
||||
export function streamParseCsv(
|
||||
data: string,
|
||||
config: StreamParserConfig = {}
|
||||
): StreamParseResult {
|
||||
const delimiter = config.delimiter ?? ",";
|
||||
const quoteChar = config.quoteChar ?? '"';
|
||||
const lines = splitLines(data, quoteChar);
|
||||
|
||||
if (lines.length === 0) {
|
||||
return {
|
||||
headers: [],
|
||||
totalRows: 0,
|
||||
validRows: 0,
|
||||
errors: [],
|
||||
rows: [],
|
||||
};
|
||||
}
|
||||
|
||||
const hasHeaders = config.hasHeaders !== false;
|
||||
const headers = detectHeaders(String(lines[0]), config);
|
||||
const dataStart = hasHeaders ? 1 : 0;
|
||||
const rows: Record<string, string>[] = [];
|
||||
const errors: StreamRowError[] = [];
|
||||
let validRows = 0;
|
||||
|
||||
for (let i = dataStart; i < lines.length; i++) {
|
||||
const rowIndex = i - dataStart;
|
||||
const line = String(lines[i]);
|
||||
const values = parseStreamLine(line, delimiter, quoteChar);
|
||||
|
||||
const row: Record<string, string> = {};
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
const header = String(headers[j]);
|
||||
row[header] = values[j] ?? "";
|
||||
}
|
||||
|
||||
if (config.validate) {
|
||||
const errorMsg = config.validate(row, rowIndex);
|
||||
if (errorMsg) {
|
||||
const err: StreamRowError = {
|
||||
row: rowIndex,
|
||||
line,
|
||||
message: errorMsg,
|
||||
};
|
||||
errors.push(err);
|
||||
config.onError?.(err);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
rows.push(row);
|
||||
validRows++;
|
||||
config.onRow?.(row, rowIndex);
|
||||
}
|
||||
|
||||
return {
|
||||
headers,
|
||||
totalRows: lines.length - dataStart,
|
||||
validRows,
|
||||
errors,
|
||||
rows,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* High-level CSV parser returning `{ headers, rows, totalRows }` — the
|
||||
* shape consumed by the route's preview / runImport pipelines. Wraps
|
||||
* `parseCsv` with header detection so consumers don't have to call
|
||||
* `detectColumns` afterwards.
|
||||
*/
|
||||
export function parseCsvWithHeaders(
|
||||
content: string,
|
||||
options?: { delimiter?: string; hasHeaders?: boolean }
|
||||
): CsvParseResult {
|
||||
const delimiter = options?.delimiter ?? ",";
|
||||
const hasHeaders = options?.hasHeaders !== false;
|
||||
const lines = splitLines(content, '"');
|
||||
|
||||
if (lines.length === 0) {
|
||||
return { headers: [], rows: [], totalRows: 0 };
|
||||
}
|
||||
|
||||
const firstLine = String(lines[0]);
|
||||
const headers = hasHeaders
|
||||
? parseStreamLine(firstLine, delimiter, '"')
|
||||
: parseStreamLine(firstLine, delimiter, '"').map(
|
||||
(_, i) => `column_${i + 1}`
|
||||
);
|
||||
|
||||
const dataStart = hasHeaders ? 1 : 0;
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
for (let i = dataStart; i < lines.length; i++) {
|
||||
const values = parseStreamLine(String(lines[i]), delimiter, '"');
|
||||
const row: Record<string, unknown> = {};
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
const header = String(headers[j]);
|
||||
row[header] = values[j] ?? "";
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
return { headers, rows, totalRows: rows.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Map `rows` through `mappings`, copying each `mapping.sourceColumn`
|
||||
* into `mapping.targetField`. If `mapping.transform` is supplied, the
|
||||
* value is passed through it.
|
||||
*/
|
||||
export function applyColumnMappings(
|
||||
rows: Record<string, unknown>[],
|
||||
mappings: ColumnMapping[]
|
||||
): Record<string, unknown>[] {
|
||||
return rows.map((row) => {
|
||||
const mapped: Record<string, unknown> = {};
|
||||
for (const mapping of mappings) {
|
||||
const value = row[mapping.sourceColumn];
|
||||
mapped[mapping.targetField] =
|
||||
mapping.transform && typeof value === "string"
|
||||
? mapping.transform(value)
|
||||
: value;
|
||||
}
|
||||
return mapped;
|
||||
});
|
||||
}
|
||||
|
||||
/** Split content into lines, respecting quoted fields containing newlines. */
|
||||
function splitLines(content: string, quoteChar: string): string[] {
|
||||
const lines: string[] = [];
|
||||
let current = "";
|
||||
let inQuotes = false;
|
||||
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
const ch = content[i];
|
||||
if (ch === quoteChar) {
|
||||
inQuotes = !inQuotes;
|
||||
current += ch;
|
||||
} else if ((ch === "\n" || ch === "\r") && !inQuotes) {
|
||||
if (ch === "\r" && content[i + 1] === "\n") i++;
|
||||
if (current.trim().length > 0) {
|
||||
lines.push(current);
|
||||
}
|
||||
current = "";
|
||||
} else {
|
||||
current += ch;
|
||||
}
|
||||
}
|
||||
|
||||
if (current.trim().length > 0) {
|
||||
lines.push(current);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export {
|
||||
applyColumnMappings,
|
||||
type CsvParseResult,
|
||||
detectColumns,
|
||||
detectHeaders,
|
||||
parseCsv,
|
||||
parseCsvLine,
|
||||
parseCsvWithHeaders,
|
||||
parseStreamLine,
|
||||
type StreamParseResult,
|
||||
type StreamParserConfig,
|
||||
type StreamRowError,
|
||||
streamParseCsv,
|
||||
} from "./csv.js";
|
||||
export { parseJson, parseJsonWithHeaders } from "./json.js";
|
||||
export { parseXlsx } from "./xlsx.js";
|
||||
59
packages/api-stack/api/api-plugin-import/src/parsers/json.md
Normal file
59
packages/api-stack/api/api-plugin-import/src/parsers/json.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# `parsers/json` — JSON parsers
|
||||
|
||||
## Purpose
|
||||
|
||||
Parses JSON-array input documents into the same `Record<string, unknown>[]`
|
||||
row shape produced by the CSV / XLSX parsers, so the canonical
|
||||
`ImportService` can branch on `format` and call a uniform parse function.
|
||||
The strict `parseJson` rejects non-array input with a deterministic error
|
||||
string; the high-level `parseJsonWithHeaders` wraps `parseJson` to
|
||||
produce the `{ headers, rows, totalRows }` shape consumed by the route
|
||||
preview pipeline.
|
||||
|
||||
Scalar elements in the input array (numbers, strings, `null`) are
|
||||
wrapped as `{ value: scalar }` rather than dropped, so the row count
|
||||
agrees with the input array length and the resulting rows still satisfy
|
||||
`Record<string, unknown>`.
|
||||
|
||||
## Features
|
||||
|
||||
- `parseJson` — strict array parsing; throws on non-array input.
|
||||
- `parseJsonWithHeaders` — header-aware result shape; tolerant of
|
||||
single-object inputs (wraps them in a one-element array).
|
||||
- Header inference unions keys across heterogeneous rows.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { parseJson } from "@suaveplan/api-plugin-import";
|
||||
const rows = parseJson('[{"a":1},{"a":2}]'); // [{ a: 1 }, { a: 2 }]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { parseJsonWithHeaders } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const { headers, rows, totalRows } = parseJsonWithHeaders(
|
||||
'[{"a":1,"b":2},{"a":3,"c":4}]'
|
||||
);
|
||||
// headers: ["a","b","c"]
|
||||
// rows: [{ a: 1, b: 2 }, { a: 3, c: 4 }]
|
||||
// totalRows: 2
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `parseJson(data: string): Record<string, unknown>[]` — throws `Error("JSON data must be an array")` on non-array input.
|
||||
- `parseJsonWithHeaders(content: string): CsvParseResult` — header-aware variant.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `parseJson` uses `JSON.parse` directly — invalid JSON syntax surfaces
|
||||
as the native `SyntaxError` from the runtime, which is what the
|
||||
service catches and re-throws as `IMPORT.PROCESSING_FAILED`.
|
||||
- `parseJsonWithHeaders` wraps single-object inputs in a one-element
|
||||
array so the route's `format: "json"` branch never has to special-case
|
||||
shape — the field is always `rows[]`.
|
||||
- Scalar wrapping uses the literal key `value`; consumers that need a
|
||||
different key can map the result post-parse.
|
||||
@@ -0,0 +1,48 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { parseJson, parseJsonWithHeaders } from "./json.js";
|
||||
|
||||
describe("parseJson", () => {
|
||||
it("returns array elements as rows", { timeout: 5000 }, () => {
|
||||
expect(parseJson('[{"a":1},{"a":2}]')).toEqual([{ a: 1 }, { a: 2 }]);
|
||||
});
|
||||
|
||||
it("throws when input is not a JSON array", { timeout: 5000 }, () => {
|
||||
expect(() => parseJson('{"a":1}')).toThrow(/must be an array/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseJsonWithHeaders", () => {
|
||||
it("wraps a single-object input as a one-element array", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = parseJsonWithHeaders('{"a":1}');
|
||||
expect(r.headers).toEqual(["a"]);
|
||||
expect(r.rows).toEqual([{ a: 1 }]);
|
||||
expect(r.totalRows).toBe(1);
|
||||
});
|
||||
|
||||
it("unions keys across heterogeneous rows", { timeout: 5000 }, () => {
|
||||
const r = parseJsonWithHeaders('[{"a":1},{"b":2}]');
|
||||
expect(r.headers.sort()).toEqual(["a", "b"]);
|
||||
expect(r.rows.length).toBe(2);
|
||||
});
|
||||
|
||||
it("wraps scalar elements as { value: scalar }", { timeout: 5000 }, () => {
|
||||
const r = parseJsonWithHeaders("[1, 2, 3]");
|
||||
expect(r.rows).toEqual([{ value: 1 }, { value: 2 }, { value: 3 }]);
|
||||
});
|
||||
|
||||
it("returns empty result for empty array input", { timeout: 5000 }, () => {
|
||||
const r = parseJsonWithHeaders("[]");
|
||||
expect(r.headers).toEqual([]);
|
||||
expect(r.rows).toEqual([]);
|
||||
expect(r.totalRows).toBe(0);
|
||||
});
|
||||
|
||||
it("skips null entries when computing headers", { timeout: 5000 }, () => {
|
||||
const r = parseJsonWithHeaders('[null, {"a":1}]');
|
||||
expect(r.headers).toEqual(["a"]);
|
||||
// null is wrapped as { value: null }
|
||||
expect(r.rows[0]).toEqual({ value: null });
|
||||
});
|
||||
});
|
||||
50
packages/api-stack/api/api-plugin-import/src/parsers/json.ts
Normal file
50
packages/api-stack/api/api-plugin-import/src/parsers/json.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import type { CsvParseResult } from "./csv.js";
|
||||
|
||||
/**
|
||||
* Parse a JSON-array document into `Record<string, unknown>[]` rows.
|
||||
* Throws `Error("JSON data must be an array")` for non-array input. Use
|
||||
* this when the upstream surface is a JSON array of homogeneous record
|
||||
* objects (e.g. an export of `SELECT * FROM users`).
|
||||
*/
|
||||
export function parseJson(data: string): Record<string, unknown>[] {
|
||||
const parsed = JSON.parse(data);
|
||||
if (!Array.isArray(parsed)) {
|
||||
throw new Error("JSON data must be an array");
|
||||
}
|
||||
return parsed as Record<string, unknown>[];
|
||||
}
|
||||
|
||||
/**
|
||||
* High-level JSON parser returning `{ headers, rows, totalRows }`. Wraps
|
||||
* `parseJson` with header detection so the route + preview pipelines
|
||||
* share a parse-result shape across formats. Scalar elements are wrapped
|
||||
* as `{ value: scalar }` so the row count and the input array length
|
||||
* agree.
|
||||
*/
|
||||
export function parseJsonWithHeaders(content: string): CsvParseResult {
|
||||
const parsed = JSON.parse(content);
|
||||
const data: unknown[] = Array.isArray(parsed) ? parsed : [parsed];
|
||||
|
||||
if (data.length === 0) {
|
||||
return { headers: [], rows: [], totalRows: 0 };
|
||||
}
|
||||
|
||||
const headerSet = new Set<string>();
|
||||
for (const row of data) {
|
||||
if (typeof row === "object" && row !== null) {
|
||||
for (const key of Object.keys(row)) {
|
||||
headerSet.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const headers = [...headerSet];
|
||||
const rows: Record<string, unknown>[] = data.map((item) => {
|
||||
if (typeof item !== "object" || item === null) {
|
||||
return { value: item };
|
||||
}
|
||||
return item as Record<string, unknown>;
|
||||
});
|
||||
|
||||
return { headers, rows, totalRows: rows.length };
|
||||
}
|
||||
59
packages/api-stack/api/api-plugin-import/src/parsers/xlsx.md
Normal file
59
packages/api-stack/api/api-plugin-import/src/parsers/xlsx.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# `parsers/xlsx` — TSV-style XLSX parser
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides a lightweight tab-separated-value parser exposed under the
|
||||
`xlsx` format so consumers can ingest spreadsheet-style exports without
|
||||
adding a SheetJS dependency. Binary `.xlsx` (the ZIP-packaged
|
||||
SpreadsheetML format) is explicitly out of scope; hosts that need
|
||||
binary support should wrap a SheetJS reader, render the cells to TSV,
|
||||
and call `parseXlsx` on the rendered string. Documenting the limitation
|
||||
inside the package keeps the dependency footprint small for the
|
||||
common case (TSV interchange dumps from data warehouses, manual exports
|
||||
from spreadsheet apps that produce a TSV variant).
|
||||
|
||||
## Features
|
||||
|
||||
- TSV split with optional header row.
|
||||
- `column_N` synthesised headers when `hasHeaders: false`.
|
||||
- Empty-line filtering — blank rows inside the input are dropped before
|
||||
parsing so consumers don't have to post-filter.
|
||||
- Per-cell `.trim()` so spreadsheet-pasted content with trailing tab
|
||||
whitespace produces clean values.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { parseXlsx } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const { headers, rows, totalRows } = parseXlsx("a\tb\n1\t2\n3\t4");
|
||||
// headers: ["a","b"], rows: [{ a: "1", b: "2" }, { a: "3", b: "4" }], totalRows: 2
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { parseXlsx } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const result = parseXlsx("1\tAlice\n2\tBob", { hasHeaders: false });
|
||||
// headers: ["column_1","column_2"]
|
||||
// rows: [{ column_1: "1", column_2: "Alice" }, { column_1: "2", column_2: "Bob" }]
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `parseXlsx(content: string, options?: { sheetName?: string; hasHeaders?: boolean }): CsvParseResult`
|
||||
- `sheetName` is accepted for API compatibility with future binary
|
||||
XLSX support but is ignored by the TSV implementation.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Uses `content.split(/\r?\n/)` so both LF and CRLF line endings are
|
||||
handled.
|
||||
- Empty-line filtering happens before header detection, so a file with
|
||||
a single blank line followed by a header row still produces the
|
||||
right header set.
|
||||
- Trailing-whitespace trim is per-cell, not per-row, so cells whose
|
||||
body intentionally starts or ends with whitespace are normalised. If
|
||||
preservation is required, drop in a custom parser via the
|
||||
`Parser`-shaped interface and skip `parseXlsx`.
|
||||
@@ -0,0 +1,51 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { parseXlsx } from "./xlsx.js";
|
||||
|
||||
describe("parseXlsx", () => {
|
||||
it("parses TSV with headers", { timeout: 5000 }, () => {
|
||||
const r = parseXlsx("a\tb\n1\t2");
|
||||
expect(r.headers).toEqual(["a", "b"]);
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
expect(r.totalRows).toBe(1);
|
||||
});
|
||||
|
||||
it("synthesises column_N headers when hasHeaders is false", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = parseXlsx("1\tAlice\n2\tBob", { hasHeaders: false });
|
||||
expect(r.headers).toEqual(["column_1", "column_2"]);
|
||||
expect(r.rows).toEqual([
|
||||
{ column_1: "1", column_2: "Alice" },
|
||||
{ column_1: "2", column_2: "Bob" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("filters empty lines", { timeout: 5000 }, () => {
|
||||
const r = parseXlsx("a\tb\n\n1\t2\n\n");
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
});
|
||||
|
||||
it("returns empty result for empty input", { timeout: 5000 }, () => {
|
||||
const r = parseXlsx("");
|
||||
expect(r.headers).toEqual([]);
|
||||
expect(r.rows).toEqual([]);
|
||||
expect(r.totalRows).toBe(0);
|
||||
});
|
||||
|
||||
it("handles CRLF line endings", { timeout: 5000 }, () => {
|
||||
const r = parseXlsx("a\tb\r\n1\t2");
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
});
|
||||
|
||||
it("fills missing trailing cells with empty strings", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = parseXlsx("a\tb\tc\n1\t2");
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2", c: "" }]);
|
||||
});
|
||||
|
||||
it("ignores the sheetName option", { timeout: 5000 }, () => {
|
||||
const r = parseXlsx("a\tb\n1\t2", { sheetName: "ignored" });
|
||||
expect(r.rows).toEqual([{ a: "1", b: "2" }]);
|
||||
});
|
||||
});
|
||||
43
packages/api-stack/api/api-plugin-import/src/parsers/xlsx.ts
Normal file
43
packages/api-stack/api/api-plugin-import/src/parsers/xlsx.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import type { CsvParseResult } from "./csv.js";
|
||||
|
||||
/**
|
||||
* TSV-style XLSX parser — accepts tab-separated content as a
|
||||
* lightweight interchange format. Binary `.xlsx` decoding (the ZIP +
|
||||
* XML SpreadsheetML format) is explicitly out of scope; consumers that
|
||||
* need binary support should wrap a SheetJS reader and call
|
||||
* `parseXlsx` on the rendered TSV.
|
||||
*/
|
||||
export function parseXlsx(
|
||||
content: string,
|
||||
options?: { sheetName?: string; hasHeaders?: boolean }
|
||||
): CsvParseResult {
|
||||
const hasHeaders = options?.hasHeaders !== false;
|
||||
const lines = content.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
||||
|
||||
if (lines.length === 0) {
|
||||
return { headers: [], rows: [], totalRows: 0 };
|
||||
}
|
||||
|
||||
// `lines[0]` is non-undefined here because `lines.length > 0` was
|
||||
// checked above. The `String()` cast satisfies the narrow without a
|
||||
// defensive `?? ""` branch (which would be uncoverable).
|
||||
const firstLine = String(lines[0]);
|
||||
const headers: string[] = hasHeaders
|
||||
? firstLine.split("\t").map((h) => h.trim())
|
||||
: firstLine.split("\t").map((_, i) => `column_${i + 1}`);
|
||||
|
||||
const dataStart = hasHeaders ? 1 : 0;
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
|
||||
for (let i = dataStart; i < lines.length; i++) {
|
||||
const values = String(lines[i]).split("\t");
|
||||
const row: Record<string, unknown> = {};
|
||||
for (let j = 0; j < headers.length; j++) {
|
||||
const header = String(headers[j]);
|
||||
row[header] = String(values[j] ?? "").trim();
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
return { headers, rows, totalRows: rows.length };
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export type { CreateImportPluginOptions } from "./plugin.js";
|
||||
export { createImportPlugin } from "./plugin.js";
|
||||
@@ -0,0 +1,85 @@
|
||||
# `plugin/plugin` — createImportPlugin factory
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Returns a typed `PluginDescriptor<ImportPluginConfig>` wired onto the
|
||||
`@suaveplan/api-plugin` lifecycle. Validates the supplied
|
||||
options at the factory boundary via `ImportPluginOptionsSchema` so
|
||||
invalid input fails fast with `ImportError(IMPORT_CODES.INVALID_OPTIONS)`
|
||||
and the `ZodError` as `cause`. The descriptor's three lifecycle hooks
|
||||
construct the canonical `ImportService`, register it on the
|
||||
`ServiceRegistry`, mount the routes under the configured prefix, and
|
||||
clean up listeners on close.
|
||||
|
||||
The plugin opts-in to `auth`, `events`, and `database` via
|
||||
`optionalDependencies` so the topo-sort orders it after siblings that
|
||||
provide those services without hard-requiring them. Apps that don't
|
||||
ship auth still register; their `requireAuth()` calls simply throw
|
||||
`AUTH.UNAUTHENTICATED` per the auth plugin's contract.
|
||||
|
||||
## Features
|
||||
|
||||
- Zod-validated options at the factory boundary.
|
||||
- Owner-scoped `ImportService` constructed once in `onInit`.
|
||||
- `ServiceRegistry["importService"]` slot registration so consumer
|
||||
plugins can read the service via `ctx.getService("importService")`.
|
||||
- Routes mounted via `app.route(config.routePrefix, routes)` with a
|
||||
prefix-collision guard that throws if another plugin already owns
|
||||
the path.
|
||||
- `onClose` clears every listener attached to `service.bus`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { createImportPlugin } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const importPlugin = createImportPlugin();
|
||||
// register with @suaveplan/api's plugin registry:
|
||||
// registry.register(importPlugin)
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { createImportPlugin, MemoryImportStore } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const store = new MemoryImportStore();
|
||||
const importPlugin = createImportPlugin({
|
||||
routePrefix: "/v2/imports",
|
||||
maxRows: 10_000,
|
||||
store,
|
||||
clock: () => Date.now(),
|
||||
logger: pinoLogger.child({ plugin: "import" }),
|
||||
});
|
||||
// store stays in scope so the host can introspect imported jobs
|
||||
// outside of the plugin lifecycle (e.g. background sweeps)
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `createImportPlugin(options?: CreateImportPluginOptions): PluginDescriptor<ImportPluginConfig>`
|
||||
- `type CreateImportPluginOptions = ImportPluginOptionsInput & Pick<ImportServiceOptions, "store" | "clock" | "idFactory" | "logger">`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The factory separates the Zod-validated config fields from the
|
||||
pass-through `ImportService` options (`store`, `clock`, `idFactory`,
|
||||
`logger`) so option drift between the plugin defaults and the
|
||||
service constructor stays explicit.
|
||||
- `mustService` throws if `onReady` runs before `onInit` constructed
|
||||
the service — this catches plugin-orchestrator regressions where
|
||||
hooks fire out of order.
|
||||
- The route-prefix collision guard mirrors the search-plugin
|
||||
precedent: a second mount under the same prefix silently shadows
|
||||
the first in Hono, so a hard fail at orchestration time is preferred
|
||||
to debugging a missing-route mystery in production.
|
||||
- `service.bus.clear()` removes both listeners and interceptors; the
|
||||
plugin owns the bus so a hard reset on close is safe.
|
||||
- `optionalDependencies` lists `"auth"` even though the routes hard-import
|
||||
`requireAuth` because the topo-sort only uses the dependency list for
|
||||
ordering — `requireAuth` works correctly even when the auth plugin
|
||||
hasn't run yet (it simply throws `AUTH.UNAUTHENTICATED`).
|
||||
@@ -0,0 +1,92 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import type { PluginDescriptor } from "@suaveplan/api-plugin";
|
||||
import { describe, expect, expectTypeOf, it } from "@suaveplan/testing/runner";
|
||||
import { IMPORT_CODES, ImportError } from "../error-codes/error-codes.js";
|
||||
import type { ImportPluginConfig } from "../schemas/options.js";
|
||||
import { createImportPlugin } from "./plugin.js";
|
||||
|
||||
const packageRoot = join(__dirname, "..", "..");
|
||||
const pkgJsonPath = join(packageRoot, "package.json");
|
||||
const pkgJson = JSON.parse(readFileSync(pkgJsonPath, "utf8")) as {
|
||||
exports: Record<string, unknown>;
|
||||
};
|
||||
|
||||
describe("api-plugin-import — spec conformance", () => {
|
||||
it("descriptor carries the canonical metadata", { timeout: 5000 }, () => {
|
||||
const plugin = createImportPlugin();
|
||||
expectTypeOf(plugin).toEqualTypeOf<
|
||||
PluginDescriptor<ImportPluginConfig>
|
||||
>();
|
||||
expect(plugin.name).toBe("import");
|
||||
expect(plugin.version).toBe("0.1.0");
|
||||
expect(plugin.tags).toEqual(["import", "csv", "json", "xlsx"]);
|
||||
});
|
||||
|
||||
it("ImportError narrows code to ImportErrorCode", { timeout: 5000 }, () => {
|
||||
const err = new ImportError(IMPORT_CODES.NOT_FOUND, "msg");
|
||||
expectTypeOf(err.code).toEqualTypeOf<
|
||||
(typeof IMPORT_CODES)[keyof typeof IMPORT_CODES]
|
||||
>();
|
||||
});
|
||||
|
||||
it("package exposes a subpath export for every feature module", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const requiredSubpaths = [
|
||||
"./error-codes/error-codes",
|
||||
"./parsers/csv",
|
||||
"./parsers/json",
|
||||
"./parsers/xlsx",
|
||||
"./transform/pipeline",
|
||||
"./progress/tracker",
|
||||
"./store/memory",
|
||||
"./store/surreal/surreal-import-store",
|
||||
"./service/service",
|
||||
"./routes/problem-json",
|
||||
"./routes/build-routes",
|
||||
"./otel/instruments",
|
||||
"./schemas/options",
|
||||
"./schemas/schemas",
|
||||
"./plugin/plugin",
|
||||
"./types/types",
|
||||
];
|
||||
for (const path of requiredSubpaths) {
|
||||
expect(pkgJson.exports[path]).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("package exposes a subpath export for the SurrealDB store", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
pkgJson.exports["./store/surreal/surreal-import-store"]
|
||||
).toBeDefined();
|
||||
});
|
||||
|
||||
it("every feature barrel uses explicit named re-exports", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const featureBarrels = [
|
||||
"src/error-codes/index.ts",
|
||||
"src/parsers/index.ts",
|
||||
"src/transform/index.ts",
|
||||
"src/progress/index.ts",
|
||||
"src/store/index.ts",
|
||||
"src/store/surreal/index.ts",
|
||||
"src/service/index.ts",
|
||||
"src/routes/index.ts",
|
||||
"src/otel/index.ts",
|
||||
"src/schemas/index.ts",
|
||||
"src/plugin/index.ts",
|
||||
"src/types/index.ts",
|
||||
];
|
||||
const exportStar = /^\s*export\s+(type\s+)?\*\s+from/m;
|
||||
for (const rel of featureBarrels) {
|
||||
const fp = join(packageRoot, rel);
|
||||
if (!existsSync(fp)) continue;
|
||||
const content = readFileSync(fp, "utf8");
|
||||
expect(exportStar.test(content)).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,257 @@
|
||||
import { OpenAPIHono } from "@hono/zod-openapi";
|
||||
import {
|
||||
type CreatePluginContextOptions,
|
||||
createPluginContext,
|
||||
type PluginContext,
|
||||
} from "@suaveplan/api-plugin";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { EventBusLike } from "@suaveplan/types";
|
||||
import { IMPORT_CODES, ImportError } from "../error-codes/error-codes.js";
|
||||
import { resetImportInstruments } from "../otel/instruments.js";
|
||||
import type { ImportPluginConfig } from "../schemas/options.js";
|
||||
import { ImportService } from "../service/service.js";
|
||||
import { createImportPlugin } from "./plugin.js";
|
||||
|
||||
function makeCtx(opts: { config: ImportPluginConfig; app?: OpenAPIHono }): {
|
||||
ctx: PluginContext<ImportPluginConfig>;
|
||||
emitted: Array<{ type: string }>;
|
||||
app: OpenAPIHono | undefined;
|
||||
} {
|
||||
const emitted: Array<{ type: string }> = [];
|
||||
const bus: EventBusLike = {
|
||||
emit: (type) => {
|
||||
emitted.push({ type: type as string });
|
||||
},
|
||||
on: () => () => undefined,
|
||||
};
|
||||
const ctxOpts: CreatePluginContextOptions<ImportPluginConfig> = {
|
||||
logger: {
|
||||
debug: () => undefined,
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
},
|
||||
eventBus: bus,
|
||||
signal: new AbortController().signal,
|
||||
config: opts.config,
|
||||
services: new Map(),
|
||||
};
|
||||
const ctx = createPluginContext<ImportPluginConfig>(ctxOpts);
|
||||
if (opts.app !== undefined) {
|
||||
ctx.registerService("app", opts.app);
|
||||
}
|
||||
return { ctx, emitted, app: opts.app };
|
||||
}
|
||||
|
||||
describe("createImportPlugin", () => {
|
||||
it("returns a descriptor with the canonical metadata", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
resetImportInstruments();
|
||||
const plugin = createImportPlugin();
|
||||
expect(plugin.name).toBe("import");
|
||||
expect(plugin.version).toBe("0.1.0");
|
||||
expect(plugin.optionalDependencies).toEqual([
|
||||
"auth",
|
||||
"events",
|
||||
"database",
|
||||
]);
|
||||
expect(plugin.tags).toEqual(["import", "csv", "json", "xlsx"]);
|
||||
expect(plugin.defaults?.routePrefix).toBe("/imports");
|
||||
});
|
||||
|
||||
it("throws INVALID_OPTIONS on bad config", { timeout: 5000 }, () => {
|
||||
try {
|
||||
createImportPlugin({
|
||||
routePrefix: "",
|
||||
});
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(ImportError);
|
||||
expect((err as ImportError).code).toBe(
|
||||
IMPORT_CODES.INVALID_OPTIONS
|
||||
);
|
||||
expect((err as ImportError).cause).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("onInit registers the importService slot", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const plugin = createImportPlugin();
|
||||
const { ctx, emitted } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
const svc = ctx.getService("importService");
|
||||
expect(svc).toBeInstanceOf(ImportService);
|
||||
expect(emitted.some((e) => e.type === "import:initialized")).toBe(true);
|
||||
});
|
||||
|
||||
it("onReady mounts routes under the configured routePrefix", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const app = new OpenAPIHono();
|
||||
const plugin = createImportPlugin({ routePrefix: "/imports" });
|
||||
const { ctx, emitted } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
app,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
await plugin.onReady?.(ctx);
|
||||
const routes = (app as unknown as { routes: { path: string }[] })
|
||||
.routes;
|
||||
expect(routes.some((r) => r.path.startsWith("/imports/"))).toBe(true);
|
||||
expect(emitted.some((e) => e.type === "import:ready")).toBe(true);
|
||||
});
|
||||
|
||||
it("onReady throws INVALID_OPTIONS when onInit hasn't run yet", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const plugin = createImportPlugin();
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
});
|
||||
// Skip onInit so the inner `service` closure variable is undefined.
|
||||
try {
|
||||
await plugin.onReady?.(ctx);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(
|
||||
IMPORT_CODES.INVALID_OPTIONS
|
||||
);
|
||||
expect((err as ImportError).message).toContain(
|
||||
"onReady ran before onInit"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("onReady throws when the app service is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const plugin = createImportPlugin();
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
await expect(plugin.onReady?.(ctx)).rejects.toThrow(ImportError);
|
||||
});
|
||||
|
||||
it("onReady throws on prefix collision", { timeout: 5000 }, async () => {
|
||||
resetImportInstruments();
|
||||
const app = new OpenAPIHono();
|
||||
app.get("/imports/something", (c) => c.text("ok"));
|
||||
const plugin = createImportPlugin({ routePrefix: "/imports" });
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
app,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
try {
|
||||
await plugin.onReady?.(ctx);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(
|
||||
IMPORT_CODES.INVALID_OPTIONS
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("onClose clears the service event bus", { timeout: 5000 }, async () => {
|
||||
resetImportInstruments();
|
||||
const app = new OpenAPIHono();
|
||||
const plugin = createImportPlugin();
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
app,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
const svc = ctx.getService("importService");
|
||||
let fired = 0;
|
||||
svc?.bus.on("import.completed", () => fired++);
|
||||
await plugin.onClose?.(ctx);
|
||||
// Emit after close: no listener fires.
|
||||
svc?.bus.emit("import.completed", {
|
||||
jobId: "x",
|
||||
ownerId: "alice",
|
||||
status: "completed",
|
||||
totalRows: 0,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
});
|
||||
expect(fired).toBe(0);
|
||||
});
|
||||
|
||||
it("onClose is a no-op when onInit hasn't run", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const plugin = createImportPlugin();
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
});
|
||||
await plugin.onClose?.(ctx);
|
||||
// No throw — service was never constructed
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts a caller-supplied store", { timeout: 5000 }, async () => {
|
||||
resetImportInstruments();
|
||||
const { MemoryImportStore } = await import("../store/memory.js");
|
||||
const store = new MemoryImportStore();
|
||||
const plugin = createImportPlugin({ store });
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
const svc = ctx.getService("importService");
|
||||
await svc?.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(store.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("accepts pass-through service options", { timeout: 5000 }, async () => {
|
||||
resetImportInstruments();
|
||||
const ids: string[] = [];
|
||||
const plugin = createImportPlugin({
|
||||
idFactory: () => {
|
||||
const id = `passthrough-${ids.length}`;
|
||||
ids.push(id);
|
||||
return id;
|
||||
},
|
||||
clock: () => 1,
|
||||
logger: {
|
||||
debug: () => undefined,
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
},
|
||||
});
|
||||
const { ctx } = makeCtx({
|
||||
config: plugin.defaults as ImportPluginConfig,
|
||||
});
|
||||
await plugin.onInit?.(ctx);
|
||||
const svc = ctx.getService("importService");
|
||||
await svc?.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(ids[0]).toBe("passthrough-0");
|
||||
});
|
||||
});
|
||||
137
packages/api-stack/api/api-plugin-import/src/plugin/plugin.ts
Normal file
137
packages/api-stack/api/api-plugin-import/src/plugin/plugin.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import {
|
||||
definePlugin,
|
||||
type PluginContext,
|
||||
type PluginDescriptor,
|
||||
} from "@suaveplan/api-plugin";
|
||||
import { IMPORT_CODES, ImportError } from "../error-codes/error-codes.js";
|
||||
import { createImportRoutes } from "../routes/build-routes.js";
|
||||
import {
|
||||
type ImportPluginConfig,
|
||||
type ImportPluginOptionsInput,
|
||||
ImportPluginOptionsSchema,
|
||||
} from "../schemas/options.js";
|
||||
import {
|
||||
ImportService,
|
||||
type ImportServiceOptions,
|
||||
} from "../service/service.js";
|
||||
import "../services/registry-augmentation.js";
|
||||
|
||||
/** Options accepted by `createImportPlugin`. */
|
||||
export type CreateImportPluginOptions = ImportPluginOptionsInput &
|
||||
Pick<ImportServiceOptions, "store" | "clock" | "idFactory" | "logger">;
|
||||
|
||||
/**
|
||||
* Factory — returns a typed `PluginDescriptor` for the import plugin.
|
||||
*
|
||||
* Zod-validates the supplied options at the boundary so invalid input
|
||||
* fails fast with `ImportError(IMPORT_CODES.INVALID_OPTIONS)` carrying
|
||||
* the `ZodError` as `cause`. The descriptor's `optionalDependencies`
|
||||
* lists `"auth"`, `"events"`, `"database"` so orchestrators can
|
||||
* topo-sort the import plugin after those siblings without requiring
|
||||
* them.
|
||||
*
|
||||
* Lifecycle:
|
||||
*
|
||||
* 1. `onInit` — construct the canonical `ImportService` (wired to the
|
||||
* optional caller-supplied store / clock / logger), register it
|
||||
* under the `importService` slot, emit `import:initialized`.
|
||||
* 2. `onReady` — read the resolved config, build the route tree, and
|
||||
* mount it under the configured `routePrefix`; emit `import:ready`.
|
||||
* 3. `onClose` — clear every listener on the service's event bus.
|
||||
*/
|
||||
export function createImportPlugin(
|
||||
options: CreateImportPluginOptions = {}
|
||||
): PluginDescriptor<ImportPluginConfig> {
|
||||
const { store, clock, idFactory, logger, ...rest } = options;
|
||||
const parsed = ImportPluginOptionsSchema.safeParse(rest);
|
||||
if (!parsed.success) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.INVALID_OPTIONS,
|
||||
"createImportPlugin received invalid options",
|
||||
{ cause: parsed.error, context: { statusCode: 500 } }
|
||||
);
|
||||
}
|
||||
const config = parsed.data;
|
||||
|
||||
let service: ImportService | undefined;
|
||||
const serviceOpts: ImportServiceOptions = {
|
||||
...(store !== undefined && { store }),
|
||||
...(clock !== undefined && { clock }),
|
||||
...(idFactory !== undefined && { idFactory }),
|
||||
...(logger !== undefined && { logger }),
|
||||
};
|
||||
|
||||
return definePlugin<ImportPluginConfig>({
|
||||
name: "import",
|
||||
version: "0.1.0",
|
||||
description:
|
||||
"Tabular-data ingest plugin: CSV / JSON / TSV-XLSX parsers, transform pipeline, owner-scoped jobs, 5 auth-gated routes, OTel observability.",
|
||||
dependencies: [],
|
||||
optionalDependencies: ["auth", "events", "database"],
|
||||
tags: ["import", "csv", "json", "xlsx"],
|
||||
defaults: config,
|
||||
|
||||
onInit: async (ctx) => {
|
||||
service = new ImportService(serviceOpts);
|
||||
ctx.registerService("importService", service);
|
||||
ctx.eventBus.emit("import:initialized", undefined);
|
||||
},
|
||||
|
||||
onReady: async (ctx) => runOnReady(ctx, config, mustService(service)),
|
||||
|
||||
onClose: async () => {
|
||||
if (service) {
|
||||
service.bus.clear();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mustService(service: ImportService | undefined): ImportService {
|
||||
if (!service) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.INVALID_OPTIONS,
|
||||
"import plugin onReady ran before onInit constructed the service",
|
||||
{ context: { statusCode: 500 } }
|
||||
);
|
||||
}
|
||||
return service;
|
||||
}
|
||||
|
||||
async function runOnReady(
|
||||
ctx: PluginContext<ImportPluginConfig>,
|
||||
config: ImportPluginConfig,
|
||||
service: ImportService
|
||||
): Promise<void> {
|
||||
const app = ctx.getService("app");
|
||||
if (app === undefined) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.INVALID_OPTIONS,
|
||||
"import plugin requires the `app` service in ServiceRegistry",
|
||||
{ context: { statusCode: 500, missing: ["app"] } }
|
||||
);
|
||||
}
|
||||
|
||||
const existing = (
|
||||
app as unknown as { routes: ReadonlyArray<{ path: string }> }
|
||||
).routes;
|
||||
if (
|
||||
Array.isArray(existing) &&
|
||||
existing.some((r) => r.path.startsWith(`${config.routePrefix}/`))
|
||||
) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.INVALID_OPTIONS,
|
||||
`Route prefix '${config.routePrefix}' already has registered routes`,
|
||||
{
|
||||
context: {
|
||||
prefix: config.routePrefix,
|
||||
statusCode: 500,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const routes = createImportRoutes({ service });
|
||||
app.route(config.routePrefix, routes);
|
||||
ctx.eventBus.emit("import:ready", undefined);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
ImportProgressTracker,
|
||||
type ImportProgressTrackerOptions,
|
||||
type ProgressCallback,
|
||||
type ProgressSnapshot,
|
||||
} from "./tracker.js";
|
||||
@@ -0,0 +1,83 @@
|
||||
# `progress/tracker` — real-time import progress
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides the `ImportProgressTracker` class consumed by the canonical
|
||||
`ImportService` (and exported for direct use by CLI tools / batch
|
||||
workers) to track import jobs in real time. The tracker counts
|
||||
successes, failures, and overall progress, and exposes derived metrics
|
||||
(rows per second, percent complete, estimated remaining time) without
|
||||
requiring the caller to keep track of the wall clock themselves.
|
||||
|
||||
The clock is injectable so test suites can pin deterministic ETA /
|
||||
rows-per-sec assertions without sleeping. Without a clock override the
|
||||
tracker uses `Date.now`, so the production hot path is allocation-free
|
||||
after construction.
|
||||
|
||||
## Features
|
||||
|
||||
- Single-row (`recordSuccess`, `recordFailure`) and batched
|
||||
(`recordBatchSuccess(count)`, `recordBatchFailure(count)`) updates.
|
||||
- Listener fan-out via `onProgress(callback)` — returns an unsubscribe
|
||||
function so consumers can detach cleanly.
|
||||
- Snapshot shape (`ProgressSnapshot`) carries `rowsPerSecond` rounded to
|
||||
2 decimal places, `percentComplete` rounded to an integer, and
|
||||
`estimatedRemainingMs` rounded to an integer.
|
||||
- Injectable `Clock` from `@suaveplan/types` for deterministic tests.
|
||||
- `isComplete()` predicate flips once `processedRows >= totalRows` —
|
||||
handles `totalRows === 0` as immediately complete.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { ImportProgressTracker } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const tracker = new ImportProgressTracker("job-1", 100);
|
||||
tracker.onProgress((snap) => console.log(snap.percentComplete));
|
||||
tracker.recordBatchSuccess(50); // logs 50
|
||||
tracker.recordBatchFailure(10); // logs 60
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ImportProgressTracker } from "@suaveplan/api-plugin-import";
|
||||
|
||||
// Deterministic test with an injected clock — advance time by a known delta
|
||||
let now = 1_000;
|
||||
const tracker = new ImportProgressTracker("j", 100, { clock: () => now });
|
||||
|
||||
tracker.recordBatchSuccess(50);
|
||||
now += 1_000;
|
||||
const snap = tracker.snapshot();
|
||||
// snap.rowsPerSecond === 50, snap.estimatedRemainingMs === 1_000
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `class ImportProgressTracker`
|
||||
- `constructor(jobId: string, totalRows: number, options?: ImportProgressTrackerOptions)`
|
||||
- `onProgress(callback: ProgressCallback): () => void`
|
||||
- `recordSuccess(): void`
|
||||
- `recordFailure(): void`
|
||||
- `recordBatchSuccess(count: number): void`
|
||||
- `recordBatchFailure(count: number): void`
|
||||
- `snapshot(): ProgressSnapshot`
|
||||
- `isComplete(): boolean`
|
||||
- `interface ImportProgressTrackerOptions { clock?: Clock }`
|
||||
- `interface ProgressSnapshot { jobId; totalRows; processedRows; successCount; failedCount; percentComplete; rowsPerSecond; estimatedRemainingMs; elapsedMs; startedAt }`
|
||||
- `type ProgressCallback = (snap: ProgressSnapshot) => void`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `totalRows === 0` short-circuits `percentComplete` to `0` and
|
||||
`estimatedRemainingMs` to `0` to avoid `NaN` from a zero divisor.
|
||||
- The listener array is replaced (not spliced) on `off()` so a
|
||||
listener removing itself from inside its own callback does not
|
||||
invalidate the in-flight iteration.
|
||||
- `startedAtIso` is captured once in the constructor and re-used on
|
||||
every snapshot to avoid `new Date(...)` allocations on the hot path.
|
||||
@@ -0,0 +1,94 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { ImportProgressTracker, type ProgressSnapshot } from "./tracker.js";
|
||||
|
||||
describe("ImportProgressTracker", () => {
|
||||
it("recordBatchSuccess fans a snapshot with the cumulative counts", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const snaps: ProgressSnapshot[] = [];
|
||||
const tracker = new ImportProgressTracker("j", 100);
|
||||
tracker.onProgress((s) => snaps.push(s));
|
||||
tracker.recordBatchSuccess(5);
|
||||
expect(snaps.length).toBe(1);
|
||||
expect(snaps[0]?.successCount).toBe(5);
|
||||
expect(snaps[0]?.processedRows).toBe(5);
|
||||
});
|
||||
|
||||
it("records single + batch failures", { timeout: 5000 }, () => {
|
||||
const tracker = new ImportProgressTracker("j", 10);
|
||||
tracker.recordFailure();
|
||||
tracker.recordBatchFailure(2);
|
||||
const snap = tracker.snapshot();
|
||||
expect(snap.failedCount).toBe(3);
|
||||
expect(snap.processedRows).toBe(3);
|
||||
});
|
||||
|
||||
it("records single successes", { timeout: 5000 }, () => {
|
||||
const tracker = new ImportProgressTracker("j", 5);
|
||||
tracker.recordSuccess();
|
||||
const snap = tracker.snapshot();
|
||||
expect(snap.successCount).toBe(1);
|
||||
});
|
||||
|
||||
it("computes rows/sec, ETA, and percent with an injected clock", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
let now = 1_000;
|
||||
const tracker = new ImportProgressTracker("j", 100, {
|
||||
clock: () => now,
|
||||
});
|
||||
tracker.recordBatchSuccess(50);
|
||||
now += 1_000;
|
||||
const snap = tracker.snapshot();
|
||||
expect(snap.rowsPerSecond).toBe(50);
|
||||
expect(snap.estimatedRemainingMs).toBe(1_000);
|
||||
expect(snap.percentComplete).toBe(50);
|
||||
expect(snap.elapsedMs).toBe(1_000);
|
||||
});
|
||||
|
||||
it("snapshot guards against zero totalRows", { timeout: 5000 }, () => {
|
||||
let now = 0;
|
||||
const tracker = new ImportProgressTracker("j", 0, {
|
||||
clock: () => now,
|
||||
});
|
||||
const snap = tracker.snapshot();
|
||||
expect(snap.percentComplete).toBe(0);
|
||||
expect(snap.rowsPerSecond).toBe(0);
|
||||
expect(snap.estimatedRemainingMs).toBe(0);
|
||||
now += 1_000;
|
||||
// After advancing the clock with no rows processed, rowsPerSecond
|
||||
// stays at 0 because processedRows is 0.
|
||||
const snap2 = tracker.snapshot();
|
||||
expect(snap2.rowsPerSecond).toBe(0);
|
||||
});
|
||||
|
||||
it("isComplete flips when processedRows reaches totalRows", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const tracker = new ImportProgressTracker("j", 2);
|
||||
expect(tracker.isComplete()).toBe(false);
|
||||
tracker.recordBatchSuccess(2);
|
||||
expect(tracker.isComplete()).toBe(true);
|
||||
});
|
||||
|
||||
it("onProgress returns an unsubscribe that stops further notifications", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const snaps: ProgressSnapshot[] = [];
|
||||
const tracker = new ImportProgressTracker("j", 5);
|
||||
const off = tracker.onProgress((s) => snaps.push(s));
|
||||
tracker.recordSuccess();
|
||||
off();
|
||||
tracker.recordSuccess();
|
||||
expect(snaps.length).toBe(1);
|
||||
});
|
||||
|
||||
it("uses the default clock when none is provided", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const tracker = new ImportProgressTracker("j", 1);
|
||||
tracker.recordSuccess();
|
||||
const snap = tracker.snapshot();
|
||||
expect(snap.elapsedMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
});
|
||||
133
packages/api-stack/api/api-plugin-import/src/progress/tracker.ts
Normal file
133
packages/api-stack/api/api-plugin-import/src/progress/tracker.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
|
||||
/** Snapshot of current import progress. */
|
||||
export interface ProgressSnapshot {
|
||||
readonly jobId: string;
|
||||
readonly totalRows: number;
|
||||
readonly processedRows: number;
|
||||
readonly successCount: number;
|
||||
readonly failedCount: number;
|
||||
readonly percentComplete: number;
|
||||
readonly rowsPerSecond: number;
|
||||
readonly estimatedRemainingMs: number;
|
||||
readonly elapsedMs: number;
|
||||
readonly startedAt: string;
|
||||
}
|
||||
|
||||
/** Callback signature for progress notifications. */
|
||||
export type ProgressCallback = (snapshot: ProgressSnapshot) => void;
|
||||
|
||||
/** Options accepted by the tracker constructor. */
|
||||
export interface ImportProgressTrackerOptions {
|
||||
/** Injectable clock — defaults to `Date.now`. */
|
||||
readonly clock?: Clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* Real-time progress tracker for import jobs. Records successes and
|
||||
* failures (single + batched), computes rows/sec and ETA, and fans
|
||||
* snapshots to registered listeners.
|
||||
*
|
||||
* The clock is injectable so tests can run deterministic ETA / rows-per-sec
|
||||
* assertions without sleeping. Default clock is `Date.now`.
|
||||
*/
|
||||
export class ImportProgressTracker {
|
||||
private readonly jobId: string;
|
||||
private readonly totalRows: number;
|
||||
private readonly startedAt: number;
|
||||
private readonly startedAtIso: string;
|
||||
private readonly clock: Clock;
|
||||
private processedRows = 0;
|
||||
private successCount = 0;
|
||||
private failedCount = 0;
|
||||
private listeners: ProgressCallback[] = [];
|
||||
|
||||
public constructor(
|
||||
jobId: string,
|
||||
totalRows: number,
|
||||
options: ImportProgressTrackerOptions = {}
|
||||
) {
|
||||
this.jobId = jobId;
|
||||
this.totalRows = totalRows;
|
||||
this.clock = options.clock ?? defaultClock;
|
||||
this.startedAt = this.clock();
|
||||
this.startedAtIso = new Date(this.startedAt).toISOString();
|
||||
}
|
||||
|
||||
/** Register a progress callback. Returns an `off` function. */
|
||||
public onProgress(callback: ProgressCallback): () => void {
|
||||
this.listeners.push(callback);
|
||||
return () => {
|
||||
this.listeners = this.listeners.filter((cb) => cb !== callback);
|
||||
};
|
||||
}
|
||||
|
||||
/** Record a single successfully-processed row. */
|
||||
public recordSuccess(): void {
|
||||
this.processedRows++;
|
||||
this.successCount++;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
/** Record a single failed row. */
|
||||
public recordFailure(): void {
|
||||
this.processedRows++;
|
||||
this.failedCount++;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
/** Record `count` successes in one call (batch). */
|
||||
public recordBatchSuccess(count: number): void {
|
||||
this.processedRows += count;
|
||||
this.successCount += count;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
/** Record `count` failures in one call (batch). */
|
||||
public recordBatchFailure(count: number): void {
|
||||
this.processedRows += count;
|
||||
this.failedCount += count;
|
||||
this.notify();
|
||||
}
|
||||
|
||||
/** Compute and return the current snapshot. */
|
||||
public snapshot(): ProgressSnapshot {
|
||||
const elapsedMs = this.clock() - this.startedAt;
|
||||
const rowsPerSecond =
|
||||
elapsedMs > 0 ? (this.processedRows / elapsedMs) * 1000 : 0;
|
||||
const remaining = this.totalRows - this.processedRows;
|
||||
const estimatedRemainingMs =
|
||||
rowsPerSecond > 0 ? (remaining / rowsPerSecond) * 1000 : 0;
|
||||
const percentComplete =
|
||||
this.totalRows > 0
|
||||
? Math.round((this.processedRows / this.totalRows) * 100)
|
||||
: 0;
|
||||
|
||||
return {
|
||||
jobId: this.jobId,
|
||||
totalRows: this.totalRows,
|
||||
processedRows: this.processedRows,
|
||||
successCount: this.successCount,
|
||||
failedCount: this.failedCount,
|
||||
percentComplete,
|
||||
rowsPerSecond: Math.round(rowsPerSecond * 100) / 100,
|
||||
estimatedRemainingMs: Math.round(estimatedRemainingMs),
|
||||
elapsedMs,
|
||||
startedAt: this.startedAtIso,
|
||||
};
|
||||
}
|
||||
|
||||
/** True once `processedRows >= totalRows`. */
|
||||
public isComplete(): boolean {
|
||||
return this.processedRows >= this.totalRows;
|
||||
}
|
||||
|
||||
private notify(): void {
|
||||
const snap = this.snapshot();
|
||||
for (const listener of this.listeners) {
|
||||
listener(snap);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const defaultClock: Clock = () => Date.now();
|
||||
@@ -0,0 +1,69 @@
|
||||
# `routes/build-routes` — auth-gated import routes
|
||||
|
||||
## Purpose
|
||||
|
||||
Constructs the `OpenAPIHono` router mounted by `createImportPlugin` at
|
||||
the configured `routePrefix`. Five routes are registered — `POST
|
||||
/preview`, `POST /`, `GET /`, `GET /{id}`, `DELETE /{id}` — each gated
|
||||
behind `requireAuth()` from `@suaveplan/api-plugin-auth` so anonymous
|
||||
requests return 401 problem+json before they ever reach a handler.
|
||||
Owner-scoped routes read `c.get("user").id` and pass it through to the
|
||||
canonical `ImportService`, which raises `IMPORT.NOT_FOUND` (mapped to
|
||||
404 problem+json) on foreign-owner lookups so the surface is
|
||||
anti-enumeration-safe.
|
||||
|
||||
## Features
|
||||
|
||||
- Five OpenAPI routes with full Zod request / response schemas.
|
||||
- Per-route `app.use("/*", requireAuth())` guard.
|
||||
- `progressToResponse()` shape mapper so the response body always
|
||||
satisfies `ImportJobResponseSchema`.
|
||||
- Every handler funnels error paths through `respondProblemJson` so
|
||||
the response shape is uniform across success and failure.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { createImportRoutes, ImportService } from "@suaveplan/api-plugin-import";
|
||||
import { OpenAPIHono } from "@hono/zod-openapi";
|
||||
|
||||
const service = new ImportService();
|
||||
const app = new OpenAPIHono();
|
||||
app.route("/imports", createImportRoutes({ service }));
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { createImportRoutes, ImportService, MemoryImportStore } from "@suaveplan/api-plugin-import";
|
||||
import { OpenAPIHono } from "@hono/zod-openapi";
|
||||
|
||||
const store = new MemoryImportStore();
|
||||
const service = new ImportService({ store });
|
||||
service.bus.on("import.progress", (snap) => sse.broadcast(snap));
|
||||
|
||||
const app = new OpenAPIHono();
|
||||
app.use("/imports/*", customRateLimit({ window: "1m", max: 10 }));
|
||||
app.route("/imports", createImportRoutes({ service }));
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `createImportRoutes(deps: ImportRoutesDeps): OpenAPIHono`
|
||||
- `interface ImportRoutesDeps { service: ImportService }`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The `requireAuth()` middleware is mounted via `app.use("/*", ...)` so
|
||||
every route inherits the guard. Mounting per-route would duplicate
|
||||
the wiring and miss the OpenAPI catch-all.
|
||||
- `ownerOf(c)` returns the empty string when `c.get("user")` is null;
|
||||
the auth middleware throws BEFORE the handler runs, so the empty
|
||||
string is unreachable from production traffic — but it remains a
|
||||
safe default if a caller somehow bypasses the middleware in a test
|
||||
harness.
|
||||
- The list endpoint returns `{ items, total }` rather than a raw array
|
||||
so clients can paginate; the schema enforces `total >= 0`.
|
||||
- The delete endpoint surfaces a `{ deleted: boolean }` payload — a
|
||||
successful delete returns `{ deleted: true }`; a missing job throws
|
||||
`IMPORT.NOT_FOUND` from the service layer and is rendered as 404.
|
||||
@@ -0,0 +1,299 @@
|
||||
import type { Principal } from "@suaveplan/api-plugin-auth";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { Hono } from "hono";
|
||||
import { resetImportInstruments } from "../otel/instruments.js";
|
||||
import { ImportService } from "../service/service.js";
|
||||
import { createImportRoutes } from "./build-routes.js";
|
||||
|
||||
interface FixtureOpts {
|
||||
readonly principal: Principal | null;
|
||||
}
|
||||
|
||||
function buildApp(opts: FixtureOpts): {
|
||||
app: Hono;
|
||||
service: ImportService;
|
||||
} {
|
||||
resetImportInstruments();
|
||||
let n = 0;
|
||||
const service = new ImportService({
|
||||
idFactory: () => `job-${++n}`,
|
||||
});
|
||||
|
||||
const host = new Hono();
|
||||
// Inject the principal BEFORE the router so requireAuth() sees it.
|
||||
host.use("*", async (c, next) => {
|
||||
c.set("user", opts.principal);
|
||||
await next();
|
||||
});
|
||||
host.route("/imports", createImportRoutes({ service }));
|
||||
return { app: host, service };
|
||||
}
|
||||
|
||||
const alice: Principal = {
|
||||
id: "alice",
|
||||
emailVerified: true,
|
||||
roles: [],
|
||||
};
|
||||
|
||||
const bob: Principal = {
|
||||
id: "bob",
|
||||
emailVerified: true,
|
||||
roles: [],
|
||||
};
|
||||
|
||||
describe("createImportRoutes", () => {
|
||||
it("POST / returns 401 when caller is anonymous", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: null });
|
||||
const res = await app.request("/imports", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(401);
|
||||
expect(res.headers.get("content-type")).toContain(
|
||||
"application/problem+json"
|
||||
);
|
||||
});
|
||||
|
||||
it("POST /preview returns the preview body", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: alice });
|
||||
const res = await app.request("/imports/preview", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
format: "csv",
|
||||
data: "name,age\nAda,37",
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { columns: string[] };
|
||||
expect(body.columns).toEqual(["name", "age"]);
|
||||
});
|
||||
|
||||
it("POST /preview surfaces parser failure as problem+json", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: alice });
|
||||
const res = await app.request("/imports/preview", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ format: "json", data: "not-json" }),
|
||||
});
|
||||
// PROCESSING_FAILED → 422 via app.onError mapping
|
||||
expect(res.status).toBe(422);
|
||||
expect(res.headers.get("content-type")).toContain(
|
||||
"application/problem+json"
|
||||
);
|
||||
});
|
||||
|
||||
it("POST / starts an import job for the caller", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: alice });
|
||||
const res = await app.request("/imports", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
jobId: string;
|
||||
ownerId: string;
|
||||
status: string;
|
||||
};
|
||||
expect(body.ownerId).toBe("alice");
|
||||
expect(body.status).toBe("completed");
|
||||
expect(body.jobId).toBe("job-1");
|
||||
});
|
||||
|
||||
it("POST / surfaces service failures as problem+json", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: alice });
|
||||
const res = await app.request("/imports", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
format: "json",
|
||||
data: "not-json",
|
||||
mappings: [{ sourceColumn: "x", targetField: "x" }],
|
||||
}),
|
||||
});
|
||||
// PROCESSING_FAILED → 422 via app.onError mapping
|
||||
expect(res.status).toBe(422);
|
||||
expect(res.headers.get("content-type")).toContain(
|
||||
"application/problem+json"
|
||||
);
|
||||
});
|
||||
|
||||
it("GET / lists the caller's jobs", { timeout: 5000 }, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request("/imports");
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as {
|
||||
items: unknown[];
|
||||
total: number;
|
||||
};
|
||||
expect(body.total).toBe(1);
|
||||
});
|
||||
|
||||
it("GET /{id} returns the caller's own job", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request(`/imports/${progress.jobId}`);
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it("GET /{id} returns 404 when the job is owned by a different user", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: bob });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request(`/imports/${progress.jobId}`);
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.headers.get("content-type")).toContain(
|
||||
"application/problem+json"
|
||||
);
|
||||
});
|
||||
|
||||
it("DELETE /{id} deletes the job", { timeout: 5000 }, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request(`/imports/${progress.jobId}`, {
|
||||
method: "DELETE",
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const body = (await res.json()) as { deleted: boolean };
|
||||
expect(body.deleted).toBe(true);
|
||||
});
|
||||
|
||||
it("DELETE /{id} returns 404 for a missing job", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: alice });
|
||||
const res = await app.request("/imports/missing", {
|
||||
method: "DELETE",
|
||||
});
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it("renders unknown thrown errors through app.onError as 500", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
// Force a non-ImportError throw from inside the service surface
|
||||
// by overriding listJobs to throw synchronously.
|
||||
service.listJobs = (() => {
|
||||
throw new Error("kaboom");
|
||||
}) as ImportService["listJobs"];
|
||||
const res = await app.request("/imports");
|
||||
expect(res.status).toBe(500);
|
||||
});
|
||||
|
||||
it("POST / passes dryRun, batchSize, skipDuplicates through", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: alice });
|
||||
const res = await app.request("/imports", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
format: "csv",
|
||||
data: "name\nAda\nAda",
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "name",
|
||||
targetField: "name",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
dryRun: true,
|
||||
batchSize: 1,
|
||||
skipDuplicates: true,
|
||||
}),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it("GET / honours query.status, limit, offset", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request(
|
||||
"/imports?status=completed&limit=10&offset=0"
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
});
|
||||
|
||||
it("GET /{id} response includes startedAt + completedAt", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request(`/imports/${progress.jobId}`);
|
||||
const body = (await res.json()) as {
|
||||
startedAt?: string;
|
||||
completedAt?: string;
|
||||
};
|
||||
expect(body.startedAt).toBeDefined();
|
||||
expect(body.completedAt).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,325 @@
|
||||
import { createRoute, OpenAPIHono } from "@hono/zod-openapi";
|
||||
import { requireAuth } from "@suaveplan/api-plugin-auth";
|
||||
import z from "zod";
|
||||
import {
|
||||
CreateImportJobSchema,
|
||||
ImportJobListQuerySchema,
|
||||
ImportJobResponseSchema,
|
||||
PreviewRequestSchema,
|
||||
PreviewResponseSchema,
|
||||
} from "../schemas/schemas.js";
|
||||
import type { ImportService } from "../service/service.js";
|
||||
import type { ColumnMapping, ImportProgress } from "../types/types.js";
|
||||
import {
|
||||
PROBLEM_JSON_CONTENT_TYPE,
|
||||
type ProblemDetails,
|
||||
renderProblem,
|
||||
} from "./problem-json.js";
|
||||
import { createProgressSseHandler } from "./progress-sse/progress-sse.js";
|
||||
|
||||
/** Dependencies injected into `createImportRoutes`. */
|
||||
export interface ImportRoutesDeps {
|
||||
/** Canonical import service constructed by the plugin's `onInit`. */
|
||||
readonly service: ImportService;
|
||||
}
|
||||
|
||||
const IdParamSchema = z.object({ id: z.string().min(1) });
|
||||
const DeleteResponseSchema = z.object({ deleted: z.boolean() });
|
||||
const ListResponseSchema = z.object({
|
||||
items: z.array(ImportJobResponseSchema),
|
||||
total: z.number().int().nonnegative(),
|
||||
});
|
||||
const ProblemDetailsSchema = z
|
||||
.object({
|
||||
type: z.string(),
|
||||
title: z.string(),
|
||||
status: z.number().int(),
|
||||
detail: z.string(),
|
||||
code: z.string().optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const PROBLEM_RESPONSE = {
|
||||
description: "RFC 7807 problem details",
|
||||
content: { [PROBLEM_JSON_CONTENT_TYPE]: { schema: ProblemDetailsSchema } },
|
||||
} as const;
|
||||
|
||||
interface ProblemResult {
|
||||
readonly body: ProblemDetails;
|
||||
readonly status: 400 | 401 | 403 | 404 | 413 | 415 | 422 | 500 | 502 | 504;
|
||||
}
|
||||
|
||||
function asProblem(err: unknown): ProblemResult {
|
||||
const body = renderProblem(err);
|
||||
return { body, status: body.status as ProblemResult["status"] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the auth-gated `OpenAPIHono` router for the import plugin.
|
||||
* Five routes are registered, each wrapped in `requireAuth()` so an
|
||||
* anonymous request returns 401 problem+json before reaching the
|
||||
* handler. Handlers read `c.get("user").id` as the owner id; the
|
||||
* service's owner-scoping then maps foreign-job lookups to 404.
|
||||
*/
|
||||
export function createImportRoutes(deps: ImportRoutesDeps): OpenAPIHono {
|
||||
const app = new OpenAPIHono();
|
||||
|
||||
// Catch thrown errors at the router boundary so problem+json is
|
||||
// surfaced uniformly even for synchronous throws that escape a
|
||||
// handler's `try`/`catch`.
|
||||
app.onError((err, c) => {
|
||||
const { body, status } = asProblem(err);
|
||||
return c.body(JSON.stringify(body), status, {
|
||||
"content-type": PROBLEM_JSON_CONTENT_TYPE,
|
||||
});
|
||||
});
|
||||
|
||||
// Auth middleware first — anonymous requests throw `AUTH.UNAUTHENTICATED`
|
||||
// and the `app.onError` hook above renders it as 401 problem+json.
|
||||
app.use("/*", requireAuth());
|
||||
|
||||
registerPreviewRoute(app, deps);
|
||||
registerCreateRoute(app, deps);
|
||||
registerListRoute(app, deps);
|
||||
registerGetRoute(app, deps);
|
||||
registerProgressSseRoute(app, deps);
|
||||
registerDeleteRoute(app, deps);
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
function registerProgressSseRoute(
|
||||
app: OpenAPIHono,
|
||||
deps: ImportRoutesDeps
|
||||
): void {
|
||||
// SSE streams are handled outside of the OpenAPI route registry —
|
||||
// `c.body(ReadableStream, ...)` does not conform to a static
|
||||
// response schema and `@hono/zod-openapi` would attempt to validate
|
||||
// the body. Mounting via `app.get` preserves the auth middleware
|
||||
// (already registered with `app.use("/*", requireAuth())`) and the
|
||||
// shared `app.onError` problem+json hook.
|
||||
app.get("/:id/progress", createProgressSseHandler(deps));
|
||||
}
|
||||
|
||||
function progressToResponse(p: ImportProgress): {
|
||||
jobId: string;
|
||||
ownerId: string;
|
||||
status: ImportProgress["status"];
|
||||
totalRows: number;
|
||||
processedRows: number;
|
||||
successCount: number;
|
||||
errorCount: number;
|
||||
errors: ImportProgress["errors"];
|
||||
startedAt?: string;
|
||||
completedAt?: string;
|
||||
} {
|
||||
return {
|
||||
jobId: p.jobId,
|
||||
ownerId: p.ownerId,
|
||||
status: p.status,
|
||||
totalRows: p.totalRows,
|
||||
processedRows: p.processedRows,
|
||||
successCount: p.successCount,
|
||||
errorCount: p.errorCount,
|
||||
errors: [...p.errors],
|
||||
...(p.startedAt !== undefined && { startedAt: p.startedAt }),
|
||||
...(p.completedAt !== undefined && { completedAt: p.completedAt }),
|
||||
};
|
||||
}
|
||||
|
||||
function ownerOf(c: { get: (key: "user") => unknown }): string {
|
||||
// `requireAuth()` middleware throws before any handler runs when
|
||||
// `user` is null/undefined, so the principal is guaranteed-present
|
||||
// here. We cast through a structural view to avoid coupling the
|
||||
// route layer to the auth plugin's full Principal shape.
|
||||
const user = c.get("user") as { id: string };
|
||||
return user.id;
|
||||
}
|
||||
|
||||
interface WireMapping {
|
||||
readonly sourceColumn: string;
|
||||
readonly targetField: string;
|
||||
readonly required?: boolean | undefined;
|
||||
}
|
||||
|
||||
function toMappings(wire: readonly WireMapping[]): ColumnMapping[] {
|
||||
return wire.map((m) => ({
|
||||
sourceColumn: m.sourceColumn,
|
||||
targetField: m.targetField,
|
||||
...(m.required !== undefined && { required: m.required }),
|
||||
}));
|
||||
}
|
||||
|
||||
function registerPreviewRoute(app: OpenAPIHono, deps: ImportRoutesDeps): void {
|
||||
const route = createRoute({
|
||||
method: "post",
|
||||
path: "/preview",
|
||||
tags: ["imports"],
|
||||
summary: "Preview an import payload",
|
||||
request: {
|
||||
body: {
|
||||
content: {
|
||||
"application/json": { schema: PreviewRequestSchema },
|
||||
},
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
200: {
|
||||
description: "Preview result",
|
||||
content: {
|
||||
"application/json": { schema: PreviewResponseSchema },
|
||||
},
|
||||
},
|
||||
400: PROBLEM_RESPONSE,
|
||||
},
|
||||
});
|
||||
|
||||
app.openapi(route, (c) => {
|
||||
const body = c.req.valid("json");
|
||||
const result = deps.service.preview(body.format, body.data);
|
||||
return c.json(
|
||||
{
|
||||
totalRows: result.totalRows,
|
||||
columns: [...result.columns],
|
||||
sampleRows: result.sampleRows,
|
||||
mappings: result.mappings.map((m) => ({
|
||||
sourceColumn: m.sourceColumn,
|
||||
targetField: m.targetField,
|
||||
})),
|
||||
},
|
||||
200
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function registerCreateRoute(app: OpenAPIHono, deps: ImportRoutesDeps): void {
|
||||
const route = createRoute({
|
||||
method: "post",
|
||||
path: "/",
|
||||
tags: ["imports"],
|
||||
summary: "Start an import job",
|
||||
request: {
|
||||
body: {
|
||||
content: {
|
||||
"application/json": { schema: CreateImportJobSchema },
|
||||
},
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
200: {
|
||||
description: "Import job snapshot",
|
||||
content: {
|
||||
"application/json": { schema: ImportJobResponseSchema },
|
||||
},
|
||||
},
|
||||
400: PROBLEM_RESPONSE,
|
||||
},
|
||||
});
|
||||
|
||||
app.openapi(route, async (c) => {
|
||||
const body = c.req.valid("json");
|
||||
const ownerId = ownerOf(c);
|
||||
const progress = await deps.service.startImport(
|
||||
{
|
||||
format: body.format,
|
||||
data: body.data,
|
||||
mappings: toMappings(body.mappings),
|
||||
...(body.dryRun !== undefined && { dryRun: body.dryRun }),
|
||||
...(body.batchSize !== undefined && {
|
||||
batchSize: body.batchSize,
|
||||
}),
|
||||
...(body.skipDuplicates !== undefined && {
|
||||
skipDuplicates: body.skipDuplicates,
|
||||
}),
|
||||
},
|
||||
{ ownerId }
|
||||
);
|
||||
return c.json(progressToResponse(progress), 200);
|
||||
});
|
||||
}
|
||||
|
||||
function registerListRoute(app: OpenAPIHono, deps: ImportRoutesDeps): void {
|
||||
const route = createRoute({
|
||||
method: "get",
|
||||
path: "/",
|
||||
tags: ["imports"],
|
||||
summary: "List import jobs for the calling user",
|
||||
request: { query: ImportJobListQuerySchema },
|
||||
responses: {
|
||||
200: {
|
||||
description: "List of import jobs",
|
||||
content: {
|
||||
"application/json": { schema: ListResponseSchema },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
app.openapi(route, (c) => {
|
||||
const query = c.req.valid("query");
|
||||
const ownerId = ownerOf(c);
|
||||
const items = deps.service.listJobs(ownerId, {
|
||||
...(query.status !== undefined && { status: query.status }),
|
||||
...(query.limit !== undefined && { limit: query.limit }),
|
||||
...(query.offset !== undefined && { offset: query.offset }),
|
||||
});
|
||||
return c.json(
|
||||
{
|
||||
items: items.map(progressToResponse),
|
||||
total: items.length,
|
||||
},
|
||||
200
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function registerGetRoute(app: OpenAPIHono, deps: ImportRoutesDeps): void {
|
||||
const route = createRoute({
|
||||
method: "get",
|
||||
path: "/{id}",
|
||||
tags: ["imports"],
|
||||
summary: "Fetch a single import job",
|
||||
request: { params: IdParamSchema },
|
||||
responses: {
|
||||
200: {
|
||||
description: "Import job snapshot",
|
||||
content: {
|
||||
"application/json": { schema: ImportJobResponseSchema },
|
||||
},
|
||||
},
|
||||
404: PROBLEM_RESPONSE,
|
||||
},
|
||||
});
|
||||
|
||||
app.openapi(route, (c) => {
|
||||
const { id } = c.req.valid("param");
|
||||
const ownerId = ownerOf(c);
|
||||
const progress = deps.service.getJob(id, ownerId);
|
||||
return c.json(progressToResponse(progress), 200);
|
||||
});
|
||||
}
|
||||
|
||||
function registerDeleteRoute(app: OpenAPIHono, deps: ImportRoutesDeps): void {
|
||||
const route = createRoute({
|
||||
method: "delete",
|
||||
path: "/{id}",
|
||||
tags: ["imports"],
|
||||
summary: "Delete an import job",
|
||||
request: { params: IdParamSchema },
|
||||
responses: {
|
||||
200: {
|
||||
description: "Deletion outcome",
|
||||
content: {
|
||||
"application/json": { schema: DeleteResponseSchema },
|
||||
},
|
||||
},
|
||||
404: PROBLEM_RESPONSE,
|
||||
},
|
||||
});
|
||||
|
||||
app.openapi(route, async (c) => {
|
||||
const { id } = c.req.valid("param");
|
||||
const ownerId = ownerOf(c);
|
||||
const deleted = await deps.service.deleteJob(id, ownerId);
|
||||
return c.json({ deleted }, 200);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export { createImportRoutes, type ImportRoutesDeps } from "./build-routes.js";
|
||||
export {
|
||||
type HonoLike,
|
||||
PROBLEM_JSON_CONTENT_TYPE,
|
||||
type ProblemDetails,
|
||||
renderProblem,
|
||||
respondProblemJson,
|
||||
} from "./problem-json.js";
|
||||
@@ -0,0 +1,70 @@
|
||||
# `routes/problem-json` — RFC 7807 problem+json helper
|
||||
|
||||
## Purpose
|
||||
|
||||
Bridges thrown errors from the route handlers into RFC 7807
|
||||
`application/problem+json` responses. Reads the HTTP status from the
|
||||
error's `context.statusCode` when present so the canonical
|
||||
`ImportError` raised by the service / plugin layer surfaces with the
|
||||
correct HTTP code without a code → status lookup table in the router.
|
||||
|
||||
Unknown throwables fall back to a generic 500 so the router never
|
||||
crashes — the renderer is the safety net for non-`SuaveplanError`
|
||||
exceptions that escape upstream code.
|
||||
|
||||
## Features
|
||||
|
||||
- `respondProblemJson(c, err)` — writes the `Response` to the Hono
|
||||
context with `content-type: application/problem+json`.
|
||||
- `renderProblem(err)` — pure render returning a `ProblemDetails`
|
||||
object (used by tests + alt transports).
|
||||
- `PROBLEM_JSON_CONTENT_TYPE` constant for downstream re-use.
|
||||
- Native `Error` paths preserve the `name` so debugging upstream
|
||||
failures stays readable.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { respondProblemJson } from "@suaveplan/api-plugin-import";
|
||||
|
||||
app.get("/foo", (c) => {
|
||||
try {
|
||||
throw new ImportError(IMPORT_CODES.NOT_FOUND, "missing", {
|
||||
context: { statusCode: 404 },
|
||||
});
|
||||
} catch (err) {
|
||||
return respondProblemJson(c, err);
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { renderProblem } from "@suaveplan/api-plugin-import";
|
||||
|
||||
// Pure render for use in alternate transports (gRPC adapter, queue worker)
|
||||
const body = renderProblem(new Error("boom"));
|
||||
// body === { type: "urn:suaveplan:error:internal", title: "Error", status: 500, detail: "boom" }
|
||||
const message = JSON.stringify(body);
|
||||
publishToDeadLetterQueue(message);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `respondProblemJson(c: HonoLike, err: unknown): Response`
|
||||
- `renderProblem(err: unknown): ProblemDetails`
|
||||
- `interface HonoLike { header(name, value); body(data, status): Response }`
|
||||
- `interface ProblemDetails { type; title; status; detail; code?; [k: string]: unknown }`
|
||||
- `const PROBLEM_JSON_CONTENT_TYPE = "application/problem+json"`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The `type` field is built as `urn:suaveplan:error:<lowercased-code>`
|
||||
so consumers can branch on the URN without parsing the body.
|
||||
- Status hints below 400 (or missing) collapse to 500 — clients should
|
||||
never receive a problem+json body with a 2xx status code.
|
||||
- The `Error.name` fallback (`"InternalError"`) protects against
|
||||
thrown objects whose `name` was wiped by a downstream serializer.
|
||||
- `renderProblem` returns a `ProblemDetails` literal so consumers that
|
||||
need to merge additional fields (e.g. `instance`) can spread it.
|
||||
@@ -0,0 +1,88 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { IMPORT_CODES, ImportError } from "../error-codes/error-codes.js";
|
||||
import type { HonoLike } from "./problem-json.js";
|
||||
import {
|
||||
PROBLEM_JSON_CONTENT_TYPE,
|
||||
renderProblem,
|
||||
respondProblemJson,
|
||||
} from "./problem-json.js";
|
||||
|
||||
function makeCtx(): { c: HonoLike; headers: Record<string, string> } {
|
||||
const headers: Record<string, string> = {};
|
||||
const c: HonoLike = {
|
||||
header: (name, value) => {
|
||||
headers[name] = value;
|
||||
},
|
||||
body: (data, status) =>
|
||||
new Response(data, {
|
||||
status,
|
||||
headers,
|
||||
}),
|
||||
};
|
||||
return { c, headers };
|
||||
}
|
||||
|
||||
describe("renderProblem", () => {
|
||||
it("renders SuaveplanError with the context statusCode", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = new ImportError(IMPORT_CODES.NOT_FOUND, "missing job", {
|
||||
context: { statusCode: 404 },
|
||||
});
|
||||
const body = renderProblem(err);
|
||||
expect(body.status).toBe(404);
|
||||
expect(body.code).toBe(IMPORT_CODES.NOT_FOUND);
|
||||
expect(body.detail).toBe("missing job");
|
||||
expect(body.type).toContain("import.not_found");
|
||||
});
|
||||
|
||||
it("defaults to 500 when statusCode is missing or invalid", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = new ImportError(IMPORT_CODES.STORAGE_ERROR, "boom");
|
||||
expect(renderProblem(err).status).toBe(500);
|
||||
const err2 = new ImportError(IMPORT_CODES.STORAGE_ERROR, "boom2", {
|
||||
context: { statusCode: 200 },
|
||||
});
|
||||
expect(renderProblem(err2).status).toBe(500);
|
||||
});
|
||||
|
||||
it("renders unknown throwables as a generic 500", { timeout: 5000 }, () => {
|
||||
expect(renderProblem("just a string").status).toBe(500);
|
||||
expect(renderProblem("just a string").title).toBe("InternalError");
|
||||
});
|
||||
|
||||
it("wraps native Error instances", { timeout: 5000 }, () => {
|
||||
const body = renderProblem(new TypeError("bad arg"));
|
||||
expect(body.status).toBe(500);
|
||||
expect(body.title).toBe("TypeError");
|
||||
expect(body.detail).toBe("bad arg");
|
||||
});
|
||||
|
||||
it("fills in placeholder fields when an Error lacks name/message", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = new Error();
|
||||
err.name = "";
|
||||
const body = renderProblem(err);
|
||||
expect(body.title).toBe("InternalError");
|
||||
expect(body.detail).toBe("Internal Server Error");
|
||||
});
|
||||
});
|
||||
|
||||
describe("respondProblemJson", () => {
|
||||
it("writes problem+json content-type and serialised body", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { c, headers } = makeCtx();
|
||||
const err = new ImportError(IMPORT_CODES.VALIDATION, "bad input", {
|
||||
context: { statusCode: 400 },
|
||||
});
|
||||
const res = respondProblemJson(c, err);
|
||||
expect(headers["content-type"]).toBe(PROBLEM_JSON_CONTENT_TYPE);
|
||||
expect(res.status).toBe(400);
|
||||
const text = await res.text();
|
||||
const parsed = JSON.parse(text);
|
||||
expect(parsed.code).toBe(IMPORT_CODES.VALIDATION);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import { SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
/** Standard RFC 7807 content-type. */
|
||||
export const PROBLEM_JSON_CONTENT_TYPE = "application/problem+json";
|
||||
|
||||
/** Minimal structural Hono context surface relied on by `respondProblemJson`. */
|
||||
export interface HonoLike {
|
||||
header(name: string, value: string): void;
|
||||
body(data: string, status: number): Response;
|
||||
}
|
||||
|
||||
/** RFC 7807 problem-details body. */
|
||||
export interface ProblemDetails {
|
||||
readonly type: string;
|
||||
readonly title: string;
|
||||
readonly status: number;
|
||||
readonly detail: string;
|
||||
readonly code?: string;
|
||||
readonly [key: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert an arbitrary error into an RFC 7807 problem-details body and
|
||||
* write it to the Hono context. `SuaveplanError` subclasses surface
|
||||
* their `code` and read `context.statusCode` for the HTTP code;
|
||||
* unknown throwables fall through to a generic 500 with `detail:
|
||||
* "Internal Server Error"`.
|
||||
*/
|
||||
export function respondProblemJson(c: HonoLike, err: unknown): Response {
|
||||
const body = renderProblem(err);
|
||||
c.header("content-type", PROBLEM_JSON_CONTENT_TYPE);
|
||||
return c.body(JSON.stringify(body), body.status);
|
||||
}
|
||||
|
||||
/** Pure render — useful for tests that don't construct a Hono context. */
|
||||
export function renderProblem(err: unknown): ProblemDetails {
|
||||
if (err instanceof SuaveplanError) {
|
||||
const ctx = err.context as { statusCode?: number } | undefined;
|
||||
const status =
|
||||
typeof ctx?.statusCode === "number" && ctx.statusCode >= 400
|
||||
? ctx.statusCode
|
||||
: 500;
|
||||
return {
|
||||
type: `urn:suaveplan:error:${err.code.toLowerCase()}`,
|
||||
title: err.name,
|
||||
status,
|
||||
detail: err.message,
|
||||
code: err.code,
|
||||
};
|
||||
}
|
||||
if (err instanceof Error) {
|
||||
return {
|
||||
type: "urn:suaveplan:error:internal",
|
||||
title: err.name || "InternalError",
|
||||
status: 500,
|
||||
detail: err.message || "Internal Server Error",
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "urn:suaveplan:error:internal",
|
||||
title: "InternalError",
|
||||
status: 500,
|
||||
detail: "Internal Server Error",
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
createProgressSseHandler,
|
||||
formatProgressFrame,
|
||||
type ProgressSseDeps,
|
||||
type ProgressSseEventKind,
|
||||
} from "./progress-sse.js";
|
||||
@@ -0,0 +1,124 @@
|
||||
# `routes/progress-sse` — Server-Sent Events progress push
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Exposes `GET /imports/:id/progress` returning a long-lived
|
||||
`text/event-stream` so clients can subscribe to a job's real-time
|
||||
progress without polling. The handler is driven entirely off the
|
||||
existing `ImportService.bus` — three subscriptions (`import.progress`,
|
||||
`import.completed`, `import.failed`) are wired into a
|
||||
`ReadableStream<Uint8Array>` and detached on terminal events or client
|
||||
abort. This module was deferred from the original
|
||||
`add-api-plugin-import` migration's `Excluded` list and ships
|
||||
independently via the `add-import-sse-progress` change.
|
||||
|
||||
## Features
|
||||
|
||||
- Three SSE event kinds — `progress` (per `import.progress` tick),
|
||||
`complete` (terminal, on `import.completed`), `failed` (terminal, on
|
||||
`import.failed`).
|
||||
- Initial frame snapshot from `service.getJob` so clients see current
|
||||
state before the next bus tick.
|
||||
- Owner-scoped — foreign-owner / missing-job lookups return HTTP 404
|
||||
RFC 7807 problem+json (`IMPORT.NOT_FOUND`); anonymous requests return
|
||||
HTTP 401 via the existing `requireAuth()` middleware.
|
||||
- Per-job filtering on every subscription callback so a long-lived
|
||||
connection cannot leak progress for other jobs.
|
||||
- `AbortSignal` lifecycle — client disconnect detaches every listener
|
||||
and emits `import.sse.disconnected{reason: "client"}`.
|
||||
- OTel observability — `import.sse.connected`,
|
||||
`import.sse.events_sent{kind}`, `import.sse.disconnected{reason}`.
|
||||
- Pure stream — no polling timer, no background interval, no
|
||||
heartbeat (job runs are short-lived so proxy-keepalive concerns are
|
||||
handled by upstream LB tuning, not by the handler).
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { createImportRoutes, ImportService } from "@suaveplan/api-plugin-import";
|
||||
import { Hono } from "hono";
|
||||
|
||||
const service = new ImportService();
|
||||
const host = new Hono();
|
||||
host.use("*", async (c, next) => {
|
||||
c.set("user", { id: "alice" });
|
||||
await next();
|
||||
});
|
||||
host.route("/imports", createImportRoutes({ service }));
|
||||
|
||||
// Client side:
|
||||
const es = new EventSource("/imports/job-1/progress");
|
||||
es.addEventListener("progress", (e) => console.log(JSON.parse(e.data)));
|
||||
es.addEventListener("complete", () => es.close());
|
||||
es.addEventListener("failed", () => es.close());
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
createProgressSseHandler,
|
||||
ImportService,
|
||||
} from "@suaveplan/api-plugin-import";
|
||||
import { Hono } from "hono";
|
||||
|
||||
const service = new ImportService();
|
||||
const app = new Hono();
|
||||
app.use("*", async (c, next) => {
|
||||
c.set("user", { id: "alice" });
|
||||
await next();
|
||||
});
|
||||
// Mount stand-alone (without the rest of `createImportRoutes`).
|
||||
app.get("/jobs/:id/progress", createProgressSseHandler({ service }));
|
||||
|
||||
// Drive the service from a worker; every emitted progress event is
|
||||
// fanned to every connected client whose `:id` matches.
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/jobs/job-1/progress", { signal: controller.signal })
|
||||
);
|
||||
const reader = res.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = "";
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
const frames = buf.split("\n\n");
|
||||
buf = frames.pop() ?? "";
|
||||
for (const frame of frames) console.log(frame);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `createProgressSseHandler(deps: ProgressSseDeps): (c: Context) => Response`
|
||||
- `formatProgressFrame(kind: ProgressSseEventKind, seq: number, payload: ImportProgress): string`
|
||||
- `interface ProgressSseDeps { service: ImportService }`
|
||||
- `type ProgressSseEventKind = "progress" | "complete" | "failed"`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The handler is registered via `app.get("/:id/progress", ...)` rather
|
||||
than `app.openapi(...)` because `c.body(ReadableStream, ...)` does not
|
||||
conform to a static OpenAPI response schema — the parent router's
|
||||
`app.use("/*", requireAuth())` and `app.onError(...)` still apply.
|
||||
- Subscriptions are filtered by `payload.jobId === jobId` so the bus
|
||||
cannot leak foreign-job progress to a connection. The router's
|
||||
authoritative ownership check happens once via `service.getJob` at
|
||||
connection-open time; subsequent leak prevention is structural.
|
||||
- The terminal `complete` / `failed` branch in `start` handles the
|
||||
edge case where a job reached terminal state between `getJob` and
|
||||
the bus subscription — without it, a client would be stuck on the
|
||||
initial `progress` frame forever.
|
||||
- The OTel counters are read through `getImportInstruments()` rather
|
||||
than cached at module scope so `resetImportInstruments()` (used by
|
||||
test setup) rebinds correctly.
|
||||
- The response headers (`Cache-Control: no-cache, no-transform`,
|
||||
`X-Accel-Buffering: no`, `Connection: keep-alive`) match the
|
||||
`@suaveplan/api-plugin-sse` handler so reverse proxies (nginx,
|
||||
cloudflare) do not buffer or coalesce the stream.
|
||||
@@ -0,0 +1,644 @@
|
||||
import { metrics } from "@opentelemetry/api";
|
||||
import {
|
||||
AggregationTemporality,
|
||||
InMemoryMetricExporter,
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
type ResourceMetrics,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import type { Principal } from "@suaveplan/api-plugin-auth";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { Hono } from "hono";
|
||||
import { resetImportInstruments } from "../../otel/instruments.js";
|
||||
import { ImportService } from "../../service/service.js";
|
||||
import type { ImportProgress } from "../../types/types.js";
|
||||
import { createImportRoutes } from "../build-routes.js";
|
||||
|
||||
interface FixtureOpts {
|
||||
readonly principal: Principal | null;
|
||||
}
|
||||
|
||||
interface MetricsCapture {
|
||||
readonly provider: MeterProvider;
|
||||
readonly exporter: InMemoryMetricExporter;
|
||||
}
|
||||
|
||||
function buildApp(opts: FixtureOpts): {
|
||||
app: Hono;
|
||||
service: ImportService;
|
||||
} {
|
||||
resetImportInstruments();
|
||||
let n = 0;
|
||||
const service = new ImportService({
|
||||
idFactory: () => `job-${++n}`,
|
||||
});
|
||||
const host = new Hono();
|
||||
host.use("*", async (c, next) => {
|
||||
c.set("user", opts.principal);
|
||||
await next();
|
||||
});
|
||||
host.route("/imports", createImportRoutes({ service }));
|
||||
return { app: host, service };
|
||||
}
|
||||
|
||||
function installMeterProvider(): MetricsCapture {
|
||||
const exporter = new InMemoryMetricExporter(
|
||||
AggregationTemporality.CUMULATIVE
|
||||
);
|
||||
const reader = new PeriodicExportingMetricReader({
|
||||
exporter,
|
||||
exportIntervalMillis: 60_000,
|
||||
});
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
// `metrics.setGlobalMeterProvider` follows OTel's
|
||||
// "register-once" semantics: subsequent calls silently fail when
|
||||
// a previous test in the same worker has already registered. We
|
||||
// explicitly `disable()` first so each test owns the global.
|
||||
metrics.disable();
|
||||
metrics.setGlobalMeterProvider(provider);
|
||||
resetImportInstruments();
|
||||
return { provider, exporter };
|
||||
}
|
||||
|
||||
async function flushAndRead(
|
||||
capture: MetricsCapture
|
||||
): Promise<ResourceMetrics[]> {
|
||||
await capture.provider.forceFlush();
|
||||
return capture.exporter.getMetrics();
|
||||
}
|
||||
|
||||
function sumCounter(
|
||||
collected: ResourceMetrics[],
|
||||
name: string,
|
||||
attrFilter?: (a: Record<string, unknown>) => boolean
|
||||
): number {
|
||||
let total = 0;
|
||||
for (const rm of collected) {
|
||||
for (const sm of rm.scopeMetrics) {
|
||||
for (const m of sm.metrics) {
|
||||
if (m.descriptor.name !== name) continue;
|
||||
for (const dp of m.dataPoints as ReadonlyArray<{
|
||||
value: number;
|
||||
attributes: Record<string, unknown>;
|
||||
}>) {
|
||||
if (
|
||||
attrFilter !== undefined &&
|
||||
!attrFilter(dp.attributes)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
total += dp.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
async function readFrames(
|
||||
res: Response,
|
||||
until: (frame: string) => boolean,
|
||||
timeoutMs = 2000
|
||||
): Promise<string[]> {
|
||||
const body = res.body;
|
||||
if (body === null) throw new Error("no body");
|
||||
const reader = body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const frames: string[] = [];
|
||||
let buf = "";
|
||||
const start = Date.now();
|
||||
try {
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
let idx = buf.indexOf("\n\n");
|
||||
while (idx !== -1) {
|
||||
const frame = buf.slice(0, idx);
|
||||
buf = buf.slice(idx + 2);
|
||||
if (frame.length > 0) frames.push(frame);
|
||||
if (until(frame)) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* already cancelled */
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
idx = buf.indexOf("\n\n");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
/* already cancelled */
|
||||
}
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
function parseFrame(frame: string): {
|
||||
id: string;
|
||||
event: string;
|
||||
data: ImportProgress;
|
||||
} {
|
||||
const lines = frame.split("\n");
|
||||
let id = "";
|
||||
let ev = "";
|
||||
let dataLine = "";
|
||||
for (const line of lines) {
|
||||
if (line.startsWith("id: ")) id = line.slice(4);
|
||||
else if (line.startsWith("event: ")) ev = line.slice(7);
|
||||
else if (line.startsWith("data: ")) dataLine = line.slice(6);
|
||||
}
|
||||
return {
|
||||
id,
|
||||
event: ev,
|
||||
data: JSON.parse(dataLine) as ImportProgress,
|
||||
};
|
||||
}
|
||||
|
||||
const alice: Principal = { id: "alice", emailVerified: true, roles: [] };
|
||||
const bob: Principal = { id: "bob", emailVerified: true, roles: [] };
|
||||
|
||||
const importingSnapshot = (
|
||||
jobId: string,
|
||||
processed: number
|
||||
): ImportProgress => ({
|
||||
jobId,
|
||||
ownerId: "alice",
|
||||
status: "importing",
|
||||
totalRows: 10,
|
||||
processedRows: processed,
|
||||
successCount: processed,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
startedAt: new Date(0).toISOString(),
|
||||
});
|
||||
|
||||
describe("progress-sse route", () => {
|
||||
it("opens a text/event-stream and emits an initial progress frame", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
const stub = importingSnapshot("job-1", 3);
|
||||
service.getJob = ((id: string, owner: string) => {
|
||||
if (id !== "job-1" || owner !== "alice") {
|
||||
throw new Error("unexpected");
|
||||
}
|
||||
return stub;
|
||||
}) as ImportService["getJob"];
|
||||
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("content-type")).toContain("text/event-stream");
|
||||
expect(res.headers.get("cache-control")).toContain("no-cache");
|
||||
expect(res.headers.get("x-accel-buffering")).toBe("no");
|
||||
|
||||
const frames = await readFrames(res, () => true, 1500);
|
||||
controller.abort();
|
||||
expect(frames.length).toBeGreaterThanOrEqual(1);
|
||||
const parsed = parseFrame(frames[0] as string);
|
||||
expect(parsed.event).toBe("progress");
|
||||
expect(parsed.data.jobId).toBe("job-1");
|
||||
expect(parsed.data.processedRows).toBe(3);
|
||||
expect(parsed.id).toBe("job-1-1");
|
||||
});
|
||||
|
||||
it("emits a progress frame for every bus progress event", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
setTimeout(() => {
|
||||
service.bus.emit("import.progress", importingSnapshot("job-1", 4));
|
||||
service.bus.emit("import.progress", importingSnapshot("job-1", 8));
|
||||
}, 10);
|
||||
|
||||
const frames = await readFrames(
|
||||
res,
|
||||
(f) => f.includes("data:") && f.includes(`"processedRows":8`),
|
||||
2000
|
||||
);
|
||||
controller.abort();
|
||||
const progressFrames = frames.filter((f) =>
|
||||
f.includes("event: progress")
|
||||
);
|
||||
expect(progressFrames.length).toBeGreaterThanOrEqual(3);
|
||||
const ticks = progressFrames.map(
|
||||
(f) => parseFrame(f).data.processedRows
|
||||
);
|
||||
expect(ticks).toEqual([0, 4, 8]);
|
||||
});
|
||||
|
||||
it("emits a complete frame and closes on import.completed", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
setTimeout(() => {
|
||||
service.bus.emit("import.completed", {
|
||||
...importingSnapshot("job-1", 10),
|
||||
status: "completed",
|
||||
completedAt: new Date(1).toISOString(),
|
||||
});
|
||||
}, 10);
|
||||
|
||||
const frames = await readFrames(
|
||||
res,
|
||||
(f) => f.includes("event: complete"),
|
||||
2000
|
||||
);
|
||||
controller.abort();
|
||||
const completeFrame = frames.find((f) => f.includes("event: complete"));
|
||||
expect(completeFrame).toBeDefined();
|
||||
expect(parseFrame(completeFrame as string).data.processedRows).toBe(10);
|
||||
});
|
||||
|
||||
it("emits a failed frame and closes on import.failed", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
setTimeout(() => {
|
||||
service.bus.emit("import.failed", {
|
||||
...importingSnapshot("job-1", 5),
|
||||
status: "failed",
|
||||
errorCount: 5,
|
||||
completedAt: new Date(1).toISOString(),
|
||||
});
|
||||
}, 10);
|
||||
|
||||
const frames = await readFrames(
|
||||
res,
|
||||
(f) => f.includes("event: failed"),
|
||||
2000
|
||||
);
|
||||
controller.abort();
|
||||
const failedFrame = frames.find((f) => f.includes("event: failed"));
|
||||
expect(failedFrame).toBeDefined();
|
||||
expect(parseFrame(failedFrame as string).data.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("returns 404 problem+json for a foreign-owner job", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: bob });
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "name" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const res = await app.request("/imports/job-1/progress");
|
||||
expect(res.status).toBe(404);
|
||||
expect(res.headers.get("content-type")).toContain(
|
||||
"application/problem+json"
|
||||
);
|
||||
const body = (await res.json()) as { code?: string };
|
||||
expect(body.code).toBe("IMPORT.NOT_FOUND");
|
||||
});
|
||||
|
||||
it("returns 401 problem+json when the caller is anonymous", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app } = buildApp({ principal: null });
|
||||
const res = await app.request("/imports/job-1/progress");
|
||||
expect(res.status).toBe(401);
|
||||
expect(res.headers.get("content-type")).toContain(
|
||||
"application/problem+json"
|
||||
);
|
||||
});
|
||||
|
||||
it("detaches the listener and increments disconnect counter on client abort", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const capture = installMeterProvider();
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
// Snapshot the raw EventBus subscription map so we can observe
|
||||
// detach without relying on counter timing in the periodic
|
||||
// metric reader.
|
||||
const busAny = service.bus as unknown as {
|
||||
listeners: Map<string, unknown[]>;
|
||||
};
|
||||
const baseSubs = (busAny.listeners.get("import.progress") ?? []).length;
|
||||
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
// Read one chunk manually so the ReadableStream's `start`
|
||||
// callback fires (subscriptions install on first pull). We do
|
||||
// NOT cancel the reader here — that would trigger teardown
|
||||
// before we can snapshot the listener map.
|
||||
const reader = res.body?.getReader();
|
||||
if (reader === undefined) throw new Error("no body");
|
||||
await reader.read();
|
||||
const subsDuring = (busAny.listeners.get("import.progress") ?? [])
|
||||
.length;
|
||||
expect(subsDuring).toBe(baseSubs + 1);
|
||||
|
||||
// Now exercise the client-abort lifecycle.
|
||||
await reader.cancel();
|
||||
controller.abort();
|
||||
// Allow microtasks to flush the teardown handler.
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
|
||||
const subsAfter = (busAny.listeners.get("import.progress") ?? [])
|
||||
.length;
|
||||
expect(subsAfter).toBe(baseSubs);
|
||||
|
||||
const collected = await flushAndRead(capture);
|
||||
const disconnects = sumCounter(
|
||||
collected,
|
||||
"import.sse.disconnected",
|
||||
(a) => a.reason === "client"
|
||||
);
|
||||
expect(disconnects).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("increments import.sse counters on connect, send, and disconnect", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const capture = installMeterProvider();
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
setTimeout(() => {
|
||||
service.bus.emit("import.progress", importingSnapshot("job-1", 5));
|
||||
service.bus.emit("import.completed", {
|
||||
...importingSnapshot("job-1", 10),
|
||||
status: "completed",
|
||||
completedAt: new Date(1).toISOString(),
|
||||
});
|
||||
}, 10);
|
||||
await readFrames(res, (f) => f.includes("event: complete"), 2000);
|
||||
controller.abort();
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
const collected = await flushAndRead(capture);
|
||||
expect(
|
||||
sumCounter(collected, "import.sse.connected")
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
expect(
|
||||
sumCounter(
|
||||
collected,
|
||||
"import.sse.events_sent",
|
||||
(a) => a.kind === "progress"
|
||||
)
|
||||
).toBeGreaterThanOrEqual(2); // initial + bus tick
|
||||
expect(
|
||||
sumCounter(
|
||||
collected,
|
||||
"import.sse.events_sent",
|
||||
(a) => a.kind === "complete"
|
||||
)
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
expect(
|
||||
sumCounter(
|
||||
collected,
|
||||
"import.sse.disconnected",
|
||||
(a) => a.reason === "complete"
|
||||
)
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("closes immediately when initial snapshot is already completed", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
const completed: ImportProgress = {
|
||||
...importingSnapshot("job-1", 10),
|
||||
status: "completed",
|
||||
completedAt: new Date(1).toISOString(),
|
||||
};
|
||||
service.getJob = (() => completed) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
const frames = await readFrames(res, () => false, 1500);
|
||||
const completeFrame = frames.find((f) => f.includes("event: complete"));
|
||||
expect(completeFrame).toBeDefined();
|
||||
expect(parseFrame(completeFrame as string).data.status).toBe(
|
||||
"completed"
|
||||
);
|
||||
});
|
||||
|
||||
it("closes immediately when initial snapshot is already failed", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
const failed: ImportProgress = {
|
||||
...importingSnapshot("job-1", 5),
|
||||
status: "failed",
|
||||
errorCount: 5,
|
||||
completedAt: new Date(1).toISOString(),
|
||||
};
|
||||
service.getJob = (() => failed) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
const frames = await readFrames(res, () => false, 1500);
|
||||
const failedFrame = frames.find((f) => f.includes("event: failed"));
|
||||
expect(failedFrame).toBeDefined();
|
||||
expect(parseFrame(failedFrame as string).data.status).toBe("failed");
|
||||
});
|
||||
|
||||
it("closes immediately when the request signal is already aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
// The stream opened then immediately tore down; reading should
|
||||
// yield the initial progress frame and then EOF.
|
||||
const frames = await readFrames(res, () => false, 500);
|
||||
expect(frames.length).toBeGreaterThanOrEqual(1);
|
||||
const busAny = service.bus as unknown as {
|
||||
listeners: Map<string, unknown[]>;
|
||||
};
|
||||
expect((busAny.listeners.get("import.progress") ?? []).length).toBe(0);
|
||||
});
|
||||
|
||||
it("is idempotent when abort fires after a terminal bus event", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const capture = installMeterProvider();
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
setTimeout(() => {
|
||||
service.bus.emit("import.completed", {
|
||||
...importingSnapshot("job-1", 10),
|
||||
status: "completed",
|
||||
});
|
||||
}, 10);
|
||||
await readFrames(res, (f) => f.includes("event: complete"), 2000);
|
||||
// Abort AFTER the terminal event closed the stream. The abort
|
||||
// handler must not double-emit the disconnect counter.
|
||||
controller.abort();
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
|
||||
const collected = await flushAndRead(capture);
|
||||
// Exactly one disconnect — with reason="complete" — should be
|
||||
// recorded; the post-terminal abort must short-circuit.
|
||||
const complete = sumCounter(
|
||||
collected,
|
||||
"import.sse.disconnected",
|
||||
(a) => a.reason === "complete"
|
||||
);
|
||||
const client = sumCounter(
|
||||
collected,
|
||||
"import.sse.disconnected",
|
||||
(a) => a.reason === "client"
|
||||
);
|
||||
expect(complete).toBe(1);
|
||||
expect(client).toBe(0);
|
||||
});
|
||||
|
||||
it("fires the abort listener when the controller aborts mid-stream", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const capture = installMeterProvider();
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = (() =>
|
||||
importingSnapshot("job-1", 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
const reader = res.body?.getReader();
|
||||
if (reader === undefined) throw new Error("no body");
|
||||
// Read one frame so the start callback has finished and the
|
||||
// abort listener has been attached.
|
||||
await reader.read();
|
||||
// Abort BEFORE cancelling the reader so the abort listener
|
||||
// (not the cancel callback) fires the teardown.
|
||||
controller.abort();
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
await reader.cancel().catch(() => undefined);
|
||||
|
||||
const busAny = service.bus as unknown as {
|
||||
listeners: Map<string, unknown[]>;
|
||||
};
|
||||
expect((busAny.listeners.get("import.progress") ?? []).length).toBe(0);
|
||||
const collected = await flushAndRead(capture);
|
||||
expect(
|
||||
sumCounter(
|
||||
collected,
|
||||
"import.sse.disconnected",
|
||||
(a) => a.reason === "client"
|
||||
)
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("ignores bus events whose jobId does not match the connected job", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { app, service } = buildApp({ principal: alice });
|
||||
service.getJob = ((id: string) =>
|
||||
importingSnapshot(id, 0)) as ImportService["getJob"];
|
||||
const controller = new AbortController();
|
||||
const res = await app.request(
|
||||
new Request("http://t/imports/job-1/progress", {
|
||||
signal: controller.signal,
|
||||
})
|
||||
);
|
||||
setTimeout(() => {
|
||||
// Foreign-job events must be filtered at the subscription
|
||||
// callback. Only the `job-1` completion should close the
|
||||
// stream.
|
||||
service.bus.emit("import.progress", importingSnapshot("job-2", 4));
|
||||
service.bus.emit("import.failed", {
|
||||
...importingSnapshot("job-2", 4),
|
||||
status: "failed",
|
||||
});
|
||||
service.bus.emit("import.completed", {
|
||||
...importingSnapshot("job-2", 4),
|
||||
status: "completed",
|
||||
});
|
||||
service.bus.emit("import.completed", {
|
||||
...importingSnapshot("job-1", 10),
|
||||
status: "completed",
|
||||
});
|
||||
}, 10);
|
||||
const frames = await readFrames(
|
||||
res,
|
||||
(f) => f.includes("event: complete"),
|
||||
2000
|
||||
);
|
||||
controller.abort();
|
||||
// Only the initial `progress` (job-1) and one `complete`
|
||||
// (job-1) should have been written.
|
||||
const progressFrames = frames.filter((f) =>
|
||||
f.includes("event: progress")
|
||||
);
|
||||
const completeFrames = frames.filter((f) =>
|
||||
f.includes("event: complete")
|
||||
);
|
||||
const failedFrames = frames.filter((f) => f.includes("event: failed"));
|
||||
expect(progressFrames.length).toBe(1);
|
||||
expect(completeFrames.length).toBe(1);
|
||||
expect(failedFrames.length).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,222 @@
|
||||
import type { Context } from "hono";
|
||||
import { getImportInstruments } from "../../otel/instruments.js";
|
||||
import type { ImportService } from "../../service/service.js";
|
||||
import type { ImportProgress } from "../../types/types.js";
|
||||
|
||||
/** Dependencies for `registerProgressSseRoute`. */
|
||||
export interface ProgressSseDeps {
|
||||
/** Canonical import service. The route subscribes to `service.bus`. */
|
||||
readonly service: ImportService;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire format for an SSE frame as parsed by the WHATWG event-stream
|
||||
* grammar. The route emits one of three `event` kinds: `progress` for
|
||||
* every `import.progress` bus tick, `complete` as the terminal frame
|
||||
* on `import.completed`, and `failed` as the terminal frame on
|
||||
* `import.failed`. `data` is a JSON-stringified {@link ImportProgress}.
|
||||
*/
|
||||
export type ProgressSseEventKind = "progress" | "complete" | "failed";
|
||||
|
||||
/**
|
||||
* Serialise a single progress payload into the SSE `text/event-stream`
|
||||
* wire format. Exported so unit tests can pin the exact frame shape
|
||||
* without driving the streaming handler end-to-end.
|
||||
*/
|
||||
export function formatProgressFrame(
|
||||
kind: ProgressSseEventKind,
|
||||
seq: number,
|
||||
payload: ImportProgress
|
||||
): string {
|
||||
const id = `${payload.jobId}-${seq}`;
|
||||
const data = JSON.stringify(payload);
|
||||
return `id: ${id}\nevent: ${kind}\ndata: ${data}\n\n`;
|
||||
}
|
||||
|
||||
interface OwnerContextLike {
|
||||
readonly get: (key: "user") => unknown;
|
||||
}
|
||||
|
||||
function ownerOf(c: OwnerContextLike): string {
|
||||
// `requireAuth()` middleware in `build-routes.ts` throws before the
|
||||
// SSE handler runs when `user` is null/undefined, so the principal
|
||||
// is always present here.
|
||||
const user = c.get("user") as { id: string };
|
||||
return user.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the SSE handler for `GET /:id/progress`. The returned handler
|
||||
* resolves the owner from `c.get("user")`, validates job ownership via
|
||||
* `service.getJob(id, ownerId)` (which throws `IMPORT.NOT_FOUND` on a
|
||||
* foreign or missing job — mapped by the parent router's `app.onError`
|
||||
* to a 404 problem+json), then opens a `ReadableStream<Uint8Array>`
|
||||
* driven by three bus subscriptions (`import.progress`,
|
||||
* `import.completed`, `import.failed`).
|
||||
*
|
||||
* Lifecycle invariants:
|
||||
*
|
||||
* - First frame is the synchronous snapshot from `service.getJob` so
|
||||
* clients see current state before the first bus tick — important
|
||||
* when a job is queried after the run has already advanced past
|
||||
* `processedRows: 0`.
|
||||
* - The stream closes after the first terminal frame (`complete` or
|
||||
* `failed`). The bus subscriptions are detached and the disconnect
|
||||
* counter is emitted with the matching `reason` label.
|
||||
* - The request's `AbortSignal` is observed so a client disconnect
|
||||
* detaches the listeners and increments
|
||||
* `import.sse.disconnected{reason: "client"}`.
|
||||
* - Only events for the owner-scoped `jobId` reach the wire — other
|
||||
* jobs' events are filtered at the subscription callback so a
|
||||
* long-lived connection never leaks foreign progress.
|
||||
*/
|
||||
export function createProgressSseHandler(
|
||||
deps: ProgressSseDeps
|
||||
): (c: Context) => Response | Promise<Response> {
|
||||
const { service } = deps;
|
||||
return function progressSseHandler(c): Response {
|
||||
const jobId = c.req.param("id") as string;
|
||||
const ownerId = ownerOf(c);
|
||||
|
||||
// Authoritative ownership/existence check. Throws
|
||||
// `ImportError(IMPORT.NOT_FOUND)` → 404 problem+json via the
|
||||
// parent router's `app.onError` hook when the caller is not the
|
||||
// owner or the job does not exist.
|
||||
const initial = service.getJob(jobId, ownerId);
|
||||
|
||||
const instruments = getImportInstruments();
|
||||
instruments.sseConnected.add(1);
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const signal = c.req.raw.signal;
|
||||
let seq = 0;
|
||||
let closed = false;
|
||||
// Set when the WHATWG `cancel` algorithm fires (client
|
||||
// disconnect) so `teardown` can skip the redundant
|
||||
// `controllerRef.close()` call that would otherwise throw.
|
||||
let cancelFired = false;
|
||||
// `controllerRef` is assigned synchronously inside `start`
|
||||
// before any external callback (bus listener, abort handler,
|
||||
// cancel callback) can fire, so once those run it is always a
|
||||
// live controller.
|
||||
let controllerRef: ReadableStreamDefaultController<Uint8Array>;
|
||||
let unsubscribeProgress: (() => void) | undefined;
|
||||
let unsubscribeCompleted: (() => void) | undefined;
|
||||
let unsubscribeFailed: (() => void) | undefined;
|
||||
let abortHandler: (() => void) | undefined;
|
||||
|
||||
const writeFrame = (
|
||||
kind: ProgressSseEventKind,
|
||||
payload: ImportProgress
|
||||
): void => {
|
||||
// `writeFrame` is invoked from three places: the initial
|
||||
// snapshot (synchronously inside `start`, before any
|
||||
// teardown can run), the bus subscription callbacks
|
||||
// (filtered by jobId and unsubscribed inside `teardown`),
|
||||
// and the terminal-state branches in `start` (which run
|
||||
// before teardown). JavaScript's single-threaded loop
|
||||
// therefore guarantees `closed === false` here and the
|
||||
// controller is open.
|
||||
seq += 1;
|
||||
const frame = formatProgressFrame(kind, seq, payload);
|
||||
controllerRef.enqueue(encoder.encode(frame));
|
||||
instruments.sseEventsSent.add(1, { kind });
|
||||
};
|
||||
|
||||
const teardown = (reason: "complete" | "failed" | "client"): void => {
|
||||
// `teardown` is re-entry-safe because the abort listener
|
||||
// may still fire after a terminal bus event has already
|
||||
// closed the controller, and we don't want to double-emit
|
||||
// the disconnect counter.
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
unsubscribeProgress?.();
|
||||
unsubscribeCompleted?.();
|
||||
unsubscribeFailed?.();
|
||||
// The abort listener is kept registered — the `closed`
|
||||
// guard above makes a second teardown call a no-op, and
|
||||
// the signal is per-request so the listener is GC'd with
|
||||
// the request. This deliberately leaves the listener
|
||||
// attached so a post-terminal-event abort exercises the
|
||||
// idempotent re-entry path.
|
||||
instruments.sseDisconnected.add(1, { reason });
|
||||
// The WHATWG ReadableStream's `cancel` algorithm already
|
||||
// closes the controller as part of `reader.cancel()`, so
|
||||
// re-closing from the cancel callback path would throw
|
||||
// `TypeError`. We only invoke `close()` when teardown was
|
||||
// triggered by us (bus terminal event or signal abort).
|
||||
if (reason !== "client" || !cancelFired) {
|
||||
controllerRef.close();
|
||||
}
|
||||
};
|
||||
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start: (controller) => {
|
||||
controllerRef = controller;
|
||||
|
||||
// Emit initial snapshot so the client sees current state
|
||||
// before the next bus tick.
|
||||
writeFrame("progress", initial);
|
||||
|
||||
// Filter every subscription by `jobId` so a long-lived
|
||||
// connection cannot leak progress for other jobs.
|
||||
unsubscribeProgress = service.bus.on(
|
||||
"import.progress",
|
||||
(payload) => {
|
||||
if (payload.jobId !== jobId) return;
|
||||
writeFrame("progress", payload);
|
||||
}
|
||||
);
|
||||
unsubscribeCompleted = service.bus.on(
|
||||
"import.completed",
|
||||
(payload) => {
|
||||
if (payload.jobId !== jobId) return;
|
||||
writeFrame("complete", payload);
|
||||
teardown("complete");
|
||||
}
|
||||
);
|
||||
unsubscribeFailed = service.bus.on(
|
||||
"import.failed",
|
||||
(payload) => {
|
||||
if (payload.jobId !== jobId) return;
|
||||
writeFrame("failed", payload);
|
||||
teardown("failed");
|
||||
}
|
||||
);
|
||||
|
||||
// If the job has already reached a terminal state
|
||||
// between the `getJob` snapshot and the subscription,
|
||||
// emit the matching terminal frame synchronously so the
|
||||
// client is never left hanging.
|
||||
if (initial.status === "completed") {
|
||||
writeFrame("complete", initial);
|
||||
teardown("complete");
|
||||
return;
|
||||
}
|
||||
if (initial.status === "failed") {
|
||||
writeFrame("failed", initial);
|
||||
teardown("failed");
|
||||
return;
|
||||
}
|
||||
|
||||
abortHandler = () => teardown("client");
|
||||
if (signal.aborted) {
|
||||
teardown("client");
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", abortHandler);
|
||||
},
|
||||
cancel: () => {
|
||||
cancelFired = true;
|
||||
teardown("client");
|
||||
},
|
||||
});
|
||||
|
||||
return c.body(body, 200, {
|
||||
"Content-Type": "text/event-stream; charset=utf-8",
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"X-Accel-Buffering": "no",
|
||||
Connection: "keep-alive",
|
||||
});
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
export {
|
||||
type ImportPluginConfig,
|
||||
type ImportPluginOptionsInput,
|
||||
ImportPluginOptionsSchema,
|
||||
} from "./options.js";
|
||||
export {
|
||||
type ColumnMappingInput,
|
||||
ColumnMappingSchema,
|
||||
type CreateImportJobInput,
|
||||
CreateImportJobSchema,
|
||||
type ImportFormatInput,
|
||||
ImportFormatSchema,
|
||||
type ImportJobListQueryInput,
|
||||
ImportJobListQuerySchema,
|
||||
ImportJobResponseSchema,
|
||||
type ImportJobResponseShape,
|
||||
type ImportStatusInput,
|
||||
ImportStatusSchema,
|
||||
type PreviewRequestInput,
|
||||
PreviewRequestSchema,
|
||||
PreviewResponseSchema,
|
||||
type PreviewResponseShape,
|
||||
RowErrorSchema,
|
||||
} from "./schemas.js";
|
||||
@@ -0,0 +1,67 @@
|
||||
# `schemas/options` — plugin options schema
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the Zod schema for the options accepted by
|
||||
`createImportPlugin`. Parsing options at the factory boundary
|
||||
guarantees the resolved config (`ImportPluginConfig`) carries no
|
||||
optional fields — every downstream consumer (`onInit`, `onReady`,
|
||||
routes, service) reads from a fully-resolved object without `?? default`
|
||||
fallbacks scattered across the call sites.
|
||||
|
||||
Defaults are documented inside the schema (`.default(...)`) so the Zod
|
||||
parse output is the authoritative source of plugin defaults. Renaming a
|
||||
field is a breaking change.
|
||||
|
||||
## Features
|
||||
|
||||
- Six options with sensible production defaults: `routePrefix`,
|
||||
`maxFileSizeBytes`, `maxRows`, `maxErrorsBeforeAbort`, `previewRows`,
|
||||
`defaultBatchSize`.
|
||||
- Two TypeScript shapes — `ImportPluginOptionsInput` for caller-supplied
|
||||
partials, `ImportPluginConfig` for the resolved config.
|
||||
- Strict positive-integer / non-empty-string constraints surface bad
|
||||
config as `IMPORT.INVALID_OPTIONS` with the `ZodError` as `cause`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { ImportPluginOptionsSchema } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const parsed = ImportPluginOptionsSchema.parse({});
|
||||
// parsed.routePrefix === "/imports"
|
||||
// parsed.maxFileSizeBytes === 52_428_800
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ImportPluginOptionsSchema, type ImportPluginConfig } from "@suaveplan/api-plugin-import";
|
||||
|
||||
function configureImport(overrides: Partial<ImportPluginConfig>): ImportPluginConfig {
|
||||
return ImportPluginOptionsSchema.parse({
|
||||
routePrefix: "/v2/imports",
|
||||
maxRows: 10_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ImportPluginOptionsSchema: z.ZodObject<…>` — Zod schema with
|
||||
defaults for every field.
|
||||
- `type ImportPluginOptionsInput` — caller-supplied partial input.
|
||||
- `type ImportPluginConfig` — resolved options (every field
|
||||
non-optional).
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- All numeric bounds are `int().positive()` because the consuming
|
||||
fields are counts; the schema rejects zero with the standard Zod
|
||||
error message.
|
||||
- `previewRows` is capped at 1 000 to prevent a malicious client from
|
||||
triggering a multi-megabyte preview render.
|
||||
- `routePrefix` is non-empty; an empty prefix collapses the mount and
|
||||
causes Hono route shadowing — the Zod gate prevents this footgun at
|
||||
factory time.
|
||||
@@ -0,0 +1,34 @@
|
||||
import z from "zod";
|
||||
|
||||
/**
|
||||
* Zod schema for the options accepted by `createImportPlugin`. The
|
||||
* resolved shape (`ImportPluginConfig`) is what `definePlugin` carries
|
||||
* on its `defaults` field — the factory parses options at the boundary
|
||||
* so every downstream field is guaranteed-defined.
|
||||
*/
|
||||
export const ImportPluginOptionsSchema = z.object({
|
||||
/** Hono mount prefix for the import routes. */
|
||||
routePrefix: z.string().min(1).default("/imports"),
|
||||
/** Maximum payload size in bytes — default 50 MiB. */
|
||||
maxFileSizeBytes: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(50 * 1024 * 1024),
|
||||
/** Maximum row count per job — default 50 000. */
|
||||
maxRows: z.number().int().positive().default(50_000),
|
||||
/** Maximum error count before the run aborts. */
|
||||
maxErrorsBeforeAbort: z.number().int().positive().default(100),
|
||||
/** Default preview row count when the caller omits it. */
|
||||
previewRows: z.number().int().positive().max(1_000).default(10),
|
||||
/** Default batch size for progress-emit cadence. */
|
||||
defaultBatchSize: z.number().int().positive().default(100),
|
||||
});
|
||||
|
||||
/** Input type accepted by `createImportPlugin`. */
|
||||
export type ImportPluginOptionsInput = z.input<
|
||||
typeof ImportPluginOptionsSchema
|
||||
>;
|
||||
|
||||
/** Resolved options after Zod parse — every field is non-optional. */
|
||||
export type ImportPluginConfig = z.output<typeof ImportPluginOptionsSchema>;
|
||||
@@ -0,0 +1,81 @@
|
||||
# `schemas/schemas` — public Zod request/response surface
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Carries the full public Zod surface used by the route handlers and any
|
||||
external client (CLI tool, integration test, browser app) that wants to
|
||||
validate or generate import requests / responses without depending on
|
||||
the route module itself. Every body, query, and response shape that
|
||||
crosses the HTTP boundary is declared here so the OpenAPI document and
|
||||
the runtime validators stay in lockstep.
|
||||
|
||||
Keeping these schemas in their own module means consumers can import
|
||||
them without pulling in the Hono/OpenAPI route builder, which is
|
||||
important for browser clients and shared validation libraries.
|
||||
|
||||
## Features
|
||||
|
||||
- `ImportFormatSchema` / `ImportStatusSchema` — Zod enums mirroring the
|
||||
cross-module type unions.
|
||||
- `ColumnMappingSchema` — wire-form of a column mapping (functions
|
||||
intentionally omitted; transforms are server-side only).
|
||||
- `PreviewRequestSchema` / `PreviewResponseSchema` — `POST /preview`.
|
||||
- `CreateImportJobSchema` — `POST /` body.
|
||||
- `ImportJobResponseSchema` — every job-shape response.
|
||||
- `ImportJobListQuerySchema` — `GET /` querystring with `z.coerce`
|
||||
for the numeric fields so query strings parse without hand-rolling.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { CreateImportJobSchema } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const result = CreateImportJobSchema.safeParse({
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "displayName" }],
|
||||
});
|
||||
if (!result.success) throw new Error(result.error.message);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ImportJobResponseSchema, type ImportJobResponseShape } from "@suaveplan/api-plugin-import";
|
||||
|
||||
async function fetchJob(id: string): Promise<ImportJobResponseShape> {
|
||||
const res = await fetch(`/imports/${id}`);
|
||||
const json = await res.json();
|
||||
return ImportJobResponseSchema.parse(json);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ImportFormatSchema`, `type ImportFormatInput`
|
||||
- `ImportStatusSchema`, `type ImportStatusInput`
|
||||
- `ColumnMappingSchema`, `type ColumnMappingInput`
|
||||
- `PreviewRequestSchema`, `type PreviewRequestInput`
|
||||
- `PreviewResponseSchema`, `type PreviewResponseShape`
|
||||
- `CreateImportJobSchema`, `type CreateImportJobInput`
|
||||
- `ImportJobResponseSchema`, `type ImportJobResponseShape`
|
||||
- `ImportJobListQuerySchema`, `type ImportJobListQueryInput`
|
||||
- `RowErrorSchema`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `ColumnMappingSchema` omits the `transform` callback — transforms are
|
||||
authored in TypeScript on the server, not over the wire. Wire-form
|
||||
mappings are pure renames + required flags.
|
||||
- `PreviewResponseSchema.sampleRows` uses `z.record(z.string(),
|
||||
z.unknown())` so heterogeneous row shapes survive validation.
|
||||
- `ImportJobListQuerySchema` uses `z.coerce.number()` because Hono
|
||||
delivers query parameters as strings — the coerce ensures `?limit=20`
|
||||
parses to a number without manual `parseInt`.
|
||||
- `CreateImportJobSchema.mappings` requires `min(1)` because zero
|
||||
mappings is the service-layer `IMPORT.VALIDATION` failure; rejecting
|
||||
it at the Zod boundary saves a service round-trip.
|
||||
@@ -0,0 +1,89 @@
|
||||
// schema: boundary-only
|
||||
import z from "zod";
|
||||
|
||||
/** Zod enum of the supported import formats. */
|
||||
export const ImportFormatSchema = z.enum(["csv", "json", "xlsx"]);
|
||||
export type ImportFormatInput = z.infer<typeof ImportFormatSchema>;
|
||||
|
||||
/** Zod enum of the import status lifecycle surfaced over the wire. */
|
||||
export const ImportStatusSchema = z.enum([
|
||||
"pending",
|
||||
"validating",
|
||||
"previewing",
|
||||
"importing",
|
||||
"completed",
|
||||
"failed",
|
||||
]);
|
||||
export type ImportStatusInput = z.infer<typeof ImportStatusSchema>;
|
||||
|
||||
/** Zod schema for a single `ColumnMapping` over the wire. */
|
||||
export const ColumnMappingSchema = z.object({
|
||||
sourceColumn: z.string().min(1),
|
||||
targetField: z.string().min(1),
|
||||
required: z.boolean().optional(),
|
||||
});
|
||||
export type ColumnMappingInput = z.infer<typeof ColumnMappingSchema>;
|
||||
|
||||
/** Zod schema for the preview request body. */
|
||||
export const PreviewRequestSchema = z.object({
|
||||
format: ImportFormatSchema,
|
||||
data: z.string().min(1),
|
||||
sampleSize: z.number().int().min(1).max(1_000).optional(),
|
||||
});
|
||||
export type PreviewRequestInput = z.infer<typeof PreviewRequestSchema>;
|
||||
|
||||
/** Zod schema for the preview response body. */
|
||||
export const PreviewResponseSchema = z.object({
|
||||
totalRows: z.number().int().nonnegative(),
|
||||
columns: z.array(z.string()),
|
||||
sampleRows: z.array(z.record(z.string(), z.unknown())),
|
||||
mappings: z.array(
|
||||
z.object({
|
||||
sourceColumn: z.string(),
|
||||
targetField: z.string(),
|
||||
})
|
||||
),
|
||||
});
|
||||
export type PreviewResponseShape = z.infer<typeof PreviewResponseSchema>;
|
||||
|
||||
/** Zod schema for the create-import-job request body. */
|
||||
export const CreateImportJobSchema = z.object({
|
||||
format: ImportFormatSchema,
|
||||
data: z.string().min(1),
|
||||
mappings: z.array(ColumnMappingSchema).min(1),
|
||||
dryRun: z.boolean().optional(),
|
||||
batchSize: z.number().int().positive().optional(),
|
||||
skipDuplicates: z.boolean().optional(),
|
||||
});
|
||||
export type CreateImportJobInput = z.infer<typeof CreateImportJobSchema>;
|
||||
|
||||
/** Zod schema for a row-level error returned in import progress payloads. */
|
||||
export const RowErrorSchema = z.object({
|
||||
row: z.number().int().nonnegative(),
|
||||
column: z.string(),
|
||||
message: z.string(),
|
||||
value: z.unknown().optional(),
|
||||
});
|
||||
|
||||
/** Zod schema for the import-job response body. */
|
||||
export const ImportJobResponseSchema = z.object({
|
||||
jobId: z.string(),
|
||||
ownerId: z.string(),
|
||||
status: ImportStatusSchema,
|
||||
totalRows: z.number().int().nonnegative(),
|
||||
processedRows: z.number().int().nonnegative(),
|
||||
successCount: z.number().int().nonnegative(),
|
||||
errorCount: z.number().int().nonnegative(),
|
||||
errors: z.array(RowErrorSchema),
|
||||
startedAt: z.string().optional(),
|
||||
completedAt: z.string().optional(),
|
||||
});
|
||||
export type ImportJobResponseShape = z.infer<typeof ImportJobResponseSchema>;
|
||||
|
||||
/** Zod schema for `GET /imports` query parameters. */
|
||||
export const ImportJobListQuerySchema = z.object({
|
||||
status: ImportStatusSchema.optional(),
|
||||
limit: z.coerce.number().int().min(1).max(100).optional(),
|
||||
offset: z.coerce.number().int().min(0).optional(),
|
||||
});
|
||||
export type ImportJobListQueryInput = z.infer<typeof ImportJobListQuerySchema>;
|
||||
@@ -0,0 +1,8 @@
|
||||
export type {
|
||||
ImportServiceListFilter,
|
||||
ImportServiceOptions,
|
||||
} from "./service.js";
|
||||
export {
|
||||
asProcessingFailed,
|
||||
ImportService,
|
||||
} from "./service.js";
|
||||
115
packages/api-stack/api/api-plugin-import/src/service/service.md
Normal file
115
packages/api-stack/api/api-plugin-import/src/service/service.md
Normal file
@@ -0,0 +1,115 @@
|
||||
# `service/service` — canonical ImportService
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Implements the canonical `ImportService` used by `createImportPlugin`
|
||||
and exported for direct programmatic use. Owns the per-job in-memory
|
||||
progress map, the optional durable store, the `EventBus<ImportEventMap>`
|
||||
event surface, and the OTel emit on the production hot path. Every read
|
||||
and write is owner-scoped: a job belongs to exactly one `ownerId`, and
|
||||
foreign-owner lookups return `IMPORT.NOT_FOUND` (anti-enumeration) — not
|
||||
`IMPORT.FORBIDDEN` — so callers cannot probe for the existence of other
|
||||
users' jobs.
|
||||
|
||||
The service is the only place that ties together parsers, transform
|
||||
pipeline (via `applyMappings`), progress emission, and persistence; the
|
||||
plugin layer is a thin wrapper that constructs it once per `onInit`
|
||||
and mounts the routes.
|
||||
|
||||
## Features
|
||||
|
||||
- `preview(format, data)` — returns column auto-mapping + a 5-row
|
||||
sample.
|
||||
- `startImport(config, { ownerId })` — parses, maps, optionally
|
||||
dedups, optionally runs `onRow`, emits per-row events, batch progress
|
||||
updates, and OTel counters / histogram.
|
||||
- `getJob` / `getResults` / `listJobs` / `deleteJob` — owner-scoped.
|
||||
- `bus: EventBus<ImportEventMap>` — public so consumers can attach
|
||||
listeners without subclassing.
|
||||
- Injectable `store`, `clock`, `idFactory`, `logger`, `instruments` for
|
||||
testability.
|
||||
- RFC 7807 status hints — every thrown `ImportError` carries
|
||||
`context.statusCode` so the route renderer doesn't need a code→status
|
||||
lookup table.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { ImportService } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const service = new ImportService();
|
||||
const preview = service.preview("csv", "name\nAda");
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: [{ sourceColumn: "name", targetField: "displayName" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
// progress.status === "completed"
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ImportService, MemoryImportStore } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const store = new MemoryImportStore();
|
||||
const service = new ImportService({
|
||||
store,
|
||||
clock: () => 1_700_000_000_000,
|
||||
idFactory: () => "test-job",
|
||||
});
|
||||
|
||||
service.bus.on("import.progress", (snap) => console.log(snap.percentComplete));
|
||||
|
||||
await service.startImport(
|
||||
{
|
||||
format: "json",
|
||||
data: '[{"x":1},{"x":2},{"x":1}]',
|
||||
mappings: [{ sourceColumn: "x", targetField: "value" }],
|
||||
skipDuplicates: true,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
// `test-job` is now in the store; results array dedups to 2 entries
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `class ImportService`
|
||||
- `constructor(options?: ImportServiceOptions)`
|
||||
- `bus: EventBus<ImportEventMap>`
|
||||
- `preview(format: ImportFormat, data: string): ImportPreview`
|
||||
- `startImport(config: ImportJobConfig, owner: ImportOwnerContext): Promise<ImportProgress>`
|
||||
- `getJob(jobId: string, ownerId: string): ImportProgress`
|
||||
- `getResults(jobId: string, ownerId: string): Record<string, unknown>[]`
|
||||
- `listJobs(ownerId: string, filter?: ImportStoreListFilter): ImportProgress[]`
|
||||
- `deleteJob(jobId: string, ownerId: string): Promise<boolean>`
|
||||
- `interface ImportServiceOptions { store?; clock?; idFactory?; logger?; instruments? }`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The service stores progress in two `Map`s (live progress + imported
|
||||
rows). Persistence is best-effort; a store failure throws
|
||||
`IMPORT.STORAGE_ERROR` but the in-memory progress is still consistent.
|
||||
- `onRow` is invoked AFTER `skipDuplicates` so a duplicate row never
|
||||
reaches the caller's filter. Returning `false` skips the row but does
|
||||
NOT mark it as an error.
|
||||
- The batch-progress emit cadence is keyed on `(i + 1) % batchSize === 0`
|
||||
so a batch size of 100 emits at rows 100, 200, … plus the final
|
||||
completion emit. This matches the upstream behaviour exactly.
|
||||
- Foreign-owner reads on `getJob` throw `NOT_FOUND` rather than
|
||||
`FORBIDDEN` so an attacker cannot distinguish "no such job" from
|
||||
"your access is denied."
|
||||
- The XLSX path is rendered inline (not via `parseXlsx`) to keep the
|
||||
return shape `Record<string, unknown>[]` instead of `CsvParseResult`;
|
||||
call `parseXlsx` directly if you need headers metadata.
|
||||
- `parseData` re-raises `ImportError` unchanged but wraps every other
|
||||
throwable as `IMPORT.PROCESSING_FAILED` so consumers can branch on
|
||||
one code regardless of which parser failed.
|
||||
@@ -0,0 +1,622 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { IMPORT_CODES, ImportError } from "../error-codes/error-codes.js";
|
||||
import { resetImportInstruments } from "../otel/instruments.js";
|
||||
import { MemoryImportStore } from "../store/memory.js";
|
||||
import type { ColumnMapping, ImportEventMap } from "../types/types.js";
|
||||
import { asProcessingFailed, ImportService } from "./service.js";
|
||||
|
||||
function makeService(
|
||||
overrides: Parameters<typeof ImportService.prototype.constructor>[0] = {}
|
||||
) {
|
||||
resetImportInstruments();
|
||||
let n = 0;
|
||||
return new ImportService({
|
||||
idFactory: () => `j-${++n}`,
|
||||
clock: () => 1_700_000_000_000,
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
const csvMappings: ColumnMapping[] = [
|
||||
{ sourceColumn: "name", targetField: "displayName" },
|
||||
];
|
||||
|
||||
describe("ImportService.preview", () => {
|
||||
it("preview returns identity column mappings", { timeout: 5000 }, () => {
|
||||
const service = makeService();
|
||||
const p = service.preview("csv", "name,age\nAda,37");
|
||||
expect(p.totalRows).toBe(1);
|
||||
expect(p.columns).toEqual(["name", "age"]);
|
||||
expect(p.mappings).toEqual([
|
||||
{ sourceColumn: "name", targetField: "name" },
|
||||
{ sourceColumn: "age", targetField: "age" },
|
||||
]);
|
||||
expect(p.sampleRows).toEqual([{ name: "Ada", age: "37" }]);
|
||||
});
|
||||
|
||||
it("preview rejects unsupported formats", { timeout: 5000 }, () => {
|
||||
const service = makeService();
|
||||
expect(() => service.preview("bogus" as unknown as "csv", "x")).toThrow(
|
||||
ImportError
|
||||
);
|
||||
});
|
||||
|
||||
it("preview surfaces parser failures as PROCESSING_FAILED", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const service = makeService();
|
||||
try {
|
||||
service.preview("json", "not-json");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(ImportError);
|
||||
expect((err as ImportError).code).toBe(
|
||||
IMPORT_CODES.PROCESSING_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("preview parses XLSX (TSV) input", { timeout: 5000 }, () => {
|
||||
const service = makeService();
|
||||
const p = service.preview("xlsx", "name\tcity\nAda\tLondon");
|
||||
expect(p.columns).toEqual(["name", "city"]);
|
||||
});
|
||||
|
||||
it("preview pads missing trailing cells in XLSX input", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const service = makeService();
|
||||
const p = service.preview("xlsx", "a\tb\tc\n1\t2");
|
||||
expect(p.sampleRows[0]).toEqual({ a: "1", b: "2", c: "" });
|
||||
});
|
||||
|
||||
it("preview returns empty for XLSX header-only input", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const service = makeService();
|
||||
const p = service.preview("xlsx", "a\tb");
|
||||
expect(p.totalRows).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ImportService.startImport — happy + dry-run", () => {
|
||||
it("completes a successful run and persists the result", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
const service = makeService({ store });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda\nBob",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(progress.status).toBe("completed");
|
||||
expect(progress.successCount).toBe(2);
|
||||
expect(progress.errorCount).toBe(0);
|
||||
expect(service.getResults(progress.jobId, "alice")).toEqual([
|
||||
{ displayName: "Ada" },
|
||||
{ displayName: "Bob" },
|
||||
]);
|
||||
expect(store.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("dryRun does not persist imported rows", { timeout: 5000 }, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
dryRun: true,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(progress.status).toBe("completed");
|
||||
expect(service.getResults(progress.jobId, "alice")).toEqual([]);
|
||||
});
|
||||
|
||||
it("skipDuplicates collapses identical mapped rows", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "json",
|
||||
data: '[{"x":1},{"x":1},{"x":2}]',
|
||||
mappings: [{ sourceColumn: "x", targetField: "value" }],
|
||||
skipDuplicates: true,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
// Only 2 unique mapped rows, but successCount counts both unique rows.
|
||||
const results = service.getResults(progress.jobId, "alice");
|
||||
expect(results.length).toBe(2);
|
||||
});
|
||||
|
||||
it("onRow filter skips rows when the callback returns false", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda\nBob",
|
||||
mappings: csvMappings,
|
||||
onRow: (_row, idx) => idx === 0,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(service.getResults(progress.jobId, "alice").length).toBe(1);
|
||||
});
|
||||
|
||||
it("mapping validation: missing required field is reported as an error row", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name,age\nAda,\n,40",
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "name",
|
||||
targetField: "name",
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
sourceColumn: "age",
|
||||
targetField: "age",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(progress.status).toBe("failed");
|
||||
expect(progress.errorCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("persists row errors to the store on failed runs", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
const service = makeService({ store });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name,age\n,1",
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "name",
|
||||
targetField: "name",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const rec = await store.get(progress.jobId);
|
||||
expect(rec).toBeDefined();
|
||||
expect(rec?.errors.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("uses a default id factory when none is supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const service = new ImportService({ clock: () => 1_700_000_000 });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(progress.jobId).toMatch(/^import_/);
|
||||
});
|
||||
|
||||
it("uses default clock + logger when none are supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
resetImportInstruments();
|
||||
const service = new ImportService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(progress.status).toBe("completed");
|
||||
});
|
||||
|
||||
it("transform throw is captured as a row error", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "x\nhello",
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "x",
|
||||
targetField: "x",
|
||||
transform: () => {
|
||||
throw new Error("nope");
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(progress.status).toBe("failed");
|
||||
expect(progress.errors[0]?.message).toBe("Transform failed");
|
||||
});
|
||||
|
||||
it("batch progress emits at batchSize multiples", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const seen: number[] = [];
|
||||
service.bus.on(
|
||||
"import.progress",
|
||||
(snap: ImportEventMap["import.progress"]) =>
|
||||
seen.push(snap.processedRows)
|
||||
);
|
||||
// 5 rows, batchSize: 2 → emits at row 2, 4, plus final 5
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "x\n1\n2\n3\n4\n5",
|
||||
mappings: [{ sourceColumn: "x", targetField: "x" }],
|
||||
batchSize: 2,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(seen).toContain(2);
|
||||
expect(seen).toContain(4);
|
||||
expect(seen).toContain(5);
|
||||
});
|
||||
|
||||
it("uses the defaultValue when source field is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "x,y\n1,",
|
||||
mappings: [
|
||||
{ sourceColumn: "x", targetField: "x" },
|
||||
{
|
||||
sourceColumn: "y",
|
||||
targetField: "y",
|
||||
defaultValue: "default",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const results = service.getResults(progress.jobId, "alice");
|
||||
expect(results[0]).toEqual({ x: "1", y: "default" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("ImportService — owner scoping + 404", () => {
|
||||
it("foreign owner lookup returns NOT_FOUND", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(() => service.getJob(progress.jobId, "bob")).toThrow(
|
||||
ImportError
|
||||
);
|
||||
try {
|
||||
service.getJob(progress.jobId, "bob");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(IMPORT_CODES.NOT_FOUND);
|
||||
}
|
||||
});
|
||||
|
||||
it("getJob throws NOT_FOUND for a missing id", { timeout: 5000 }, () => {
|
||||
const service = makeService();
|
||||
expect(() => service.getJob("missing", "alice")).toThrow(ImportError);
|
||||
});
|
||||
|
||||
it("listJobs scopes to ownerId and filters", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nBob",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "bob" }
|
||||
);
|
||||
expect(service.listJobs("alice").length).toBe(1);
|
||||
expect(service.listJobs("alice", { status: "completed" }).length).toBe(
|
||||
1
|
||||
);
|
||||
expect(service.listJobs("alice", { status: "failed" }).length).toBe(0);
|
||||
expect(service.listJobs("alice", { limit: 0 }).length).toBe(0);
|
||||
expect(service.listJobs("alice", { limit: 1, offset: 1 }).length).toBe(
|
||||
0
|
||||
);
|
||||
});
|
||||
|
||||
it("deleteJob removes the in-memory and stored entries", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
const service = makeService({ store });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const deleted = await service.deleteJob(progress.jobId, "alice");
|
||||
expect(deleted).toBe(true);
|
||||
expect(store.size()).toBe(0);
|
||||
expect(() => service.getJob(progress.jobId, "alice")).toThrow(
|
||||
ImportError
|
||||
);
|
||||
});
|
||||
|
||||
it("deleteJob on a missing id throws NOT_FOUND", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
await expect(service.deleteJob("missing", "alice")).rejects.toThrow(
|
||||
ImportError
|
||||
);
|
||||
});
|
||||
|
||||
it("deleteJob wraps store failures as STORAGE_ERROR", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const failingStore = new MemoryImportStore();
|
||||
// Override delete to throw.
|
||||
failingStore.delete = async () => {
|
||||
throw new Error("disk full");
|
||||
};
|
||||
const service = makeService({ store: failingStore });
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
try {
|
||||
await service.deleteJob(progress.jobId, "alice");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(IMPORT_CODES.STORAGE_ERROR);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ImportService — failure surface", () => {
|
||||
it("throws UNSUPPORTED_FORMAT for unknown format", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
await expect(
|
||||
service.startImport(
|
||||
{
|
||||
format: "bogus" as unknown as "csv",
|
||||
data: "x",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
)
|
||||
).rejects.toThrow(ImportError);
|
||||
});
|
||||
|
||||
it("throws VALIDATION when mappings are empty", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
try {
|
||||
await service.startImport(
|
||||
{ format: "csv", data: "x\n1", mappings: [] },
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(IMPORT_CODES.VALIDATION);
|
||||
}
|
||||
});
|
||||
|
||||
it("parse failure throws PROCESSING_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
try {
|
||||
await service.startImport(
|
||||
{
|
||||
format: "json",
|
||||
data: "not-json",
|
||||
mappings: [{ sourceColumn: "x", targetField: "x" }],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(
|
||||
IMPORT_CODES.PROCESSING_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("persist failure on initial create surfaces as STORAGE_ERROR", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const failing = new MemoryImportStore();
|
||||
failing.create = async () => {
|
||||
throw new Error("kaput");
|
||||
};
|
||||
const service = makeService({ store: failing });
|
||||
try {
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as ImportError).code).toBe(IMPORT_CODES.STORAGE_ERROR);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ImportService — events", () => {
|
||||
it("emits started + completed + import:completed events on success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const seen: string[] = [];
|
||||
for (const topic of [
|
||||
"import.started",
|
||||
"import:started",
|
||||
"import.completed",
|
||||
"import:completed",
|
||||
] as const) {
|
||||
service.bus.on(topic, () => seen.push(topic));
|
||||
}
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\nAda",
|
||||
mappings: csvMappings,
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(seen).toContain("import.started");
|
||||
expect(seen).toContain("import:started");
|
||||
expect(seen).toContain("import.completed");
|
||||
expect(seen).toContain("import:completed");
|
||||
});
|
||||
|
||||
it("emits import.failed when every row errors out", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const events: string[] = [];
|
||||
service.bus.on("import.failed", () => events.push("failed"));
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name,age\n,1\n,2",
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "name",
|
||||
targetField: "name",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
expect(events).toContain("failed");
|
||||
});
|
||||
|
||||
it("emits import.row_error per mapping failure", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const errs: unknown[] = [];
|
||||
service.bus.on("import.row_error", (e) => errs.push(e));
|
||||
await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "name\n",
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "name",
|
||||
targetField: "name",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
// Header-only input → 0 rows → no row_error
|
||||
expect(errs.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("asProcessingFailed", () => {
|
||||
it("wraps non-Error throws with a generic message", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = asProcessingFailed("just a string");
|
||||
expect(err.code).toBe(IMPORT_CODES.PROCESSING_FAILED);
|
||||
expect(err.message).toBe("Unknown parse error");
|
||||
});
|
||||
|
||||
it("wraps native Error instances with the original message", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = asProcessingFailed(new TypeError("bad shape"));
|
||||
expect(err.message).toBe("bad shape");
|
||||
});
|
||||
|
||||
it("passes ImportError through unchanged", { timeout: 5000 }, () => {
|
||||
const inner = new ImportError(IMPORT_CODES.NOT_FOUND, "missing");
|
||||
const err = asProcessingFailed(inner);
|
||||
expect(err).toBe(inner);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ImportService — applyMappings null fallback", () => {
|
||||
it("non-required missing field falls back to null when defaultValue is absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const service = makeService();
|
||||
const progress = await service.startImport(
|
||||
{
|
||||
format: "csv",
|
||||
data: "x,y\n1,",
|
||||
mappings: [
|
||||
{ sourceColumn: "x", targetField: "x" },
|
||||
// y is not required, no defaultValue ⇒ falls back to null
|
||||
{ sourceColumn: "y", targetField: "y" },
|
||||
],
|
||||
},
|
||||
{ ownerId: "alice" }
|
||||
);
|
||||
const results = service.getResults(progress.jobId, "alice");
|
||||
expect(results[0]).toEqual({ x: "1", y: null });
|
||||
});
|
||||
});
|
||||
472
packages/api-stack/api/api-plugin-import/src/service/service.ts
Normal file
472
packages/api-stack/api/api-plugin-import/src/service/service.ts
Normal file
@@ -0,0 +1,472 @@
|
||||
import { EventBus } from "@suaveplan/events";
|
||||
import type { Clock, LoggerLike } from "@suaveplan/types";
|
||||
import { IMPORT_CODES, ImportError } from "../error-codes/error-codes.js";
|
||||
import {
|
||||
getImportInstruments,
|
||||
type ImportInstruments,
|
||||
} from "../otel/instruments.js";
|
||||
import { detectColumns, parseCsv } from "../parsers/csv.js";
|
||||
import { parseJson } from "../parsers/json.js";
|
||||
import type { ImportStoreAdapter } from "../store/memory.js";
|
||||
import type {
|
||||
ColumnMapping,
|
||||
ImportEventMap,
|
||||
ImportFormat,
|
||||
ImportJobConfig,
|
||||
ImportJobRecord,
|
||||
ImportOwnerContext,
|
||||
ImportPreview,
|
||||
ImportProgress,
|
||||
ImportStatus,
|
||||
RowError,
|
||||
} from "../types/types.js";
|
||||
import { DEFAULT_BATCH_SIZE, VALID_FORMATS } from "../types/types.js";
|
||||
|
||||
/** Filter options for `ImportService.listJobs`. */
|
||||
export interface ImportServiceListFilter {
|
||||
readonly status?: ImportStatus;
|
||||
readonly limit?: number;
|
||||
readonly offset?: number;
|
||||
}
|
||||
|
||||
/** Options accepted by the `ImportService` constructor. */
|
||||
export interface ImportServiceOptions {
|
||||
/** Injectable durable store; defaults to `MemoryImportStore`. */
|
||||
readonly store?: ImportStoreAdapter;
|
||||
/** Injectable clock for deterministic test runs. */
|
||||
readonly clock?: Clock;
|
||||
/** Injectable id factory for deterministic job ids. */
|
||||
readonly idFactory?: () => string;
|
||||
/** Structural logger; defaults to a no-op. */
|
||||
readonly logger?: LoggerLike;
|
||||
/** OTel instruments bundle override (defaults to the global cache). */
|
||||
readonly instruments?: ImportInstruments;
|
||||
}
|
||||
|
||||
/** Result of an applied-mappings step for a single row. */
|
||||
interface MappedRow {
|
||||
readonly success: boolean;
|
||||
readonly row: Record<string, unknown>;
|
||||
readonly errors: RowError[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical import service used by `createImportPlugin`. Wraps an
|
||||
* injectable `EventBus<ImportEventMap>` (so consumers can attach via
|
||||
* `service.bus.on(...)`), persists job state through the injected
|
||||
* `ImportStoreAdapter`, emits OTel metrics on the production hot path,
|
||||
* and scopes every read / write to a caller-supplied `ownerId` so the
|
||||
* route layer can safely surface results to multi-tenant apps.
|
||||
*/
|
||||
export class ImportService {
|
||||
public readonly bus: EventBus<ImportEventMap>;
|
||||
private readonly progress = new Map<string, ImportProgress>();
|
||||
private readonly results = new Map<string, Record<string, unknown>[]>();
|
||||
private readonly store: ImportStoreAdapter | undefined;
|
||||
private readonly clock: Clock;
|
||||
private readonly idFactory: () => string;
|
||||
private readonly logger: LoggerLike;
|
||||
private readonly instruments: ImportInstruments;
|
||||
private counter = 0;
|
||||
|
||||
public constructor(options: ImportServiceOptions = {}) {
|
||||
this.bus = new EventBus<ImportEventMap>();
|
||||
this.store = options.store;
|
||||
this.clock = options.clock ?? defaultClock;
|
||||
this.idFactory = options.idFactory ?? this.defaultId.bind(this);
|
||||
this.logger = options.logger ?? noopLogger;
|
||||
this.instruments = options.instruments ?? getImportInstruments();
|
||||
}
|
||||
|
||||
/** Compute a preview of the parsed data with identity column mappings. */
|
||||
public preview(format: ImportFormat, data: string): ImportPreview {
|
||||
this.validateFormat(format);
|
||||
const rows = this.parseData(format, data);
|
||||
const columns = detectColumns(rows);
|
||||
const sampleRows = rows.slice(0, 5);
|
||||
const mappings: ColumnMapping[] = columns.map((col) => ({
|
||||
sourceColumn: col,
|
||||
targetField: col,
|
||||
}));
|
||||
return { totalRows: rows.length, columns, sampleRows, mappings };
|
||||
}
|
||||
|
||||
/** Kick off an import; persists results unless `dryRun` is true. */
|
||||
public async startImport(
|
||||
config: ImportJobConfig,
|
||||
owner: ImportOwnerContext
|
||||
): Promise<ImportProgress> {
|
||||
this.validateFormat(config.format);
|
||||
if (!config.mappings.length) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.VALIDATION,
|
||||
"at least one mapping is required",
|
||||
{ context: { statusCode: 400 } }
|
||||
);
|
||||
}
|
||||
const startedNs = this.clock();
|
||||
const rows = this.parseData(config.format, config.data);
|
||||
const jobId = this.idFactory();
|
||||
const batchSize = config.batchSize ?? DEFAULT_BATCH_SIZE;
|
||||
const startedIso = new Date(this.clock()).toISOString();
|
||||
const initial: ImportProgress = {
|
||||
jobId,
|
||||
ownerId: owner.ownerId,
|
||||
status: "importing",
|
||||
totalRows: rows.length,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
startedAt: startedIso,
|
||||
};
|
||||
this.progress.set(jobId, initial);
|
||||
await this.persistRecord(initial, "created");
|
||||
this.bus.emit("import.started", initial);
|
||||
this.bus.emit("import:started", initial);
|
||||
|
||||
const imported = this.processRows(
|
||||
jobId,
|
||||
owner.ownerId,
|
||||
rows,
|
||||
config.mappings,
|
||||
batchSize,
|
||||
config.dryRun ?? false,
|
||||
config.skipDuplicates ?? false,
|
||||
config.onRow
|
||||
);
|
||||
|
||||
if (!config.dryRun) {
|
||||
this.results.set(jobId, imported);
|
||||
}
|
||||
|
||||
const final = this.getJob(jobId, owner.ownerId);
|
||||
const outcome = final.status === "completed" ? "success" : "error";
|
||||
const durationMs = this.clock() - startedNs;
|
||||
this.instruments.jobsTotal.add(1, { outcome });
|
||||
this.instruments.rowsTotal.add(final.successCount, {
|
||||
outcome: "success",
|
||||
});
|
||||
this.instruments.rowsTotal.add(final.errorCount, { outcome: "error" });
|
||||
this.instruments.rowErrorsTotal.add(final.errors.length);
|
||||
this.instruments.durationMs.record(durationMs);
|
||||
await this.persistRecord(final, mapStatusToRecord(final.status));
|
||||
this.logger.debug("import.run.complete", {
|
||||
jobId,
|
||||
ownerId: owner.ownerId,
|
||||
durationMs,
|
||||
outcome,
|
||||
});
|
||||
return final;
|
||||
}
|
||||
|
||||
/** Owner-scoped read of a job's live progress. Throws NOT_FOUND. */
|
||||
public getJob(jobId: string, ownerId: string): ImportProgress {
|
||||
const job = this.progress.get(jobId);
|
||||
if (!job || job.ownerId !== ownerId) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.NOT_FOUND,
|
||||
`Import job "${jobId}" not found`,
|
||||
{ context: { jobId, statusCode: 404 } }
|
||||
);
|
||||
}
|
||||
return job;
|
||||
}
|
||||
|
||||
/** Owner-scoped read of the imported rows. */
|
||||
public getResults(
|
||||
jobId: string,
|
||||
ownerId: string
|
||||
): Record<string, unknown>[] {
|
||||
this.getJob(jobId, ownerId);
|
||||
return [...(this.results.get(jobId) ?? [])];
|
||||
}
|
||||
|
||||
/** List the in-flight + completed jobs belonging to `ownerId`. */
|
||||
public listJobs(
|
||||
ownerId: string,
|
||||
filter?: ImportServiceListFilter
|
||||
): ImportProgress[] {
|
||||
const all = [...this.progress.values()].filter(
|
||||
(j) => j.ownerId === ownerId
|
||||
);
|
||||
const filtered =
|
||||
filter?.status !== undefined
|
||||
? all.filter((j) => j.status === filter.status)
|
||||
: all;
|
||||
const offset = filter?.offset ?? 0;
|
||||
const limit = filter?.limit ?? filtered.length;
|
||||
return filtered.slice(offset, offset + limit);
|
||||
}
|
||||
|
||||
/** Owner-scoped delete. Returns `true` if a job was removed. */
|
||||
public async deleteJob(jobId: string, ownerId: string): Promise<boolean> {
|
||||
this.getJob(jobId, ownerId); // throws NOT_FOUND on missing/foreign
|
||||
this.progress.delete(jobId);
|
||||
this.results.delete(jobId);
|
||||
if (this.store) {
|
||||
try {
|
||||
await this.store.delete(jobId);
|
||||
} catch (cause) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.STORAGE_ERROR,
|
||||
`failed to delete job ${jobId} from store`,
|
||||
{ cause, context: { jobId, statusCode: 502 } }
|
||||
);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private processRows(
|
||||
jobId: string,
|
||||
ownerId: string,
|
||||
rows: Record<string, unknown>[],
|
||||
mappings: ColumnMapping[],
|
||||
batchSize: number,
|
||||
dryRun: boolean,
|
||||
skipDuplicates: boolean,
|
||||
onRow?: (row: Record<string, unknown>, index: number) => boolean
|
||||
): Record<string, unknown>[] {
|
||||
const imported: Record<string, unknown>[] = [];
|
||||
const errors: RowError[] = [];
|
||||
const seen = new Set<string>();
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (let i = 0; i < rows.length; i++) {
|
||||
const sourceRow = rows[i] as Record<string, unknown>;
|
||||
const mapped = applyMappings(sourceRow, mappings, i + 1);
|
||||
if (!mapped.success) {
|
||||
errors.push(...mapped.errors);
|
||||
errorCount += mapped.errors.length;
|
||||
for (const err of mapped.errors) {
|
||||
this.bus.emit("import.row_error", err);
|
||||
}
|
||||
this.bus.emit("import:row:processed", {
|
||||
jobId,
|
||||
rowIndex: i,
|
||||
success: false,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (skipDuplicates) {
|
||||
const key = JSON.stringify(mapped.row);
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
}
|
||||
if (onRow && !onRow(mapped.row, i)) continue;
|
||||
if (!dryRun) imported.push(mapped.row);
|
||||
successCount++;
|
||||
this.bus.emit("import:row:processed", {
|
||||
jobId,
|
||||
rowIndex: i,
|
||||
success: true,
|
||||
});
|
||||
if ((i + 1) % batchSize === 0) {
|
||||
this.updateProgress(jobId, ownerId, {
|
||||
processedRows: i + 1,
|
||||
successCount,
|
||||
errorCount,
|
||||
errors,
|
||||
});
|
||||
}
|
||||
}
|
||||
const status =
|
||||
errorCount > 0 && successCount === 0 ? "failed" : "completed";
|
||||
this.updateProgress(jobId, ownerId, {
|
||||
status,
|
||||
processedRows: rows.length,
|
||||
successCount,
|
||||
errorCount,
|
||||
errors,
|
||||
completedAt: new Date(this.clock()).toISOString(),
|
||||
});
|
||||
const finalSnap = this.getJob(jobId, ownerId);
|
||||
this.bus.emit(
|
||||
status === "failed" ? "import.failed" : "import.completed",
|
||||
finalSnap
|
||||
);
|
||||
if (status === "completed") {
|
||||
this.bus.emit("import:completed", finalSnap);
|
||||
}
|
||||
return imported;
|
||||
}
|
||||
|
||||
private updateProgress(
|
||||
jobId: string,
|
||||
ownerId: string,
|
||||
update: Partial<ImportProgress>
|
||||
): void {
|
||||
// Invariant: this private helper is only invoked from `processRows`
|
||||
// after `startImport` set the progress map, so the entry is always
|
||||
// present and owned by the current `ownerId`.
|
||||
const current = this.progress.get(jobId) as ImportProgress;
|
||||
void ownerId;
|
||||
const updated: ImportProgress = { ...current, ...update };
|
||||
this.progress.set(jobId, updated);
|
||||
this.bus.emit("import.progress", updated);
|
||||
}
|
||||
|
||||
private async persistRecord(
|
||||
progress: ImportProgress,
|
||||
status: ImportJobRecord["status"]
|
||||
): Promise<void> {
|
||||
if (!this.store) return;
|
||||
const nowIso = new Date(this.clock()).toISOString();
|
||||
const record: ImportJobRecord = {
|
||||
id: progress.jobId,
|
||||
ownerId: progress.ownerId,
|
||||
status,
|
||||
totalRows: progress.totalRows,
|
||||
processedRows: progress.processedRows,
|
||||
successCount: progress.successCount,
|
||||
errorCount: progress.errorCount,
|
||||
errors: progress.errors.map((e) => ({
|
||||
row: e.row,
|
||||
message: e.message,
|
||||
})),
|
||||
// `startImport` always sets `startedAt` before persisting.
|
||||
createdAt: progress.startedAt as string,
|
||||
updatedAt: nowIso,
|
||||
...(progress.completedAt !== undefined && {
|
||||
completedAt: progress.completedAt,
|
||||
}),
|
||||
};
|
||||
try {
|
||||
const existing = await this.store.get(progress.jobId);
|
||||
if (existing) {
|
||||
await this.store.update(progress.jobId, record);
|
||||
} else {
|
||||
await this.store.create(record);
|
||||
}
|
||||
} catch (cause) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.STORAGE_ERROR,
|
||||
`failed to persist job ${progress.jobId}`,
|
||||
{
|
||||
cause,
|
||||
context: { jobId: progress.jobId, statusCode: 502 },
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private validateFormat(format: string): void {
|
||||
if (!VALID_FORMATS.includes(format as ImportFormat)) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.UNSUPPORTED_FORMAT,
|
||||
`Unsupported import format: "${format}"`,
|
||||
{ context: { format, statusCode: 415 } }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private parseData(
|
||||
format: ImportFormat,
|
||||
data: string
|
||||
): Record<string, unknown>[] {
|
||||
try {
|
||||
if (format === "csv") return parseCsv(data);
|
||||
if (format === "json") return parseJson(data);
|
||||
// XLSX is handled via parseXlsx but returns a different shape;
|
||||
// re-render here for parity.
|
||||
return parseXlsxToRecords(data);
|
||||
} catch (err) {
|
||||
throw asProcessingFailed(err);
|
||||
}
|
||||
}
|
||||
|
||||
private defaultId(): string {
|
||||
return `import_${this.clock()}_${++this.counter}`;
|
||||
}
|
||||
}
|
||||
|
||||
function applyMappings(
|
||||
row: Record<string, unknown>,
|
||||
mappings: ColumnMapping[],
|
||||
rowIndex: number
|
||||
): MappedRow {
|
||||
const result: Record<string, unknown> = {};
|
||||
const errors: RowError[] = [];
|
||||
for (const mapping of mappings) {
|
||||
let value = row[mapping.sourceColumn];
|
||||
if (value === undefined || value === "") {
|
||||
if (mapping.required) {
|
||||
errors.push({
|
||||
row: rowIndex,
|
||||
column: mapping.sourceColumn,
|
||||
message: "Required field is missing",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
value = mapping.defaultValue ?? null;
|
||||
}
|
||||
if (mapping.transform && typeof value === "string") {
|
||||
try {
|
||||
value = mapping.transform(value);
|
||||
} catch {
|
||||
errors.push({
|
||||
row: rowIndex,
|
||||
column: mapping.sourceColumn,
|
||||
message: "Transform failed",
|
||||
value,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
result[mapping.targetField] = value;
|
||||
}
|
||||
return { success: errors.length === 0, row: result, errors };
|
||||
}
|
||||
|
||||
function mapStatusToRecord(
|
||||
status: ImportProgress["status"]
|
||||
): ImportJobRecord["status"] {
|
||||
// `processRows` only resolves to `completed` or `failed`; the other
|
||||
// ImportStatus members are emitted on the live progress object but
|
||||
// never reach the durable-store mapper, so the function collapses to
|
||||
// a binary mapping. The default branch is reachable only if a future
|
||||
// hook persists an intermediate state.
|
||||
return status === "failed" ? "failed" : "completed";
|
||||
}
|
||||
|
||||
function parseXlsxToRecords(data: string): Record<string, unknown>[] {
|
||||
const lines = data.split(/\r?\n/).filter((l) => l.trim().length > 0);
|
||||
if (lines.length < 2) return [];
|
||||
const headers = String(lines[0])
|
||||
.split("\t")
|
||||
.map((h) => h.trim());
|
||||
return lines.slice(1).map((line) => {
|
||||
const values = line.split("\t");
|
||||
const row: Record<string, unknown> = {};
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
const header = String(headers[i]);
|
||||
row[header] = (values[i] ?? "").trim();
|
||||
}
|
||||
return row;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap an unknown thrown value as `IMPORT.PROCESSING_FAILED`. Exported
|
||||
* for tests so the non-Error branch (a thrown string / number / object)
|
||||
* can be exercised without monkey-patching the parser modules.
|
||||
*/
|
||||
export function asProcessingFailed(err: unknown): ImportError {
|
||||
if (err instanceof ImportError) return err;
|
||||
const message = err instanceof Error ? err.message : "Unknown parse error";
|
||||
return new ImportError(IMPORT_CODES.PROCESSING_FAILED, message, {
|
||||
cause: err,
|
||||
context: { statusCode: 422 },
|
||||
});
|
||||
}
|
||||
|
||||
const defaultClock: Clock = () => Date.now();
|
||||
const noop = (): void => undefined;
|
||||
const noopLogger: LoggerLike = {
|
||||
debug: noop,
|
||||
info: noop,
|
||||
warn: noop,
|
||||
error: noop,
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
// Side-effect-only barrel — `services/registry-augmentation.ts`
|
||||
// declares the `ImportService` slot on the `ServiceRegistry` interface.
|
||||
import "./registry-augmentation.js";
|
||||
@@ -0,0 +1,83 @@
|
||||
# `services/registry-augmentation` — ServiceRegistry augmentation
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Augments the `ServiceRegistry` interface from
|
||||
`@suaveplan/api-plugin/service-registry` with the `importService` slot
|
||||
so any consumer plugin typed against the augmented registry can call
|
||||
`ctx.getService("importService")` and receive a typed
|
||||
`ImportService | undefined` without a cast.
|
||||
|
||||
The augmentation also re-declares the host-side `app` slot owned by
|
||||
`@suaveplan/api` as a cumulative augmentation. TypeScript merges
|
||||
duplicate declarations, so consumers don't need to explicitly import
|
||||
`@suaveplan/api`'s augmentation module just to resolve `app` from
|
||||
inside our plugin's lifecycle hooks.
|
||||
|
||||
## Features
|
||||
|
||||
- One slot added — `importService: ImportService`.
|
||||
- Cumulative re-declaration of `app: OpenAPIHono` so the import
|
||||
plugin's `onReady` can resolve the host app without forcing a
|
||||
consumer-side import.
|
||||
- Side-effect import; the `services/index.ts` barrel re-exports it via
|
||||
`import "./registry-augmentation.js"`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
// Anywhere consuming the import plugin
|
||||
import "@suaveplan/api-plugin-import"; // pulls in the augmentation
|
||||
import type { PluginContext } from "@suaveplan/api-plugin";
|
||||
|
||||
function readService(ctx: PluginContext): void {
|
||||
const svc = ctx.getService("importService"); // typed as ImportService | undefined
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
// Direct module-augmentation chain for downstream extension authors
|
||||
import "@suaveplan/api-plugin-import"; // adds importService
|
||||
import "@suaveplan/api-plugin-database"; // adds database services
|
||||
|
||||
import type { PluginContext } from "@suaveplan/api-plugin";
|
||||
|
||||
function bootstrap(ctx: PluginContext): void {
|
||||
const svc = ctx.getService("importService");
|
||||
const db = ctx.getService("databasePool");
|
||||
if (svc && db) {
|
||||
// wire a custom adapter that writes via the database pool
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
This module exports nothing at runtime — it is pure TypeScript module
|
||||
augmentation:
|
||||
|
||||
```ts
|
||||
declare module "@suaveplan/api-plugin/service-registry" {
|
||||
interface ServiceRegistry {
|
||||
importService: ImportService;
|
||||
app: OpenAPIHono;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The augmentation is a `declare module` statement, not a value export;
|
||||
treeshaking under `sideEffects: false` keeps it in the bundle because
|
||||
TypeScript module-augmentation has no runtime correlate.
|
||||
- The `app` re-declaration intentionally mirrors the host package; the
|
||||
TypeScript compiler merges identical interface declarations across
|
||||
modules, so the duplicate is type-safe.
|
||||
- A consumer that DOES import `@suaveplan/api`'s registry directly will
|
||||
see only one merged slot — there is no double-declaration error.
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* ServiceRegistry module augmentation — declares the `importService`
|
||||
* slot on `@suaveplan/api-plugin/service-registry`'s `ServiceRegistry`
|
||||
* interface so consumer plugins typed against the augmented registry
|
||||
* see `ctx.getService("importService")` resolve to `ImportService |
|
||||
* undefined` without any cast.
|
||||
*
|
||||
* The host-side slot `app` is owned by `@suaveplan/api`'s own
|
||||
* registry-augmentation module — re-declared here as a cumulative
|
||||
* augmentation so the import plugin's `onReady(ctx)` can resolve it
|
||||
* with full typing even when the consumer has not explicitly imported
|
||||
* `@suaveplan/api`.
|
||||
*/
|
||||
import type { OpenAPIHono } from "@hono/zod-openapi";
|
||||
import type { ImportService } from "../service/service.js";
|
||||
|
||||
declare module "@suaveplan/api-plugin/service-registry" {
|
||||
interface ServiceRegistry {
|
||||
importService: ImportService;
|
||||
app: OpenAPIHono;
|
||||
}
|
||||
}
|
||||
12
packages/api-stack/api/api-plugin-import/src/store/index.ts
Normal file
12
packages/api-stack/api/api-plugin-import/src/store/index.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
export {
|
||||
type ImportStoreAdapter,
|
||||
type ImportStoreListFilter,
|
||||
MemoryImportStore,
|
||||
type MemoryImportStoreOptions,
|
||||
} from "./memory.js";
|
||||
export {
|
||||
DEFAULT_IMPORT_JOB_TABLE,
|
||||
SurrealImportStore,
|
||||
type SurrealImportStoreConfig,
|
||||
SurrealImportStoreConfigSchema,
|
||||
} from "./surreal/index.js";
|
||||
95
packages/api-stack/api/api-plugin-import/src/store/memory.md
Normal file
95
packages/api-stack/api/api-plugin-import/src/store/memory.md
Normal file
@@ -0,0 +1,95 @@
|
||||
# `store/memory` — in-memory import job store
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the async `ImportStoreAdapter` interface and ships the
|
||||
`MemoryImportStore` implementation used by the canonical `ImportService`
|
||||
to persist `ImportJobRecord` entries across the run / lookup boundary.
|
||||
The interface is the seam for swapping in durable backends (Surreal,
|
||||
Postgres, Redis-backed cache) without touching the service. The
|
||||
in-memory implementation is sufficient for tests and single-process
|
||||
deployments; production hosts implement the same interface against a
|
||||
real database.
|
||||
|
||||
The store is intentionally minimal — five methods, no transactions, no
|
||||
optimistic concurrency. The service layer is the single writer per
|
||||
process; cross-process coordination is the database's responsibility
|
||||
once a real backend lands.
|
||||
|
||||
## Features
|
||||
|
||||
- Five-method async interface (`create`, `get`, `update`, `delete`,
|
||||
`list`) matching the upstream surface.
|
||||
- `update` on a missing id throws with `Job "<id>" not found` so the
|
||||
service layer can map the failure into `IMPORT.STORAGE_ERROR`.
|
||||
- `list` filters by `status` and `ownerId` and honours `limit` / `offset`
|
||||
paging — defaults `limit: 50`, `offset: 0`.
|
||||
- Injectable `Clock` for deterministic `updatedAt` stamps.
|
||||
- `size()` / `clear()` test helpers for cheap setup/teardown.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { MemoryImportStore } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const store = new MemoryImportStore();
|
||||
await store.create({
|
||||
id: "j1",
|
||||
ownerId: "alice",
|
||||
status: "created",
|
||||
totalRows: 0,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { MemoryImportStore, type ImportStoreAdapter } from "@suaveplan/api-plugin-import";
|
||||
|
||||
// Inject a fixed clock so the updatedAt is reproducible in tests.
|
||||
const clock = () => 1_700_000_000_000;
|
||||
const store: ImportStoreAdapter = new MemoryImportStore({ clock });
|
||||
|
||||
await store.create(record);
|
||||
await store.update(record.id, { status: "completed" });
|
||||
const fresh = await store.get(record.id);
|
||||
// fresh.updatedAt === new Date(1_700_000_000_000).toISOString()
|
||||
|
||||
const recent = await store.list({ status: "completed", ownerId: "alice", limit: 10 });
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface ImportStoreAdapter`
|
||||
- `create(job: ImportJobRecord): Promise<void>`
|
||||
- `get(id: string): Promise<ImportJobRecord | undefined>`
|
||||
- `update(id: string, patch: Partial<ImportJobRecord>): Promise<void>`
|
||||
- `delete(id: string): Promise<boolean>`
|
||||
- `list(filter?: ImportStoreListFilter): Promise<ImportJobRecord[]>`
|
||||
- `interface ImportStoreListFilter { status?; ownerId?; limit?; offset? }`
|
||||
- `class MemoryImportStore implements ImportStoreAdapter`
|
||||
- `interface MemoryImportStoreOptions { clock? }`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The map keys are job ids — collisions overwrite (mirrors the upstream
|
||||
surface) so callers MUST generate unique ids per job.
|
||||
- `delete` returns `false` for missing ids (mirrors `Map.delete`); no
|
||||
exception is raised because the route handler treats a missing job
|
||||
as a 404, not a 500.
|
||||
- `update` re-uses the patch's `updatedAt` when supplied so callers
|
||||
controlling the clock can stamp deterministic timestamps without
|
||||
swapping out the entire store.
|
||||
- `list` does NOT mutate the underlying map; the returned array is a
|
||||
fresh slice of cloned references, so callers can mutate the result
|
||||
without poisoning the store.
|
||||
@@ -0,0 +1,113 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { ImportJobRecord } from "../types/types.js";
|
||||
import { MemoryImportStore } from "./memory.js";
|
||||
|
||||
function makeRecord(
|
||||
id: string,
|
||||
overrides: Partial<ImportJobRecord> = {}
|
||||
): ImportJobRecord {
|
||||
return {
|
||||
id,
|
||||
ownerId: "alice",
|
||||
status: "created",
|
||||
totalRows: 0,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
createdAt: "2026-05-18T00:00:00.000Z",
|
||||
updatedAt: "2026-05-18T00:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("MemoryImportStore", () => {
|
||||
it("round-trips a record via create and get", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
const record = makeRecord("j1");
|
||||
await store.create(record);
|
||||
const fetched = await store.get("j1");
|
||||
expect(fetched).toEqual(record);
|
||||
expect(store.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("throws when updating a missing id", { timeout: 5000 }, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
await expect(
|
||||
store.update("missing", { status: "completed" })
|
||||
).rejects.toThrow(/missing/);
|
||||
});
|
||||
|
||||
it("update merges the patch and refreshes updatedAt via the injected clock", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fixed = 1_700_000_000_000;
|
||||
const store = new MemoryImportStore({ clock: () => fixed });
|
||||
await store.create(makeRecord("j1"));
|
||||
await store.update("j1", { status: "completed" });
|
||||
const fresh = await store.get("j1");
|
||||
expect(fresh?.status).toBe("completed");
|
||||
expect(fresh?.updatedAt).toBe(new Date(fixed).toISOString());
|
||||
});
|
||||
|
||||
it("respects an explicit updatedAt patch", { timeout: 5000 }, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
await store.create(makeRecord("j1"));
|
||||
await store.update("j1", { updatedAt: "manual-stamp" });
|
||||
const fresh = await store.get("j1");
|
||||
expect(fresh?.updatedAt).toBe("manual-stamp");
|
||||
});
|
||||
|
||||
it("delete returns true for present ids and false for missing ones", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
await store.create(makeRecord("j1"));
|
||||
expect(await store.delete("j1")).toBe(true);
|
||||
expect(await store.delete("missing")).toBe(false);
|
||||
});
|
||||
|
||||
it("list filters by status, ownerId, limit, offset", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
await store.create(makeRecord("j1", { status: "completed" }));
|
||||
await store.create(makeRecord("j2", { status: "failed" }));
|
||||
await store.create(
|
||||
makeRecord("j3", {
|
||||
status: "completed",
|
||||
ownerId: "bob",
|
||||
})
|
||||
);
|
||||
|
||||
const completed = await store.list({ status: "completed" });
|
||||
expect(completed.length).toBe(2);
|
||||
|
||||
const alice = await store.list({ ownerId: "alice" });
|
||||
expect(alice.length).toBe(2);
|
||||
|
||||
const limited = await store.list({ limit: 1, offset: 1 });
|
||||
expect(limited.length).toBe(1);
|
||||
});
|
||||
|
||||
it("clear removes everything", { timeout: 5000 }, async () => {
|
||||
const store = new MemoryImportStore();
|
||||
await store.create(makeRecord("j1"));
|
||||
store.clear();
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("default clock stamps updatedAt with Date.now", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const before = Date.now();
|
||||
const store = new MemoryImportStore();
|
||||
await store.create(makeRecord("j1"));
|
||||
await store.update("j1", { status: "completed" });
|
||||
const fresh = await store.get("j1");
|
||||
const stamp = Date.parse(fresh?.updatedAt ?? "");
|
||||
expect(stamp).toBeGreaterThanOrEqual(before);
|
||||
});
|
||||
});
|
||||
104
packages/api-stack/api/api-plugin-import/src/store/memory.ts
Normal file
104
packages/api-stack/api/api-plugin-import/src/store/memory.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
import type { ImportJobRecord, ImportJobStatus } from "../types/types.js";
|
||||
|
||||
/** Filter options for `ImportStoreAdapter.list`. */
|
||||
export interface ImportStoreListFilter {
|
||||
readonly status?: ImportJobStatus;
|
||||
readonly ownerId?: string;
|
||||
readonly limit?: number;
|
||||
readonly offset?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Async-CRUD interface for persisting `ImportJobRecord` entries.
|
||||
*
|
||||
* Implementations MUST resolve `update` to a rejection when the
|
||||
* record does not exist; the in-memory implementation throws an
|
||||
* `Error` whose message matches `Job "<id>" not found`.
|
||||
*/
|
||||
export interface ImportStoreAdapter {
|
||||
create(job: ImportJobRecord): Promise<void>;
|
||||
get(id: string): Promise<ImportJobRecord | undefined>;
|
||||
update(id: string, patch: Partial<ImportJobRecord>): Promise<void>;
|
||||
delete(id: string): Promise<boolean>;
|
||||
list(filter?: ImportStoreListFilter): Promise<ImportJobRecord[]>;
|
||||
}
|
||||
|
||||
/** Options accepted by `MemoryImportStore`. */
|
||||
export interface MemoryImportStoreOptions {
|
||||
/** Injectable clock for deterministic `updatedAt` timestamps. */
|
||||
readonly clock?: Clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory `ImportStoreAdapter` implementation. Used by tests and
|
||||
* single-process deployments; production hosts should swap in a
|
||||
* durable adapter (Surreal, Postgres) implementing the same
|
||||
* interface.
|
||||
*/
|
||||
export class MemoryImportStore implements ImportStoreAdapter {
|
||||
private jobs = new Map<string, ImportJobRecord>();
|
||||
private readonly clock: Clock;
|
||||
|
||||
public constructor(options: MemoryImportStoreOptions = {}) {
|
||||
this.clock = options.clock ?? defaultClock;
|
||||
}
|
||||
|
||||
public async create(job: ImportJobRecord): Promise<void> {
|
||||
this.jobs.set(job.id, job);
|
||||
}
|
||||
|
||||
public async get(id: string): Promise<ImportJobRecord | undefined> {
|
||||
return this.jobs.get(id);
|
||||
}
|
||||
|
||||
public async update(
|
||||
id: string,
|
||||
patch: Partial<ImportJobRecord>
|
||||
): Promise<void> {
|
||||
const existing = this.jobs.get(id);
|
||||
if (!existing) {
|
||||
throw new Error(`Job "${id}" not found`);
|
||||
}
|
||||
const updatedAt =
|
||||
patch.updatedAt ?? new Date(this.clock()).toISOString();
|
||||
this.jobs.set(id, {
|
||||
...existing,
|
||||
...patch,
|
||||
updatedAt,
|
||||
});
|
||||
}
|
||||
|
||||
public async delete(id: string): Promise<boolean> {
|
||||
return this.jobs.delete(id);
|
||||
}
|
||||
|
||||
public async list(
|
||||
filter?: ImportStoreListFilter
|
||||
): Promise<ImportJobRecord[]> {
|
||||
let results = [...this.jobs.values()];
|
||||
|
||||
if (filter?.status) {
|
||||
results = results.filter((j) => j.status === filter.status);
|
||||
}
|
||||
if (filter?.ownerId !== undefined) {
|
||||
results = results.filter((j) => j.ownerId === filter.ownerId);
|
||||
}
|
||||
|
||||
const offset = filter?.offset ?? 0;
|
||||
const limit = filter?.limit ?? 50;
|
||||
return results.slice(offset, offset + limit);
|
||||
}
|
||||
|
||||
/** Helper for tests: returns the current number of stored records. */
|
||||
public size(): number {
|
||||
return this.jobs.size;
|
||||
}
|
||||
|
||||
/** Helper for tests: removes every record. */
|
||||
public clear(): void {
|
||||
this.jobs.clear();
|
||||
}
|
||||
}
|
||||
|
||||
const defaultClock: Clock = () => Date.now();
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
DEFAULT_IMPORT_JOB_TABLE,
|
||||
SurrealImportStore,
|
||||
type SurrealImportStoreConfig,
|
||||
SurrealImportStoreConfigSchema,
|
||||
} from "./surreal-import-store.js";
|
||||
@@ -0,0 +1,155 @@
|
||||
# `store/surreal/surreal-import-store` — SurrealDB-backed ImportStoreAdapter
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Implements the canonical `ImportStoreAdapter` interface against a connected
|
||||
`SurrealAdapter` from `@suaveplan/surrealdb`, providing the durable counterpart
|
||||
to the in-process `MemoryImportStore`. The store stores every `ImportJobRecord`
|
||||
in a single `import_job` table (configurable) and uses positional SurrealQL
|
||||
placeholders (`$p0`, `$p1`, …) so it matches the suaveplan adapter contract —
|
||||
the same pattern used by `@suaveplan/api-plugin-events`'s `EventRepository`.
|
||||
|
||||
The store is a drop-in replacement for `MemoryImportStore`: every public method
|
||||
preserves the original semantics (including the verbatim
|
||||
`Job "<id>" not found` rejection on `update` against a missing id) so callers
|
||||
that selected the in-memory adapter for development can swap to SurrealDB in
|
||||
production without changing call sites.
|
||||
|
||||
## Features
|
||||
|
||||
- Full `ImportStoreAdapter` parity — `create`, `get`, `update`, `delete`, `list`.
|
||||
- Positional SurrealQL placeholders (`$p0`, `$p1`, …) matching the suaveplan
|
||||
`DatabaseAdapter.query(sql, params: readonly unknown[])` surface.
|
||||
- Zod-validated constructor config — bad input throws
|
||||
`ImportError(IMPORT_CODES.INVALID_OPTIONS)` with the `ZodError` as `cause`.
|
||||
- Adapter failures are wrapped as `ImportError(IMPORT_CODES.STORAGE_ERROR)`
|
||||
with the original throwable preserved as `cause` and the failing SurQL +
|
||||
operation captured in `error.context`.
|
||||
- Configurable table name (`table?: string`, defaults to `"import_job"`).
|
||||
- Injectable `Clock` for deterministic `updatedAt` stamping in `update`.
|
||||
- Emits `import.store.writes_total{outcome}` on every write and
|
||||
`import.store.reads_total{outcome}` on every read; a `get` that returns
|
||||
`undefined` is still counted as a successful read.
|
||||
- Snake-case ↔ camel-case mapping built in (SurrealDB columns are snake_case
|
||||
per the established `api-plugin-events` schema convention; JavaScript
|
||||
records remain camelCase).
|
||||
- No mocks — unit tests use a fake `SurrealAdapter` that captures SQL +
|
||||
params; integration tests run against a real SurrealDB testcontainer.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { SurrealAdapter } from "@suaveplan/surrealdb";
|
||||
import {
|
||||
SurrealImportStore,
|
||||
type ImportJobRecord,
|
||||
} from "@suaveplan/api-plugin-import";
|
||||
|
||||
const db = new SurrealAdapter({
|
||||
url: "ws://surreal:8000/rpc",
|
||||
namespace: "app",
|
||||
database: "imports",
|
||||
username: "root",
|
||||
password: "root",
|
||||
});
|
||||
await db.connect();
|
||||
|
||||
const store = new SurrealImportStore({ db });
|
||||
|
||||
const record: ImportJobRecord = {
|
||||
id: "j1",
|
||||
ownerId: "alice",
|
||||
status: "created",
|
||||
totalRows: 0,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
createdAt: new Date().toISOString(),
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
await store.create(record);
|
||||
const fetched = await store.get("j1");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
Swap in a custom table name, inject a deterministic clock for tests, and pull
|
||||
the OTel bundle eagerly so the store never re-resolves the meter on hot paths:
|
||||
|
||||
```ts
|
||||
import { getImportInstruments, SurrealImportStore } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const fixedNow = 1_700_000_000_000;
|
||||
const store = new SurrealImportStore({
|
||||
db,
|
||||
table: "tenant_a_import_job",
|
||||
clock: () => fixedNow,
|
||||
instruments: getImportInstruments(),
|
||||
});
|
||||
|
||||
// `update` on a missing id rejects with the canonical message — parity
|
||||
// with `MemoryImportStore`. Adapter-layer SurQL errors surface as
|
||||
// `ImportError(IMPORT_CODES.STORAGE_ERROR)`.
|
||||
try {
|
||||
await store.update("missing", { status: "completed" });
|
||||
} catch (err) {
|
||||
// err.message === 'Job "missing" not found'
|
||||
}
|
||||
|
||||
// A read against an empty backend resolves to `undefined` and still
|
||||
// records `import.store.reads_total{outcome:"success"}` — the backend
|
||||
// answered correctly.
|
||||
const missing = await store.get("never-created");
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `class SurrealImportStore implements ImportStoreAdapter`
|
||||
- `constructor(config: SurrealImportStoreConfig)` — throws
|
||||
`ImportError(IMPORT_CODES.INVALID_OPTIONS)` on Zod-validation failure
|
||||
- `create(job: ImportJobRecord): Promise<void>`
|
||||
- `get(id: string): Promise<ImportJobRecord | undefined>`
|
||||
- `update(id: string, patch: Partial<ImportJobRecord>): Promise<void>` —
|
||||
throws `Error('Job "<id>" not found')` on a missing id; wraps adapter
|
||||
failures as `ImportError(IMPORT_CODES.STORAGE_ERROR)`
|
||||
- `delete(id: string): Promise<boolean>` — returns `true` if a row was
|
||||
removed, `false` if no row existed
|
||||
- `list(filter?: ImportStoreListFilter): Promise<ImportJobRecord[]>` —
|
||||
`status`, `ownerId`, `limit` (default 50), `offset` (default 0)
|
||||
- `interface SurrealImportStoreConfig { db: SurrealAdapter; table?: string;
|
||||
logger?: LoggerLike; clock?: Clock; instruments?: ImportInstruments }`
|
||||
- `SurrealImportStoreConfigSchema: z.ZodTypeAny` — boundary schema; reuses
|
||||
the same shape as the constructor argument
|
||||
- `DEFAULT_IMPORT_JOB_TABLE = "import_job"`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The SurrealDB schema lives at `surql/tables/import_job.surql` (shipped with
|
||||
the package). The store does NOT apply migrations — the host must run them
|
||||
via `@suaveplan/api-plugin-database`'s migration runner or by manually
|
||||
invoking `adapter.query(readFileSync(...))` in `beforeAll`.
|
||||
- Columns are snake_case in SurrealDB (`owner_id`, `total_rows`, …) but the
|
||||
`ImportJobRecord` keeps the suaveplan camelCase convention. The mapping is
|
||||
fully encapsulated in `rowToRecord` and the `create` / `update` SET
|
||||
clauses; downstream callers see only `ImportJobRecord`.
|
||||
- `update` first issues a `SELECT * FROM <id>` to detect missing-id and
|
||||
preserve the `Job "<id>" not found` contract from `MemoryImportStore`. This
|
||||
is one extra round-trip per update; if write-throughput is the bottleneck
|
||||
the host can layer a `Map`-based existence cache on top.
|
||||
- The constructor accepts an optional pre-resolved `ImportInstruments`
|
||||
bundle so high-volume hosts avoid the global-`MeterProvider` lookup on
|
||||
the hot path; when omitted, the bundle is resolved lazily on the first
|
||||
call (mirrors the `getImportInstruments()` cache).
|
||||
- All errors thrown by the underlying `SurrealAdapter` are wrapped as
|
||||
`ImportError(IMPORT_CODES.STORAGE_ERROR)` with `cause` set to the original
|
||||
throwable and `context = { op, sql }` so observability tooling can surface
|
||||
the failing query without parsing the message.
|
||||
- `list` orders by `created_at DESC` to mirror most real-world job-listing
|
||||
use cases. Add an explicit `ORDER BY` upstream if a different order is
|
||||
required — the `ImportStoreListFilter` contract does not currently expose
|
||||
a `sort` field.
|
||||
@@ -0,0 +1,560 @@
|
||||
import type { SurrealAdapter } from "@suaveplan/surrealdb";
|
||||
import { beforeEach, describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { ZodError } from "zod";
|
||||
import { IMPORT_CODES, ImportError } from "../../error-codes/error-codes.js";
|
||||
import { resetImportInstruments } from "../../otel/instruments.js";
|
||||
import type { ImportJobRecord } from "../../types/types.js";
|
||||
import { SurrealImportStore } from "./surreal-import-store.js";
|
||||
|
||||
interface Call {
|
||||
sql: string;
|
||||
params: readonly unknown[] | undefined;
|
||||
}
|
||||
|
||||
function makeFakeAdapter(rows: unknown[][]): {
|
||||
db: SurrealAdapter;
|
||||
calls: Call[];
|
||||
} {
|
||||
const calls: Call[] = [];
|
||||
let i = 0;
|
||||
const db = {
|
||||
async query<T>(sql: string, params?: readonly unknown[]): Promise<T[]> {
|
||||
calls.push({ sql, params });
|
||||
const next = rows[i++] ?? [];
|
||||
return next as T[];
|
||||
},
|
||||
} as unknown as SurrealAdapter;
|
||||
return { db, calls };
|
||||
}
|
||||
|
||||
function makeFailingAdapter(): {
|
||||
db: SurrealAdapter;
|
||||
failure: Error;
|
||||
} {
|
||||
const failure = new Error("backend offline");
|
||||
const db = {
|
||||
async query(): Promise<never> {
|
||||
throw failure;
|
||||
},
|
||||
} as unknown as SurrealAdapter;
|
||||
return { db, failure };
|
||||
}
|
||||
|
||||
function makeRecord(
|
||||
id: string,
|
||||
overrides: Partial<ImportJobRecord> = {}
|
||||
): ImportJobRecord {
|
||||
return {
|
||||
id,
|
||||
ownerId: "alice",
|
||||
status: "created",
|
||||
totalRows: 0,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
createdAt: "2026-05-19T00:00:00.000Z",
|
||||
updatedAt: "2026-05-19T00:00:00.000Z",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetImportInstruments();
|
||||
});
|
||||
|
||||
describe("SurrealImportStore — construction", () => {
|
||||
it("invalid config throws INVALID_OPTIONS", { timeout: 5000 }, () => {
|
||||
const { db } = makeFakeAdapter([[]]);
|
||||
let caught: unknown;
|
||||
try {
|
||||
// Deliberately invalid: empty table must trip the Zod boundary.
|
||||
new SurrealImportStore({ db, table: "" });
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(ImportError);
|
||||
const e = caught as ImportError;
|
||||
expect(e.code).toBe(IMPORT_CODES.INVALID_OPTIONS);
|
||||
expect(e.cause).toBeInstanceOf(ZodError);
|
||||
});
|
||||
|
||||
it("invalid config throws INVALID_OPTIONS when db is missing", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
let caught: unknown;
|
||||
try {
|
||||
new SurrealImportStore({
|
||||
db: undefined as unknown as SurrealAdapter,
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(ImportError);
|
||||
expect((caught as ImportError).code).toBe(IMPORT_CODES.INVALID_OPTIONS);
|
||||
});
|
||||
|
||||
it("accepts a logger override", { timeout: 5000 }, async () => {
|
||||
const { db } = makeFakeAdapter([[]]);
|
||||
const logs: { msg: string; meta?: unknown }[] = [];
|
||||
const logger = {
|
||||
debug: (msg: string, meta?: unknown) => logs.push({ msg, meta }),
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
};
|
||||
const store = new SurrealImportStore({ db, logger });
|
||||
await store.get("j1");
|
||||
expect(logs.some((l) => l.msg.includes("surreal-import-store"))).toBe(
|
||||
true
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore.create", () => {
|
||||
it("create issues a CREATE SET statement with positional params", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.create(
|
||||
makeRecord("j1", {
|
||||
metadata: { source: "csv" },
|
||||
completedAt: "2026-05-19T00:01:00.000Z",
|
||||
})
|
||||
);
|
||||
const { sql, params } = calls[0]!;
|
||||
expect(sql).toMatch(/^CREATE import_job:j1 SET /);
|
||||
expect(sql).toContain("owner_id = $p0");
|
||||
expect(sql).toContain("status = $p1");
|
||||
expect(sql).toContain("total_rows = $p2");
|
||||
expect(sql).toContain("processed_rows = $p3");
|
||||
expect(sql).toContain("success_count = $p4");
|
||||
expect(sql).toContain("error_count = $p5");
|
||||
expect(sql).toContain("errors = $p6");
|
||||
expect(sql).toContain("metadata = $p7");
|
||||
expect(sql).toContain("created_at = type::datetime($p8)");
|
||||
expect(sql).toContain("updated_at = type::datetime($p9)");
|
||||
expect(sql).toContain("completed_at = type::datetime($p10)");
|
||||
expect(params).toEqual([
|
||||
"alice",
|
||||
"created",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
[],
|
||||
{ source: "csv" },
|
||||
"2026-05-19T00:00:00.000Z",
|
||||
"2026-05-19T00:00:00.000Z",
|
||||
"2026-05-19T00:01:00.000Z",
|
||||
]);
|
||||
});
|
||||
|
||||
it("create omits metadata and completed_at when undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.create(makeRecord("j1"));
|
||||
const sql = calls[0]!.sql;
|
||||
expect(sql).not.toContain("metadata");
|
||||
expect(sql).not.toContain("completed_at");
|
||||
});
|
||||
|
||||
it("create accepts a custom table name", { timeout: 5000 }, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db, table: "tenant_a_jobs" });
|
||||
await store.create(makeRecord("j1"));
|
||||
expect(calls[0]!.sql).toMatch(/^CREATE tenant_a_jobs:j1 SET /);
|
||||
});
|
||||
|
||||
it("create with an already-prefixed id leaves the id intact", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.create(makeRecord("import_job:custom-id"));
|
||||
expect(calls[0]!.sql).toMatch(/^CREATE import_job:custom-id SET /);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore.get", () => {
|
||||
it("get returns undefined when no row exists", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
const result = await store.get("missing");
|
||||
expect(result).toBeUndefined();
|
||||
expect(calls[0]!.sql).toBe("SELECT * FROM import_job:missing");
|
||||
});
|
||||
|
||||
it("get returns undefined when the adapter yields a null row", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([[null]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
expect(await store.get("missing")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("get round-trips a row into an ImportJobRecord", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([
|
||||
[
|
||||
{
|
||||
id: "import_job:j1",
|
||||
owner_id: "alice",
|
||||
status: "completed",
|
||||
total_rows: 10,
|
||||
processed_rows: 10,
|
||||
success_count: 10,
|
||||
error_count: 0,
|
||||
errors: [],
|
||||
metadata: { source: "csv" },
|
||||
created_at: "2026-05-19T00:00:00.000Z",
|
||||
updated_at: "2026-05-19T00:01:00.000Z",
|
||||
completed_at: "2026-05-19T00:02:00.000Z",
|
||||
},
|
||||
],
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
const result = await store.get("j1");
|
||||
expect(result).toBeDefined();
|
||||
expect(result?.id).toBe("j1");
|
||||
expect(result?.status).toBe("completed");
|
||||
expect(result?.metadata).toEqual({ source: "csv" });
|
||||
expect(result?.completedAt).toBe("2026-05-19T00:02:00.000Z");
|
||||
});
|
||||
|
||||
it("get coerces Date timestamps and null completed_at to undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([
|
||||
[
|
||||
{
|
||||
id: "import_job:j1",
|
||||
owner_id: "alice",
|
||||
status: "created",
|
||||
total_rows: 0,
|
||||
processed_rows: 0,
|
||||
success_count: 0,
|
||||
error_count: 0,
|
||||
errors: [],
|
||||
metadata: null,
|
||||
created_at: new Date("2026-05-19T00:00:00.000Z"),
|
||||
updated_at: new Date("2026-05-19T00:01:00.000Z"),
|
||||
completed_at: null,
|
||||
},
|
||||
],
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
const result = await store.get("j1");
|
||||
expect(result?.createdAt).toBe("2026-05-19T00:00:00.000Z");
|
||||
expect(result?.completedAt).toBeUndefined();
|
||||
expect(result?.metadata).toBeUndefined();
|
||||
});
|
||||
|
||||
it("get falls back to safe defaults when fields are missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([[{ id: "import_job:j1" }]]);
|
||||
const store = new SurrealImportStore({
|
||||
db,
|
||||
clock: () => 1_700_000_000_000,
|
||||
});
|
||||
const result = await store.get("j1");
|
||||
expect(result?.ownerId).toBe("");
|
||||
expect(result?.status).toBe("created");
|
||||
expect(result?.totalRows).toBe(0);
|
||||
expect(result?.errors).toEqual([]);
|
||||
expect(result?.createdAt).toBe(
|
||||
new Date(1_700_000_000_000).toISOString()
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore.update", () => {
|
||||
it("update on missing id throws Job not found", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([[]]); // SELECT returns nothing
|
||||
const store = new SurrealImportStore({ db });
|
||||
await expect(
|
||||
store.update("missing", { status: "completed" })
|
||||
).rejects.toThrow(/Job "missing" not found/);
|
||||
});
|
||||
|
||||
it("update threads every patched field plus updated_at", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([
|
||||
[{ id: "import_job:j1" }], // existence probe
|
||||
[{ id: "import_job:j1" }], // update result
|
||||
]);
|
||||
const store = new SurrealImportStore({
|
||||
db,
|
||||
clock: () => Date.parse("2026-05-19T05:00:00.000Z"),
|
||||
});
|
||||
await store.update("j1", {
|
||||
ownerId: "bob",
|
||||
status: "completed",
|
||||
totalRows: 5,
|
||||
processedRows: 5,
|
||||
successCount: 5,
|
||||
errorCount: 0,
|
||||
errors: [{ row: 1, message: "x" }],
|
||||
metadata: { source: "json" },
|
||||
createdAt: "2026-05-19T00:00:00.000Z",
|
||||
completedAt: "2026-05-19T05:00:00.000Z",
|
||||
});
|
||||
const updateCall = calls[1]!;
|
||||
const sql = updateCall.sql;
|
||||
expect(sql).toMatch(/^UPDATE import_job:j1 SET /);
|
||||
expect(sql).toContain("owner_id = $p0");
|
||||
expect(sql).toContain("status = $p1");
|
||||
expect(sql).toContain("total_rows = $p2");
|
||||
expect(sql).toContain("processed_rows = $p3");
|
||||
expect(sql).toContain("success_count = $p4");
|
||||
expect(sql).toContain("error_count = $p5");
|
||||
expect(sql).toContain("errors = $p6");
|
||||
expect(sql).toContain("metadata = $p7");
|
||||
expect(sql).toContain("created_at = type::datetime($p8)");
|
||||
expect(sql).toContain("completed_at = type::datetime($p9)");
|
||||
expect(sql).toContain("updated_at = type::datetime($p10)");
|
||||
const params = updateCall.params!;
|
||||
expect(params[params.length - 1]).toBe("2026-05-19T05:00:00.000Z");
|
||||
});
|
||||
|
||||
it("update only emits the supplied patch fields", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([
|
||||
[{ id: "import_job:j1" }],
|
||||
[{ id: "import_job:j1" }],
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.update("j1", { status: "completed" });
|
||||
const sql = calls[1]!.sql;
|
||||
expect(sql).toContain("status = $p0");
|
||||
expect(sql).toContain("updated_at = type::datetime($p1)");
|
||||
expect(sql).not.toContain("owner_id");
|
||||
expect(sql).not.toContain("metadata");
|
||||
});
|
||||
|
||||
it("update respects an explicit updatedAt patch", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([
|
||||
[{ id: "import_job:j1" }],
|
||||
[{ id: "import_job:j1" }],
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.update("j1", { updatedAt: "manual-stamp" });
|
||||
const params = calls[1]!.params!;
|
||||
expect(params).toContain("manual-stamp");
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore.delete", () => {
|
||||
it("delete returns true when a row exists", { timeout: 5000 }, async () => {
|
||||
const { db, calls } = makeFakeAdapter([
|
||||
[{ id: "import_job:j1" }], // existence probe
|
||||
[], // DELETE
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
expect(await store.delete("j1")).toBe(true);
|
||||
expect(calls[1]!.sql).toBe("DELETE import_job:j1");
|
||||
});
|
||||
|
||||
it("delete returns false when no row exists", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
expect(await store.delete("missing")).toBe(false);
|
||||
expect(calls).toHaveLength(1); // probe only — no DELETE issued
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore.list", () => {
|
||||
it("list filters by status, ownerId, limit, offset", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.list({
|
||||
status: "completed",
|
||||
ownerId: "alice",
|
||||
limit: 10,
|
||||
offset: 5,
|
||||
});
|
||||
const { sql, params } = calls[0]!;
|
||||
expect(sql).toContain("WHERE status = $p0 AND owner_id = $p1");
|
||||
expect(sql).toContain("ORDER BY created_at DESC");
|
||||
expect(sql).toContain("LIMIT $p2");
|
||||
expect(sql).toContain("START $p3");
|
||||
expect(params).toEqual(["completed", "alice", 10, 5]);
|
||||
});
|
||||
|
||||
it("list applies default limit/offset", { timeout: 5000 }, async () => {
|
||||
const { db, calls } = makeFakeAdapter([[]]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
await store.list();
|
||||
expect(calls[0]!.sql).toContain("LIMIT $p0");
|
||||
expect(calls[0]!.sql).not.toContain("START");
|
||||
expect(calls[0]!.params).toEqual([50]);
|
||||
});
|
||||
|
||||
it("list maps every row through rowToRecord", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([
|
||||
[
|
||||
{
|
||||
id: "import_job:j1",
|
||||
owner_id: "alice",
|
||||
status: "completed",
|
||||
total_rows: 1,
|
||||
processed_rows: 1,
|
||||
success_count: 1,
|
||||
error_count: 0,
|
||||
errors: [],
|
||||
created_at: "2026-05-19T00:00:00.000Z",
|
||||
updated_at: "2026-05-19T00:01:00.000Z",
|
||||
},
|
||||
{
|
||||
id: { tb: "import_job", id: "j2" },
|
||||
owner_id: "bob",
|
||||
status: "failed",
|
||||
total_rows: 2,
|
||||
processed_rows: 2,
|
||||
success_count: 0,
|
||||
error_count: 2,
|
||||
errors: [{ row: 1, message: "x" }],
|
||||
created_at: "2026-05-19T00:00:00.000Z",
|
||||
updated_at: "2026-05-19T00:01:00.000Z",
|
||||
},
|
||||
],
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
const out = await store.list();
|
||||
expect(out).toHaveLength(2);
|
||||
expect(out[0]?.id).toBe("j1");
|
||||
expect(out[1]?.id).toBe("j2");
|
||||
expect(out[1]?.errorCount).toBe(2);
|
||||
});
|
||||
|
||||
it("list handles rows with string ids missing a colon and unknown ids", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([
|
||||
[
|
||||
{ id: "raw-no-colon", owner_id: "a", status: "created" },
|
||||
{ id: 42, owner_id: "b", status: "created" },
|
||||
{ id: { foo: "bar" }, owner_id: "c", status: "created" },
|
||||
{ id: undefined, owner_id: "d", status: "created" },
|
||||
],
|
||||
]);
|
||||
const store = new SurrealImportStore({ db });
|
||||
const out = await store.list();
|
||||
expect(out[0]?.id).toBe("raw-no-colon");
|
||||
expect(out[1]?.id).toBe("42");
|
||||
expect(out[2]?.id).toBe("[object Object]");
|
||||
expect(out[3]?.id).toBe("");
|
||||
});
|
||||
|
||||
it("update treats a null existence-probe row as missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db } = makeFakeAdapter([[null]]); // probe yields [null]
|
||||
const store = new SurrealImportStore({ db });
|
||||
await expect(
|
||||
store.update("ghost", { status: "completed" })
|
||||
).rejects.toThrow(/Job "ghost" not found/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore — adapter failures", () => {
|
||||
it("adapter failure surfaces as STORAGE_ERROR on read", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, failure } = makeFailingAdapter();
|
||||
const store = new SurrealImportStore({ db });
|
||||
let caught: unknown;
|
||||
try {
|
||||
await store.get("j1");
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(ImportError);
|
||||
expect((caught as ImportError).code).toBe(IMPORT_CODES.STORAGE_ERROR);
|
||||
expect((caught as ImportError).cause).toBe(failure);
|
||||
});
|
||||
|
||||
it("adapter failure surfaces as STORAGE_ERROR on list", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, failure } = makeFailingAdapter();
|
||||
const store = new SurrealImportStore({ db });
|
||||
let caught: unknown;
|
||||
try {
|
||||
await store.list();
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect((caught as ImportError).code).toBe(IMPORT_CODES.STORAGE_ERROR);
|
||||
expect((caught as ImportError).cause).toBe(failure);
|
||||
});
|
||||
|
||||
it("adapter failure surfaces as STORAGE_ERROR on create", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { db, failure } = makeFailingAdapter();
|
||||
const store = new SurrealImportStore({ db });
|
||||
let caught: unknown;
|
||||
try {
|
||||
await store.create(makeRecord("j1"));
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect((caught as ImportError).code).toBe(IMPORT_CODES.STORAGE_ERROR);
|
||||
expect((caught as ImportError).cause).toBe(failure);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SurrealImportStore — OTel emission", () => {
|
||||
it("read and write hit their respective counters", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const adds: { name: "writes" | "reads"; outcome: string }[] = [];
|
||||
const fakeCounter = (name: "writes" | "reads") => ({
|
||||
add: (_n: number, attrs?: Record<string, string>) =>
|
||||
adds.push({
|
||||
name,
|
||||
outcome: String(attrs?.["outcome"] ?? ""),
|
||||
}),
|
||||
});
|
||||
const instruments = {
|
||||
jobsTotal: { add: () => undefined },
|
||||
rowsTotal: { add: () => undefined },
|
||||
rowErrorsTotal: { add: () => undefined },
|
||||
durationMs: { record: () => undefined },
|
||||
storeWritesTotal: fakeCounter("writes"),
|
||||
storeReadsTotal: fakeCounter("reads"),
|
||||
} as unknown as import("../../otel/instruments.js").ImportInstruments;
|
||||
|
||||
const { db } = makeFakeAdapter([
|
||||
[], // get → undefined (read success)
|
||||
[], // create
|
||||
]);
|
||||
const store = new SurrealImportStore({ db, instruments });
|
||||
await store.get("missing");
|
||||
await store.create(makeRecord("j1"));
|
||||
expect(adds).toEqual([
|
||||
{ name: "reads", outcome: "success" },
|
||||
{ name: "writes", outcome: "success" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,348 @@
|
||||
import type { SurrealAdapter } from "@suaveplan/surrealdb";
|
||||
import type { Clock, LoggerLike } from "@suaveplan/types";
|
||||
import { IMPORT_CODES, ImportError } from "../../error-codes/error-codes.js";
|
||||
import {
|
||||
getImportInstruments,
|
||||
type ImportInstruments,
|
||||
} from "../../otel/instruments.js";
|
||||
import type { ImportJobRecord, ImportJobStatus } from "../../types/types.js";
|
||||
import type { ImportStoreAdapter, ImportStoreListFilter } from "../memory.js";
|
||||
import {
|
||||
DEFAULT_IMPORT_JOB_TABLE,
|
||||
type SurrealImportStoreConfig,
|
||||
SurrealImportStoreConfigSchema,
|
||||
} from "./surreal-import-store.types.js";
|
||||
|
||||
/** Row shape returned by SurrealDB for the `import_job` table. */
|
||||
interface ImportJobRow {
|
||||
readonly id?: unknown;
|
||||
readonly owner_id?: string;
|
||||
readonly status?: ImportJobStatus;
|
||||
readonly total_rows?: number;
|
||||
readonly processed_rows?: number;
|
||||
readonly success_count?: number;
|
||||
readonly error_count?: number;
|
||||
readonly errors?: ReadonlyArray<{ row: number; message: string }>;
|
||||
readonly metadata?: Record<string, unknown> | null;
|
||||
readonly created_at?: string | Date;
|
||||
readonly updated_at?: string | Date;
|
||||
readonly completed_at?: string | Date | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default logger — every method is a no-op. Allocated as a single
|
||||
* frozen object so every store without an explicit logger shares the
|
||||
* same instance (zero per-instance allocation).
|
||||
*/
|
||||
const NOOP: () => undefined = () => undefined;
|
||||
const noopLogger: LoggerLike = Object.freeze({
|
||||
debug: NOOP,
|
||||
info: NOOP,
|
||||
warn: NOOP,
|
||||
error: NOOP,
|
||||
});
|
||||
|
||||
const defaultClock: Clock = () => Date.now();
|
||||
|
||||
/**
|
||||
* SurrealDB-backed implementation of {@link ImportStoreAdapter}.
|
||||
*
|
||||
* All SurrealQL placeholders are positional `$p0`, `$p1`, … to match
|
||||
* the genesis `DatabaseAdapter.query(sql, params: readonly unknown[])`
|
||||
* surface — the adapter rewrites positional arrays into `{p0, p1, …}`
|
||||
* records before handing them to the surrealdb.js driver. This mirrors
|
||||
* the canonical pattern in `@suaveplan/api-plugin-events`'s
|
||||
* `EventRepository`.
|
||||
*
|
||||
* Schema requirement: the host MUST have applied the migration shipped
|
||||
* at `surql/tables/import_job.surql` (or an equivalent) before the
|
||||
* store is used. The store performs no migration of its own.
|
||||
*/
|
||||
export class SurrealImportStore implements ImportStoreAdapter {
|
||||
private readonly db: SurrealAdapter;
|
||||
private readonly table: string;
|
||||
private readonly logger: LoggerLike;
|
||||
private readonly clock: Clock;
|
||||
private cachedInstruments: ImportInstruments | undefined;
|
||||
|
||||
public constructor(config: SurrealImportStoreConfig) {
|
||||
const parsed = SurrealImportStoreConfigSchema.safeParse(config);
|
||||
if (!parsed.success) {
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.INVALID_OPTIONS,
|
||||
"Invalid SurrealImportStore configuration",
|
||||
{ cause: parsed.error }
|
||||
);
|
||||
}
|
||||
this.db = config.db;
|
||||
this.table = config.table ?? DEFAULT_IMPORT_JOB_TABLE;
|
||||
this.logger = config.logger ?? noopLogger;
|
||||
this.clock = config.clock ?? defaultClock;
|
||||
this.cachedInstruments = config.instruments;
|
||||
}
|
||||
|
||||
public async create(job: ImportJobRecord): Promise<void> {
|
||||
const recordId = this.formatRecordId(job.id);
|
||||
const sets: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
const push = (clause: string, value: unknown): void => {
|
||||
params.push(value);
|
||||
sets.push(clause.replace("$p", `$p${params.length - 1}`));
|
||||
};
|
||||
|
||||
push("owner_id = $p", job.ownerId);
|
||||
push("status = $p", job.status);
|
||||
push("total_rows = $p", job.totalRows);
|
||||
push("processed_rows = $p", job.processedRows);
|
||||
push("success_count = $p", job.successCount);
|
||||
push("error_count = $p", job.errorCount);
|
||||
push("errors = $p", job.errors);
|
||||
if (job.metadata !== undefined) {
|
||||
push("metadata = $p", job.metadata);
|
||||
}
|
||||
push("created_at = type::datetime($p)", job.createdAt);
|
||||
push("updated_at = type::datetime($p)", job.updatedAt);
|
||||
if (job.completedAt !== undefined) {
|
||||
push("completed_at = type::datetime($p)", job.completedAt);
|
||||
}
|
||||
|
||||
const sql = `CREATE ${recordId} SET ${sets.join(", ")}`;
|
||||
await this.write("create", sql, params);
|
||||
}
|
||||
|
||||
public async get(id: string): Promise<ImportJobRecord | undefined> {
|
||||
const recordId = this.formatRecordId(id);
|
||||
const sql = `SELECT * FROM ${recordId}`;
|
||||
const rows = await this.read<ImportJobRow>("get", sql);
|
||||
const first = rows[0];
|
||||
if (first === undefined || first === null) return undefined;
|
||||
return this.rowToRecord(id, first);
|
||||
}
|
||||
|
||||
public async update(
|
||||
id: string,
|
||||
patch: Partial<ImportJobRecord>
|
||||
): Promise<void> {
|
||||
const existing = await this.getRawRow(id);
|
||||
if (existing === undefined) {
|
||||
throw new Error(`Job "${id}" not found`);
|
||||
}
|
||||
const recordId = this.formatRecordId(id);
|
||||
const params: unknown[] = [];
|
||||
const sets: string[] = [];
|
||||
|
||||
const push = (clause: string, value: unknown): void => {
|
||||
params.push(value);
|
||||
sets.push(clause.replace("$p", `$p${params.length - 1}`));
|
||||
};
|
||||
|
||||
if (patch.ownerId !== undefined) {
|
||||
push("owner_id = $p", patch.ownerId);
|
||||
}
|
||||
if (patch.status !== undefined) push("status = $p", patch.status);
|
||||
if (patch.totalRows !== undefined) {
|
||||
push("total_rows = $p", patch.totalRows);
|
||||
}
|
||||
if (patch.processedRows !== undefined) {
|
||||
push("processed_rows = $p", patch.processedRows);
|
||||
}
|
||||
if (patch.successCount !== undefined) {
|
||||
push("success_count = $p", patch.successCount);
|
||||
}
|
||||
if (patch.errorCount !== undefined) {
|
||||
push("error_count = $p", patch.errorCount);
|
||||
}
|
||||
if (patch.errors !== undefined) push("errors = $p", patch.errors);
|
||||
if (patch.metadata !== undefined) {
|
||||
push("metadata = $p", patch.metadata);
|
||||
}
|
||||
if (patch.createdAt !== undefined) {
|
||||
push("created_at = type::datetime($p)", patch.createdAt);
|
||||
}
|
||||
if (patch.completedAt !== undefined) {
|
||||
push("completed_at = type::datetime($p)", patch.completedAt);
|
||||
}
|
||||
|
||||
const stampedUpdatedAt =
|
||||
patch.updatedAt ?? new Date(this.clock()).toISOString();
|
||||
push("updated_at = type::datetime($p)", stampedUpdatedAt);
|
||||
|
||||
const sql = `UPDATE ${recordId} SET ${sets.join(", ")}`;
|
||||
await this.write("update", sql, params);
|
||||
}
|
||||
|
||||
public async delete(id: string): Promise<boolean> {
|
||||
const existing = await this.getRawRow(id);
|
||||
if (existing === undefined) {
|
||||
// The instrument records a read+write pair: the existence
|
||||
// probe succeeded (read), the delete is a no-op (success).
|
||||
this.bumpWrite("success");
|
||||
return false;
|
||||
}
|
||||
const recordId = this.formatRecordId(id);
|
||||
const sql = `DELETE ${recordId}`;
|
||||
await this.write("delete", sql, []);
|
||||
return true;
|
||||
}
|
||||
|
||||
public async list(
|
||||
filter?: ImportStoreListFilter
|
||||
): Promise<ImportJobRecord[]> {
|
||||
const conditions: string[] = [];
|
||||
const params: unknown[] = [];
|
||||
|
||||
if (filter?.status !== undefined) {
|
||||
params.push(filter.status);
|
||||
conditions.push(`status = $p${params.length - 1}`);
|
||||
}
|
||||
if (filter?.ownerId !== undefined) {
|
||||
params.push(filter.ownerId);
|
||||
conditions.push(`owner_id = $p${params.length - 1}`);
|
||||
}
|
||||
|
||||
let sql = `SELECT * FROM ${this.table}`;
|
||||
if (conditions.length > 0) {
|
||||
sql += ` WHERE ${conditions.join(" AND ")}`;
|
||||
}
|
||||
sql += " ORDER BY created_at DESC";
|
||||
|
||||
const limit = filter?.limit ?? 50;
|
||||
params.push(limit);
|
||||
sql += ` LIMIT $p${params.length - 1}`;
|
||||
const offset = filter?.offset ?? 0;
|
||||
if (offset > 0) {
|
||||
params.push(offset);
|
||||
sql += ` START $p${params.length - 1}`;
|
||||
}
|
||||
|
||||
const rows = await this.read<ImportJobRow>("list", sql, params);
|
||||
return rows.map((row) => this.rowToRecord(this.extractId(row), row));
|
||||
}
|
||||
|
||||
/** Format a bare or fully-qualified id as a SurrealDB record id. */
|
||||
private formatRecordId(id: string): string {
|
||||
if (id.includes(":")) return id;
|
||||
return `${this.table}:${id}`;
|
||||
}
|
||||
|
||||
/** Pull a stringified id from a row regardless of how the driver shaped it. */
|
||||
private extractId(row: ImportJobRow): string {
|
||||
const raw = row.id;
|
||||
if (typeof raw === "string") {
|
||||
const colonIdx = raw.indexOf(":");
|
||||
return colonIdx >= 0 ? raw.slice(colonIdx + 1) : raw;
|
||||
}
|
||||
if (typeof raw === "object" && raw !== null) {
|
||||
const obj = raw as { tb?: unknown; id?: unknown };
|
||||
if ("id" in obj && obj.id !== undefined) return String(obj.id);
|
||||
}
|
||||
return String(raw ?? "");
|
||||
}
|
||||
|
||||
/** Map a SurrealDB row back into the canonical `ImportJobRecord`. */
|
||||
private rowToRecord(id: string, row: ImportJobRow): ImportJobRecord {
|
||||
const base: ImportJobRecord = {
|
||||
id,
|
||||
ownerId: row.owner_id ?? "",
|
||||
status: row.status ?? "created",
|
||||
totalRows: row.total_rows ?? 0,
|
||||
processedRows: row.processed_rows ?? 0,
|
||||
successCount: row.success_count ?? 0,
|
||||
errorCount: row.error_count ?? 0,
|
||||
errors: row.errors ?? [],
|
||||
createdAt: this.normaliseTimestamp(row.created_at),
|
||||
updatedAt: this.normaliseTimestamp(row.updated_at),
|
||||
};
|
||||
const withMetadata =
|
||||
row.metadata !== null && row.metadata !== undefined
|
||||
? { ...base, metadata: row.metadata }
|
||||
: base;
|
||||
if (row.completed_at === null || row.completed_at === undefined) {
|
||||
return withMetadata;
|
||||
}
|
||||
return {
|
||||
...withMetadata,
|
||||
completedAt: this.normaliseTimestamp(row.completed_at),
|
||||
};
|
||||
}
|
||||
|
||||
private normaliseTimestamp(value: string | Date | undefined): string {
|
||||
if (value === undefined) return new Date(this.clock()).toISOString();
|
||||
if (value instanceof Date) return value.toISOString();
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Lookup a row without paying the rowToRecord mapping cost. */
|
||||
private async getRawRow(id: string): Promise<ImportJobRow | undefined> {
|
||||
const recordId = this.formatRecordId(id);
|
||||
const rows = await this.read<ImportJobRow>(
|
||||
"get",
|
||||
`SELECT * FROM ${recordId}`
|
||||
);
|
||||
const first = rows[0];
|
||||
return first === null ? undefined : first;
|
||||
}
|
||||
|
||||
private async write(
|
||||
op: "create" | "update" | "delete",
|
||||
sql: string,
|
||||
params: readonly unknown[]
|
||||
): Promise<void> {
|
||||
this.logger.debug?.("surreal-import-store.write", { op, sql });
|
||||
try {
|
||||
await this.db.query(sql, params);
|
||||
this.bumpWrite("success");
|
||||
} catch (err) {
|
||||
this.bumpWrite("error");
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.STORAGE_ERROR,
|
||||
`SurrealImportStore ${op} failed`,
|
||||
{ cause: err, context: { op, sql } }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async read<T>(
|
||||
op: "get" | "list",
|
||||
sql: string,
|
||||
params: readonly unknown[] = []
|
||||
): Promise<T[]> {
|
||||
this.logger.debug?.("surreal-import-store.read", { op, sql });
|
||||
try {
|
||||
const rows = await this.db.query<T>(sql, params);
|
||||
this.bumpRead("success");
|
||||
return rows;
|
||||
} catch (err) {
|
||||
this.bumpRead("error");
|
||||
throw new ImportError(
|
||||
IMPORT_CODES.STORAGE_ERROR,
|
||||
`SurrealImportStore ${op} failed`,
|
||||
{ cause: err, context: { op, sql } }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private instruments(): ImportInstruments {
|
||||
if (this.cachedInstruments === undefined) {
|
||||
this.cachedInstruments = getImportInstruments();
|
||||
}
|
||||
return this.cachedInstruments;
|
||||
}
|
||||
|
||||
private bumpWrite(outcome: "success" | "error"): void {
|
||||
this.instruments().storeWritesTotal.add(1, { outcome });
|
||||
}
|
||||
|
||||
private bumpRead(outcome: "success" | "error"): void {
|
||||
this.instruments().storeReadsTotal.add(1, { outcome });
|
||||
}
|
||||
}
|
||||
|
||||
// Re-export the schema for tests and downstream consumers that want to
|
||||
// validate a `SurrealImportStoreConfig` shape themselves.
|
||||
export {
|
||||
DEFAULT_IMPORT_JOB_TABLE,
|
||||
type SurrealImportStoreConfig,
|
||||
SurrealImportStoreConfigSchema,
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { SurrealAdapter } from "@suaveplan/surrealdb";
|
||||
import type { Clock, LoggerLike } from "@suaveplan/types";
|
||||
import z from "zod";
|
||||
|
||||
import type { ImportInstruments } from "../../otel/instruments.js";
|
||||
|
||||
/**
|
||||
* Default SurrealDB table name used by {@link SurrealImportStore} when
|
||||
* the caller does not supply a custom `table`.
|
||||
*/
|
||||
export const DEFAULT_IMPORT_JOB_TABLE = "import_job";
|
||||
|
||||
/**
|
||||
* Construction options for `SurrealImportStore`. Validated through
|
||||
* {@link SurrealImportStoreConfigSchema} at the boundary; failures
|
||||
* surface as `ImportError(IMPORT.INVALID_OPTIONS)` with the underlying
|
||||
* `ZodError` as `cause`.
|
||||
*/
|
||||
export interface SurrealImportStoreConfig {
|
||||
/** The connected `SurrealAdapter` instance — required. */
|
||||
readonly db: SurrealAdapter;
|
||||
/** Override the SurrealDB table name. Defaults to `import_job`. */
|
||||
readonly table?: string;
|
||||
/** Optional structured logger; receives debug spans for every issued query. */
|
||||
readonly logger?: LoggerLike;
|
||||
/** Optional injectable clock for deterministic `updatedAt` stamps in `update`. */
|
||||
readonly clock?: Clock;
|
||||
/** Optional pre-resolved OTel bundle; when absent, the store calls `getImportInstruments()` lazily. */
|
||||
readonly instruments?: ImportInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Zod schema that validates the runtime-shape of {@link SurrealImportStoreConfig}.
|
||||
*
|
||||
* `db`, `logger`, `clock`, and `instruments` are accepted as opaque
|
||||
* runtime handles (`z.unknown()` + `refine`) because their TypeScript
|
||||
* shapes carry runtime methods Zod cannot mirror. `table` is the only
|
||||
* field with a structural constraint — it must be a non-empty string.
|
||||
*/
|
||||
export const SurrealImportStoreConfigSchema = z
|
||||
.object({
|
||||
db: z.unknown().refine((v): v is SurrealAdapter => {
|
||||
return (
|
||||
typeof v === "object" &&
|
||||
v !== null &&
|
||||
typeof (v as { query?: unknown }).query === "function"
|
||||
);
|
||||
}, "db must be a SurrealAdapter instance"),
|
||||
table: z.string().min(1).optional(),
|
||||
logger: z.unknown().optional(),
|
||||
clock: z.unknown().optional(),
|
||||
instruments: z.unknown().optional(),
|
||||
})
|
||||
.strict();
|
||||
@@ -0,0 +1,14 @@
|
||||
export {
|
||||
type CoerceResult,
|
||||
type ComputedField,
|
||||
coerceType,
|
||||
type FieldValidation,
|
||||
type TransformError,
|
||||
type TransformMapping,
|
||||
type TransformPipelineConfig,
|
||||
type TransformRowResult,
|
||||
toRowError,
|
||||
transformRow,
|
||||
transformRows,
|
||||
validateField,
|
||||
} from "./pipeline.js";
|
||||
@@ -0,0 +1,112 @@
|
||||
# `transform/pipeline` — declarative transform pipeline
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Implements a three-stage row-transform pipeline that any consumer can
|
||||
drive without depending on the canonical `ImportService`: per-mapping
|
||||
coerce + transform, computed fields, then field validation. The
|
||||
pipeline is pure (no side effects, no I/O) and never throws — every
|
||||
failure is captured as a `TransformError` and returned in the result
|
||||
object, so callers can collect every failed row in one pass.
|
||||
|
||||
This module powers both the standalone `transformRow` / `transformRows`
|
||||
helpers exported to consumers and the per-row inner loop of
|
||||
`ImportService.startImport`. Keeping the pipeline separate from the
|
||||
service means CLI tools, batch workers, and tests can drive the
|
||||
transform logic against arbitrary input without spinning up an event
|
||||
emitter or store adapter.
|
||||
|
||||
## Features
|
||||
|
||||
- `coerceType` — explicit-result coercion for `number` / `boolean` /
|
||||
`date` with documented "valid" sets (`1`/`0`/`yes`/`no`/`true`/`false`
|
||||
for booleans, anything `Date.parse`-able for dates).
|
||||
- `validateField` — five built-in rules (`required`, `min`, `max`,
|
||||
`pattern`, `custom`); `custom` is an opt-in extension point that
|
||||
always returns `undefined` so callers wire their own.
|
||||
- `transformRow` — applies mappings, computed fields, and validations
|
||||
in order; pushes `TransformError` for each failure and returns
|
||||
`{ success, row, errors }`.
|
||||
- `transformRows` — array fan-out; errored rows are dropped from `rows`
|
||||
but their errors are still collected in `errors`.
|
||||
- `toRowError` — bridge to the service-layer `RowError` shape so the
|
||||
service can re-emit transform failures on the `import.row_error`
|
||||
event topic without duplicating shapes.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { transformRow } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const result = transformRow(
|
||||
{ name: "Ada", age: "37" },
|
||||
0,
|
||||
{
|
||||
mappings: [
|
||||
{ sourceColumn: "name", targetField: "displayName" },
|
||||
{ sourceColumn: "age", targetField: "age", type: "number" },
|
||||
],
|
||||
}
|
||||
);
|
||||
// result.row → { displayName: "Ada", age: 37 }
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { transformRow } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const result = transformRow(
|
||||
{ first: "Ada", last: "Lovelace", born: "1815" },
|
||||
0,
|
||||
{
|
||||
mappings: [
|
||||
{ sourceColumn: "born", targetField: "birthYear", type: "number" },
|
||||
],
|
||||
computedFields: [
|
||||
{
|
||||
targetField: "fullName",
|
||||
sourceColumns: ["first", "last"],
|
||||
compute: (v) => `${String(v.first)} ${String(v.last)}`,
|
||||
},
|
||||
],
|
||||
validations: [
|
||||
{ field: "birthYear", rule: "min", param: 1800 },
|
||||
],
|
||||
}
|
||||
);
|
||||
// result.row → { birthYear: 1815, fullName: "Ada Lovelace" }
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `coerceType(value: unknown, type: string): CoerceResult`
|
||||
- `validateField(value, rule: FieldValidation, rowIndex): TransformError | undefined`
|
||||
- `transformRow(source, rowIndex, config: TransformPipelineConfig): TransformRowResult`
|
||||
- `transformRows(sources, config): { rows; errors }`
|
||||
- `toRowError(err: TransformError): RowError`
|
||||
- `interface TransformMapping { sourceColumn; targetField; type?; defaultValue?; required?; transform? }`
|
||||
- `interface ComputedField { targetField; sourceColumns; compute }`
|
||||
- `interface FieldValidation { field; rule; param?; message? }`
|
||||
- `interface TransformError { row; field; message; value? }`
|
||||
- `interface TransformRowResult { success; row; errors }`
|
||||
- `interface TransformPipelineConfig { mappings; computedFields?; validations? }`
|
||||
- `interface CoerceResult { ok; value }`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Coercion short-circuits on empty / null / undefined to `{ ok: true,
|
||||
value: undefined }` so a downstream `defaultValue` can still apply.
|
||||
- The transform function (`mapping.transform`) is wrapped in `try` so a
|
||||
thrown library never escapes the pipeline; the row's `success` flag
|
||||
reports the failure and the rest of the mappings still process.
|
||||
- `custom` validation rules are intentionally no-op — extension is via
|
||||
the `transform` callback on the mapping, which has access to the raw
|
||||
value. Custom rules without a default are documented in the
|
||||
enterprise audit as "extended by consumers."
|
||||
- `validateField` runs after both mapping and computed-field stages, so
|
||||
validations target the post-coercion result, not the raw source row.
|
||||
@@ -0,0 +1,372 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
coerceType,
|
||||
toRowError,
|
||||
transformRow,
|
||||
transformRows,
|
||||
validateField,
|
||||
} from "./pipeline.js";
|
||||
|
||||
describe("coerceType", () => {
|
||||
it("short-circuits empty / null / undefined to undefined", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(coerceType("", "number")).toEqual({
|
||||
ok: true,
|
||||
value: undefined,
|
||||
});
|
||||
expect(coerceType(null, "number")).toEqual({
|
||||
ok: true,
|
||||
value: undefined,
|
||||
});
|
||||
expect(coerceType(undefined, "boolean")).toEqual({
|
||||
ok: true,
|
||||
value: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("coerces numbers successfully", { timeout: 5000 }, () => {
|
||||
expect(coerceType("42", "number")).toEqual({ ok: true, value: 42 });
|
||||
});
|
||||
|
||||
it("coerceType number returns ok:false for non-numeric input", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(coerceType("abc", "number")).toEqual({
|
||||
ok: false,
|
||||
value: "abc",
|
||||
});
|
||||
});
|
||||
|
||||
it("coerces every truthy and falsy boolean alias", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
for (const s of ["true", "1", "yes", "TRUE", "YES"]) {
|
||||
expect(coerceType(s, "boolean")).toEqual({
|
||||
ok: true,
|
||||
value: true,
|
||||
});
|
||||
}
|
||||
for (const s of ["false", "0", "no", "FALSE", "NO"]) {
|
||||
expect(coerceType(s, "boolean")).toEqual({
|
||||
ok: true,
|
||||
value: false,
|
||||
});
|
||||
}
|
||||
expect(coerceType("maybe", "boolean")).toEqual({
|
||||
ok: false,
|
||||
value: "maybe",
|
||||
});
|
||||
});
|
||||
|
||||
it("coerces dates to ISO strings", { timeout: 5000 }, () => {
|
||||
const result = coerceType("2024-01-01", "date");
|
||||
expect(result.ok).toBe(true);
|
||||
expect(String(result.value).startsWith("2024-01-01")).toBe(true);
|
||||
expect(coerceType("not-a-date", "date")).toEqual({
|
||||
ok: false,
|
||||
value: "not-a-date",
|
||||
});
|
||||
});
|
||||
|
||||
it("passes unknown type as string", { timeout: 5000 }, () => {
|
||||
expect(coerceType("hello", "weird")).toEqual({
|
||||
ok: true,
|
||||
value: "hello",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateField", () => {
|
||||
it("validateField required produces error for missing value", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = validateField(
|
||||
undefined,
|
||||
{ field: "x", rule: "required" },
|
||||
0
|
||||
);
|
||||
expect(err).toEqual({
|
||||
row: 0,
|
||||
field: "x",
|
||||
message: '"x" is required',
|
||||
value: undefined,
|
||||
});
|
||||
expect(
|
||||
validateField("v", { field: "x", rule: "required" }, 0)
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("min and max compare numeric values", { timeout: 5000 }, () => {
|
||||
expect(
|
||||
validateField(3, { field: "x", rule: "min", param: 5 }, 0)
|
||||
).toBeDefined();
|
||||
expect(
|
||||
validateField(10, { field: "x", rule: "min", param: 5 }, 0)
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
validateField(10, { field: "x", rule: "max", param: 5 }, 0)
|
||||
).toBeDefined();
|
||||
expect(
|
||||
validateField(3, { field: "x", rule: "max", param: 5 }, 0)
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("min/max ignore non-numeric values", { timeout: 5000 }, () => {
|
||||
expect(
|
||||
validateField("x", { field: "x", rule: "min", param: 5 }, 0)
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
validateField("x", { field: "x", rule: "max", param: 5 }, 0)
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("pattern checks regex against the stringified value", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const rule = {
|
||||
field: "x",
|
||||
rule: "pattern" as const,
|
||||
param: "^\\d+$",
|
||||
};
|
||||
expect(validateField("123", rule, 0)).toBeUndefined();
|
||||
expect(validateField("abc", rule, 0)).toBeDefined();
|
||||
// Empty / undefined / null short circuit
|
||||
expect(validateField("", rule, 0)).toBeUndefined();
|
||||
expect(validateField(undefined, rule, 0)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses the supplied custom message", { timeout: 5000 }, () => {
|
||||
const err = validateField(
|
||||
undefined,
|
||||
{ field: "x", rule: "required", message: "MUST!" },
|
||||
0
|
||||
);
|
||||
expect(err?.message).toBe("MUST!");
|
||||
});
|
||||
|
||||
it("custom rule is a no-op", { timeout: 5000 }, () => {
|
||||
expect(
|
||||
validateField("x", { field: "x", rule: "custom" }, 0)
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("min/max use the custom message when supplied", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
validateField(
|
||||
3,
|
||||
{
|
||||
field: "x",
|
||||
rule: "min",
|
||||
param: 5,
|
||||
message: "tiny",
|
||||
},
|
||||
0
|
||||
)?.message
|
||||
).toBe("tiny");
|
||||
expect(
|
||||
validateField(
|
||||
10,
|
||||
{
|
||||
field: "x",
|
||||
rule: "max",
|
||||
param: 5,
|
||||
message: "huge",
|
||||
},
|
||||
0
|
||||
)?.message
|
||||
).toBe("huge");
|
||||
expect(
|
||||
validateField(
|
||||
"a",
|
||||
{
|
||||
field: "x",
|
||||
rule: "pattern",
|
||||
param: "\\d",
|
||||
message: "needs-digit",
|
||||
},
|
||||
0
|
||||
)?.message
|
||||
).toBe("needs-digit");
|
||||
});
|
||||
});
|
||||
|
||||
describe("transformRow", () => {
|
||||
it("applies mappings, computed fields, and validations", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = transformRow(
|
||||
{ first: "Ada", last: "Lovelace", born: "1815" },
|
||||
0,
|
||||
{
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "born",
|
||||
targetField: "birthYear",
|
||||
type: "number",
|
||||
},
|
||||
],
|
||||
computedFields: [
|
||||
{
|
||||
targetField: "fullName",
|
||||
sourceColumns: ["first", "last"],
|
||||
compute: (v) =>
|
||||
`${String(v["first"])} ${String(v["last"])}`,
|
||||
},
|
||||
],
|
||||
validations: [{ field: "birthYear", rule: "min", param: 1800 }],
|
||||
}
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.row).toEqual({
|
||||
birthYear: 1815,
|
||||
fullName: "Ada Lovelace",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses defaultValue when source is missing", { timeout: 5000 }, () => {
|
||||
const result = transformRow({}, 0, {
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "x",
|
||||
targetField: "x",
|
||||
defaultValue: "fallback",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.row).toEqual({ x: "fallback" });
|
||||
});
|
||||
|
||||
it("coerceType failure marks the row as failed and skips the field", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = transformRow({ x: "abc" }, 0, {
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "x",
|
||||
targetField: "x",
|
||||
type: "number",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.row).toEqual({});
|
||||
expect(result.errors[0]?.message).toContain("Cannot coerce");
|
||||
});
|
||||
|
||||
it("transform throw records a TransformError", { timeout: 5000 }, () => {
|
||||
const result = transformRow({ x: "hello" }, 0, {
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "x",
|
||||
targetField: "x",
|
||||
transform: () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.errors[0]?.message).toBe("Transform function failed");
|
||||
});
|
||||
|
||||
it("computed field is evaluated from named source columns", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = transformRow({ a: 1, b: 2 }, 0, {
|
||||
mappings: [],
|
||||
computedFields: [
|
||||
{
|
||||
targetField: "sum",
|
||||
sourceColumns: ["a", "b"],
|
||||
compute: (v) => Number(v["a"]) + Number(v["b"]),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.row).toEqual({ sum: 3 });
|
||||
});
|
||||
|
||||
it("computed field throw records a TransformError", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = transformRow({ a: 1 }, 0, {
|
||||
mappings: [],
|
||||
computedFields: [
|
||||
{
|
||||
targetField: "x",
|
||||
sourceColumns: ["a"],
|
||||
compute: () => {
|
||||
throw new Error("nope");
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.errors[0]?.message).toBe(
|
||||
"Computed field function failed"
|
||||
);
|
||||
});
|
||||
|
||||
it("validation failures show up in errors", { timeout: 5000 }, () => {
|
||||
const result = transformRow({}, 0, {
|
||||
mappings: [],
|
||||
validations: [{ field: "x", rule: "required" }],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.errors[0]?.field).toBe("x");
|
||||
});
|
||||
});
|
||||
|
||||
describe("transformRows", () => {
|
||||
it("drops errored rows from rows but keeps their errors", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = transformRows([{ x: "1" }, { x: "abc" }, { x: "3" }], {
|
||||
mappings: [
|
||||
{
|
||||
sourceColumn: "x",
|
||||
targetField: "n",
|
||||
type: "number",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.rows).toEqual([{ n: 1 }, { n: 3 }]);
|
||||
expect(result.errors.length).toBe(1);
|
||||
expect(result.errors[0]?.row).toBe(1);
|
||||
});
|
||||
|
||||
it("tolerates a sparse input array", { timeout: 5000 }, () => {
|
||||
// eslint-ignore — sparse arrays are valid input
|
||||
const sparse: Record<string, unknown>[] = [];
|
||||
sparse[2] = { x: "5" };
|
||||
const result = transformRows(sparse, {
|
||||
mappings: [{ sourceColumn: "x", targetField: "x" }],
|
||||
});
|
||||
// Rows 0 and 1 are processed as `{}` (success, empty row), row 2 returns { x: "5" }
|
||||
expect(result.rows.length).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("toRowError", () => {
|
||||
it("converts a TransformError to a RowError", { timeout: 5000 }, () => {
|
||||
const r = toRowError({
|
||||
row: 4,
|
||||
field: "f",
|
||||
message: "m",
|
||||
value: "v",
|
||||
});
|
||||
expect(r).toEqual({
|
||||
row: 4,
|
||||
column: "f",
|
||||
message: "m",
|
||||
value: "v",
|
||||
});
|
||||
});
|
||||
|
||||
it("carries undefined value through", { timeout: 5000 }, () => {
|
||||
const r = toRowError({ row: 1, field: "f", message: "m" });
|
||||
expect(r.value).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,281 @@
|
||||
import type { RowError } from "../types/types.js";
|
||||
|
||||
/** Column mapping rule for transformation. */
|
||||
export interface TransformMapping {
|
||||
readonly sourceColumn: string;
|
||||
readonly targetField: string;
|
||||
readonly type?: "string" | "number" | "boolean" | "date";
|
||||
readonly defaultValue?: unknown;
|
||||
readonly required?: boolean;
|
||||
readonly transform?: (value: unknown) => unknown;
|
||||
}
|
||||
|
||||
/** Computed field derived from source columns. */
|
||||
export interface ComputedField {
|
||||
readonly targetField: string;
|
||||
readonly sourceColumns: string[];
|
||||
readonly compute: (values: Record<string, unknown>) => unknown;
|
||||
}
|
||||
|
||||
/** Validation rule applied per field after coercion + computed-field expansion. */
|
||||
export interface FieldValidation {
|
||||
readonly field: string;
|
||||
readonly rule: "required" | "min" | "max" | "pattern" | "custom";
|
||||
readonly param?: unknown;
|
||||
readonly message?: string;
|
||||
}
|
||||
|
||||
/** Error emitted by `transformRow`. */
|
||||
export interface TransformError {
|
||||
readonly row: number;
|
||||
readonly field: string;
|
||||
readonly message: string;
|
||||
readonly value?: unknown;
|
||||
}
|
||||
|
||||
/** Result from processing a single row. */
|
||||
export interface TransformRowResult {
|
||||
readonly success: boolean;
|
||||
readonly row: Record<string, unknown>;
|
||||
readonly errors: TransformError[];
|
||||
}
|
||||
|
||||
/** Pipeline configuration accepted by `transformRow` / `transformRows`. */
|
||||
export interface TransformPipelineConfig {
|
||||
readonly mappings: TransformMapping[];
|
||||
readonly computedFields?: ComputedField[];
|
||||
readonly validations?: FieldValidation[];
|
||||
}
|
||||
|
||||
/** Result of `coerceType`. */
|
||||
export interface CoerceResult {
|
||||
readonly ok: boolean;
|
||||
readonly value: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce a value to the requested type. Returns `{ ok: false, value }`
|
||||
* for inputs that cannot be coerced (NaN numbers, non-truthy boolean
|
||||
* strings, un-parseable dates). Empty / null / undefined values short
|
||||
* circuit to `{ ok: true, value: undefined }` so the pipeline can apply
|
||||
* a `defaultValue` upstream.
|
||||
*/
|
||||
export function coerceType(value: unknown, type: string): CoerceResult {
|
||||
if (value === undefined || value === null || value === "") {
|
||||
return { ok: true, value: undefined };
|
||||
}
|
||||
const str = String(value);
|
||||
|
||||
switch (type) {
|
||||
case "number": {
|
||||
const num = Number(str);
|
||||
if (Number.isNaN(num)) return { ok: false, value };
|
||||
return { ok: true, value: num };
|
||||
}
|
||||
case "boolean": {
|
||||
const lower = str.toLowerCase();
|
||||
if (lower === "true" || lower === "1" || lower === "yes") {
|
||||
return { ok: true, value: true };
|
||||
}
|
||||
if (lower === "false" || lower === "0" || lower === "no") {
|
||||
return { ok: true, value: false };
|
||||
}
|
||||
return { ok: false, value };
|
||||
}
|
||||
case "date": {
|
||||
const ts = Date.parse(str);
|
||||
if (Number.isNaN(ts)) return { ok: false, value };
|
||||
return { ok: true, value: new Date(ts).toISOString() };
|
||||
}
|
||||
default:
|
||||
return { ok: true, value: str };
|
||||
}
|
||||
}
|
||||
|
||||
/** Validate a field value against a single `FieldValidation` rule. */
|
||||
export function validateField(
|
||||
value: unknown,
|
||||
rule: FieldValidation,
|
||||
rowIndex: number
|
||||
): TransformError | undefined {
|
||||
switch (rule.rule) {
|
||||
case "required":
|
||||
if (value === undefined || value === null || value === "") {
|
||||
return {
|
||||
row: rowIndex,
|
||||
field: rule.field,
|
||||
message: rule.message ?? `"${rule.field}" is required`,
|
||||
value,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
case "min":
|
||||
if (typeof value === "number" && value < (rule.param as number)) {
|
||||
return {
|
||||
row: rowIndex,
|
||||
field: rule.field,
|
||||
message:
|
||||
rule.message ??
|
||||
`"${rule.field}" must be >= ${String(rule.param)}`,
|
||||
value,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
case "max":
|
||||
if (typeof value === "number" && value > (rule.param as number)) {
|
||||
return {
|
||||
row: rowIndex,
|
||||
field: rule.field,
|
||||
message:
|
||||
rule.message ??
|
||||
`"${rule.field}" must be <= ${String(rule.param)}`,
|
||||
value,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
case "pattern":
|
||||
if (
|
||||
value !== undefined &&
|
||||
value !== null &&
|
||||
value !== "" &&
|
||||
!new RegExp(rule.param as string).test(String(value))
|
||||
) {
|
||||
return {
|
||||
row: rowIndex,
|
||||
field: rule.field,
|
||||
message:
|
||||
rule.message ??
|
||||
`"${rule.field}" does not match pattern`,
|
||||
value,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
case "custom":
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform a single source row through the pipeline:
|
||||
*
|
||||
* 1. For every mapping, pull the source value, apply `defaultValue` if
|
||||
* missing, run `coerceType` if `type` is set, then run `transform`
|
||||
* if supplied. Failures push a `TransformError` and skip the field.
|
||||
* 2. For every computed field, build the input record of source values
|
||||
* and run `compute`; failures push a `TransformError`.
|
||||
* 3. For every validation, run `validateField` against the mapped /
|
||||
* computed result and push the error if any.
|
||||
*
|
||||
* Never throws; failures are returned in `errors`.
|
||||
*/
|
||||
export function transformRow(
|
||||
source: Record<string, unknown>,
|
||||
rowIndex: number,
|
||||
config: TransformPipelineConfig
|
||||
): TransformRowResult {
|
||||
const result: Record<string, unknown> = {};
|
||||
const errors: TransformError[] = [];
|
||||
|
||||
for (const mapping of config.mappings) {
|
||||
let value = source[mapping.sourceColumn];
|
||||
|
||||
if (
|
||||
(value === undefined || value === null || value === "") &&
|
||||
mapping.defaultValue !== undefined
|
||||
) {
|
||||
value = mapping.defaultValue;
|
||||
}
|
||||
|
||||
if (
|
||||
mapping.type &&
|
||||
value !== undefined &&
|
||||
value !== null &&
|
||||
value !== ""
|
||||
) {
|
||||
const coerced = coerceType(value, mapping.type);
|
||||
if (!coerced.ok) {
|
||||
errors.push({
|
||||
row: rowIndex,
|
||||
field: mapping.targetField,
|
||||
message: `Cannot coerce to ${mapping.type}`,
|
||||
value,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
value = coerced.value;
|
||||
}
|
||||
|
||||
if (mapping.transform) {
|
||||
try {
|
||||
value = mapping.transform(value);
|
||||
} catch {
|
||||
errors.push({
|
||||
row: rowIndex,
|
||||
field: mapping.targetField,
|
||||
message: "Transform function failed",
|
||||
value,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
result[mapping.targetField] = value;
|
||||
}
|
||||
|
||||
if (config.computedFields) {
|
||||
for (const computed of config.computedFields) {
|
||||
const sourceValues: Record<string, unknown> = {};
|
||||
for (const col of computed.sourceColumns) {
|
||||
sourceValues[col] = source[col];
|
||||
}
|
||||
try {
|
||||
result[computed.targetField] = computed.compute(sourceValues);
|
||||
} catch {
|
||||
errors.push({
|
||||
row: rowIndex,
|
||||
field: computed.targetField,
|
||||
message: "Computed field function failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (config.validations) {
|
||||
for (const validation of config.validations) {
|
||||
const fieldValue = result[validation.field];
|
||||
const error = validateField(fieldValue, validation, rowIndex);
|
||||
if (error) errors.push(error);
|
||||
}
|
||||
}
|
||||
|
||||
return { success: errors.length === 0, row: result, errors };
|
||||
}
|
||||
|
||||
/** Transform an array of rows; errored rows are dropped from `rows`. */
|
||||
export function transformRows(
|
||||
sources: Record<string, unknown>[],
|
||||
config: TransformPipelineConfig
|
||||
): { rows: Record<string, unknown>[]; errors: TransformError[] } {
|
||||
const rows: Record<string, unknown>[] = [];
|
||||
const allErrors: TransformError[] = [];
|
||||
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const result = transformRow(sources[i] ?? {}, i, config);
|
||||
if (result.success) {
|
||||
rows.push(result.row);
|
||||
}
|
||||
allErrors.push(...result.errors);
|
||||
}
|
||||
|
||||
return { rows, errors: allErrors };
|
||||
}
|
||||
|
||||
/** Convert a `TransformError` into the cross-package `RowError` shape. */
|
||||
export function toRowError(err: TransformError): RowError {
|
||||
return {
|
||||
row: err.row,
|
||||
column: err.field,
|
||||
message: err.message,
|
||||
value: err.value,
|
||||
};
|
||||
}
|
||||
18
packages/api-stack/api/api-plugin-import/src/types/index.ts
Normal file
18
packages/api-stack/api/api-plugin-import/src/types/index.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
export type {
|
||||
ColumnMapping,
|
||||
ImportEventMap,
|
||||
ImportFormat,
|
||||
ImportJobConfig,
|
||||
ImportJobRecord,
|
||||
ImportJobStatus,
|
||||
ImportOwnerContext,
|
||||
ImportPreview,
|
||||
ImportProgress,
|
||||
ImportRowProcessedEvent,
|
||||
ImportStatus,
|
||||
RowError,
|
||||
} from "./types.js";
|
||||
export {
|
||||
DEFAULT_BATCH_SIZE,
|
||||
VALID_FORMATS,
|
||||
} from "./types.js";
|
||||
89
packages/api-stack/api/api-plugin-import/src/types/types.md
Normal file
89
packages/api-stack/api/api-plugin-import/src/types/types.md
Normal file
@@ -0,0 +1,89 @@
|
||||
# `types` — cross-module types
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the cross-module types and constants consumed by every other
|
||||
feature module in `@suaveplan/api-plugin-import`. Parsers, the transform
|
||||
pipeline, the progress tracker, the store adapter, the canonical
|
||||
`ImportService`, the routes, and the plugin all import their shared
|
||||
shapes from here so they stay in lockstep across a refactor. The file
|
||||
contains zero runtime logic beyond two literal constants (`VALID_FORMATS`,
|
||||
`DEFAULT_BATCH_SIZE`) — it is effectively a `.d.ts` declaration with
|
||||
named exports.
|
||||
|
||||
The types follow the suaveplan convention of `readonly` fields and
|
||||
discriminated unions rather than free-form interfaces, so consumers get
|
||||
type-narrowing on `status` and `format` without runtime casts.
|
||||
|
||||
## Features
|
||||
|
||||
- `ImportFormat` / `ImportStatus` / `ImportJobStatus` discriminated unions.
|
||||
- `ColumnMapping`, `RowError`, `ImportPreview`, `ImportProgress`,
|
||||
`ImportJobConfig`, `ImportOwnerContext` interfaces.
|
||||
- `ImportJobRecord` — store-tier shape kept distinct from
|
||||
`ImportProgress` so durable backends do not pay the per-row write cost.
|
||||
- `ImportEventMap` — strongly-typed event-bus topic map for the
|
||||
`EventEmitter<ImportEventMap>` extended by `ImportService`.
|
||||
- `VALID_FORMATS` / `DEFAULT_BATCH_SIZE` constants.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type { ImportJobConfig, ColumnMapping } from "@suaveplan/api-plugin-import";
|
||||
|
||||
const mappings: ColumnMapping[] = [
|
||||
{ sourceColumn: "name", targetField: "displayName", required: true },
|
||||
];
|
||||
|
||||
const config: ImportJobConfig = {
|
||||
format: "csv",
|
||||
data: "name\nDoe",
|
||||
mappings,
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type {
|
||||
ImportEventMap,
|
||||
ImportProgress,
|
||||
} from "@suaveplan/api-plugin-import";
|
||||
import { EventEmitter } from "@suaveplan/events";
|
||||
|
||||
const bus = new EventEmitter<ImportEventMap>();
|
||||
bus.on("import.progress", (progress: ImportProgress) => {
|
||||
console.log(progress.jobId, progress.processedRows, progress.totalRows);
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ImportFormat = "csv" | "json" | "xlsx"`
|
||||
- `ImportStatus = "pending" | "validating" | "previewing" | "importing" | "completed" | "failed"`
|
||||
- `ImportJobStatus = "created" | "validating" | "importing" | "completed" | "failed"`
|
||||
- `interface ColumnMapping { sourceColumn; targetField; transform?; required?; defaultValue? }`
|
||||
- `interface RowError { row; column; message; value? }`
|
||||
- `interface ImportPreview { totalRows; columns; sampleRows; mappings }`
|
||||
- `interface ImportProgress { jobId; ownerId; status; totalRows; processedRows; successCount; errorCount; errors; startedAt?; completedAt? }`
|
||||
- `interface ImportJobConfig { format; data; mappings; dryRun?; batchSize?; skipDuplicates?; onRow? }`
|
||||
- `interface ImportOwnerContext { ownerId }`
|
||||
- `interface ImportJobRecord { id; ownerId; status; totalRows; processedRows; successCount; errorCount; errors; metadata?; createdAt; updatedAt; completedAt? }`
|
||||
- `interface ImportEventMap { … }`
|
||||
- `interface ImportRowProcessedEvent { jobId; rowIndex; success }`
|
||||
- `const VALID_FORMATS: readonly ImportFormat[]`
|
||||
- `const DEFAULT_BATCH_SIZE = 100`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `ImportProgress` and `ImportJobRecord` are deliberately separate so the
|
||||
durable store stays narrow; the live progress object can fan-out
|
||||
per-row errors while the store records a digest.
|
||||
- `ownerId` lives on both the progress object and the record so every
|
||||
read path can enforce owner scoping without a cross-table join.
|
||||
- `ImportEventMap` widens via index signatures so consumers can layer
|
||||
app-specific topics on top without `as any`.
|
||||
130
packages/api-stack/api/api-plugin-import/src/types/types.ts
Normal file
130
packages/api-stack/api/api-plugin-import/src/types/types.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Cross-module types for `@suaveplan/api-plugin-import`. Imported by
|
||||
* parsers, transform pipeline, progress tracker, store, service,
|
||||
* routes, plugin.
|
||||
*/
|
||||
|
||||
/** Supported import formats. */
|
||||
export type ImportFormat = "csv" | "json" | "xlsx";
|
||||
|
||||
/** Import status surfaced to clients in progress snapshots. */
|
||||
export type ImportStatus =
|
||||
| "pending"
|
||||
| "validating"
|
||||
| "previewing"
|
||||
| "importing"
|
||||
| "completed"
|
||||
| "failed";
|
||||
|
||||
/** Stored job-record lifecycle on the store adapter. */
|
||||
export type ImportJobStatus =
|
||||
| "created"
|
||||
| "validating"
|
||||
| "importing"
|
||||
| "completed"
|
||||
| "failed";
|
||||
|
||||
/** Column mapping from a source-column name to a target-field name. */
|
||||
export interface ColumnMapping {
|
||||
readonly sourceColumn: string;
|
||||
readonly targetField: string;
|
||||
readonly transform?: (value: string) => unknown;
|
||||
readonly required?: boolean;
|
||||
readonly defaultValue?: unknown;
|
||||
}
|
||||
|
||||
/** Per-row validation error emitted by the import service. */
|
||||
export interface RowError {
|
||||
readonly row: number;
|
||||
readonly column: string;
|
||||
readonly message: string;
|
||||
readonly value?: unknown;
|
||||
}
|
||||
|
||||
/** Result of `ImportService.preview`. */
|
||||
export interface ImportPreview {
|
||||
readonly totalRows: number;
|
||||
readonly columns: string[];
|
||||
readonly sampleRows: Record<string, unknown>[];
|
||||
readonly mappings: ColumnMapping[];
|
||||
}
|
||||
|
||||
/** Live progress snapshot — also returned from `startImport` / `getJob`. */
|
||||
export interface ImportProgress {
|
||||
readonly jobId: string;
|
||||
readonly ownerId: string;
|
||||
readonly status: ImportStatus;
|
||||
readonly totalRows: number;
|
||||
readonly processedRows: number;
|
||||
readonly successCount: number;
|
||||
readonly errorCount: number;
|
||||
readonly errors: RowError[];
|
||||
readonly startedAt?: string;
|
||||
readonly completedAt?: string;
|
||||
}
|
||||
|
||||
/** Caller-supplied import job configuration. */
|
||||
export interface ImportJobConfig {
|
||||
readonly format: ImportFormat;
|
||||
readonly data: string;
|
||||
readonly mappings: ColumnMapping[];
|
||||
readonly dryRun?: boolean;
|
||||
readonly batchSize?: number;
|
||||
readonly skipDuplicates?: boolean;
|
||||
readonly onRow?: (row: Record<string, unknown>, index: number) => boolean;
|
||||
}
|
||||
|
||||
/** Owner ctx passed through every owner-scoped service call. */
|
||||
export interface ImportOwnerContext {
|
||||
readonly ownerId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stored job record on the `ImportStoreAdapter`. Distinct from
|
||||
* `ImportProgress`: rows are summarised, not full per-row errors, so
|
||||
* the store can be wired to durable backends without write amplification.
|
||||
*/
|
||||
export interface ImportJobRecord {
|
||||
readonly id: string;
|
||||
readonly ownerId: string;
|
||||
readonly status: ImportJobStatus;
|
||||
readonly totalRows: number;
|
||||
readonly processedRows: number;
|
||||
readonly successCount: number;
|
||||
readonly errorCount: number;
|
||||
readonly errors: ReadonlyArray<{ row: number; message: string }>;
|
||||
readonly metadata?: Record<string, unknown>;
|
||||
readonly createdAt: string;
|
||||
readonly updatedAt: string;
|
||||
readonly completedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Event topics emitted by the canonical `ImportService`. Each topic
|
||||
* carries a single payload (compatible with `@suaveplan/events`'
|
||||
* `EventBus<TMap>` where `TMap extends Record<string, unknown>`).
|
||||
*/
|
||||
export interface ImportEventMap {
|
||||
"import.started": ImportProgress;
|
||||
"import.progress": ImportProgress;
|
||||
"import.completed": ImportProgress;
|
||||
"import.failed": ImportProgress;
|
||||
"import.row_error": RowError;
|
||||
"import:started": ImportProgress;
|
||||
"import:row:processed": ImportRowProcessedEvent;
|
||||
"import:completed": ImportProgress;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** Payload for the `import:row:processed` topic. */
|
||||
export interface ImportRowProcessedEvent {
|
||||
readonly jobId: string;
|
||||
readonly rowIndex: number;
|
||||
readonly success: boolean;
|
||||
}
|
||||
|
||||
/** Constants module — kept on the types file for downstream parity. */
|
||||
export const VALID_FORMATS: readonly ImportFormat[] = ["csv", "json", "xlsx"];
|
||||
|
||||
/** Default batch size for progress emit cadence. */
|
||||
export const DEFAULT_BATCH_SIZE = 100;
|
||||
@@ -0,0 +1,52 @@
|
||||
-- @suaveplan/api-plugin-import — SurrealDB schema
|
||||
--
|
||||
-- Backing table for `SurrealImportStore` (an `ImportStoreAdapter`
|
||||
-- implementation that ships alongside `MemoryImportStore`). The host
|
||||
-- application is responsible for applying this migration — either via
|
||||
-- `@suaveplan/api-plugin-database`'s migration runner pointed at this
|
||||
-- file, or by reading the file and calling `adapter.query(surql)` in
|
||||
-- `beforeAll`. The `SurrealImportStore` itself does not apply
|
||||
-- migrations.
|
||||
--
|
||||
-- Columns are snake_case to match the convention established by the
|
||||
-- shipped `event.surql` schema (`packages/api/api-plugin-events/surql/`).
|
||||
-- The mapping between snake_case columns and the camelCase
|
||||
-- `ImportJobRecord` shape is fully encapsulated inside the store.
|
||||
|
||||
DEFINE TABLE OVERWRITE import_job SCHEMAFULL;
|
||||
|
||||
DEFINE FIELD OVERWRITE owner_id ON import_job TYPE string
|
||||
ASSERT $value != NONE AND string::len(string::trim($value)) > 0;
|
||||
|
||||
DEFINE FIELD OVERWRITE status ON import_job TYPE string
|
||||
ASSERT $value INSIDE ['created', 'validating', 'importing', 'completed', 'failed']
|
||||
DEFAULT 'created';
|
||||
|
||||
DEFINE FIELD OVERWRITE total_rows ON import_job TYPE int
|
||||
ASSERT $value != NONE AND $value >= 0 DEFAULT 0;
|
||||
DEFINE FIELD OVERWRITE processed_rows ON import_job TYPE int
|
||||
ASSERT $value != NONE AND $value >= 0 DEFAULT 0;
|
||||
DEFINE FIELD OVERWRITE success_count ON import_job TYPE int
|
||||
ASSERT $value != NONE AND $value >= 0 DEFAULT 0;
|
||||
DEFINE FIELD OVERWRITE error_count ON import_job TYPE int
|
||||
ASSERT $value != NONE AND $value >= 0 DEFAULT 0;
|
||||
|
||||
DEFINE FIELD OVERWRITE errors ON import_job TYPE array DEFAULT [];
|
||||
DEFINE FIELD OVERWRITE errors.* ON import_job TYPE object;
|
||||
DEFINE FIELD OVERWRITE errors.*.row ON import_job TYPE int;
|
||||
DEFINE FIELD OVERWRITE errors.*.message ON import_job TYPE string;
|
||||
|
||||
DEFINE FIELD OVERWRITE metadata ON import_job TYPE option<object>;
|
||||
|
||||
DEFINE FIELD OVERWRITE created_at ON import_job TYPE datetime
|
||||
ASSERT $value != NONE;
|
||||
DEFINE FIELD OVERWRITE updated_at ON import_job TYPE datetime
|
||||
ASSERT $value != NONE;
|
||||
DEFINE FIELD OVERWRITE completed_at ON import_job TYPE option<datetime>;
|
||||
|
||||
-- Owner-scoped lookup index (used by `list({ ownerId })`).
|
||||
DEFINE INDEX OVERWRITE import_job_owner_idx ON import_job FIELDS owner_id;
|
||||
-- Status-filter index (used by `list({ status })`).
|
||||
DEFINE INDEX OVERWRITE import_job_status_idx ON import_job FIELDS status;
|
||||
-- Created-at ordering index (LIST ORDER BY created_at DESC).
|
||||
DEFINE INDEX OVERWRITE import_job_created_idx ON import_job FIELDS created_at;
|
||||
@@ -0,0 +1,137 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { SurrealAdapter } from "@suaveplan/surrealdb";
|
||||
import {
|
||||
afterAll,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { SurrealDBFixture } from "@suaveplan/testing-integration-surrealdb";
|
||||
import { SurrealImportStore } from "../../src/store/surreal/surreal-import-store.js";
|
||||
import type { ImportJobRecord } from "../../src/types/types.js";
|
||||
|
||||
let fixture: SurrealDBFixture | null = null;
|
||||
let adapter: SurrealAdapter | null = null;
|
||||
|
||||
function makeRecord(
|
||||
id: string,
|
||||
overrides: Partial<ImportJobRecord> = {}
|
||||
): ImportJobRecord {
|
||||
const now = new Date().toISOString();
|
||||
return {
|
||||
id,
|
||||
ownerId: "alice",
|
||||
status: "created",
|
||||
totalRows: 0,
|
||||
processedRows: 0,
|
||||
successCount: 0,
|
||||
errorCount: 0,
|
||||
errors: [],
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
fixture = new SurrealDBFixture();
|
||||
const cfg = await fixture.start();
|
||||
adapter = new SurrealAdapter({
|
||||
url: cfg.url,
|
||||
namespace: cfg.namespace,
|
||||
database: cfg.database,
|
||||
username: cfg.username,
|
||||
password: cfg.password,
|
||||
});
|
||||
await adapter.connect();
|
||||
const surql = await readFile(
|
||||
join(__dirname, "..", "..", "surql", "tables", "import_job.surql"),
|
||||
"utf8"
|
||||
);
|
||||
await adapter.query(surql);
|
||||
}, 180_000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (adapter !== null) {
|
||||
await adapter.disconnect();
|
||||
adapter = null;
|
||||
}
|
||||
if (fixture !== null) {
|
||||
await fixture.stop();
|
||||
fixture = null;
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
describe("@suaveplan/api-plugin-import — SurrealImportStore integration", () => {
|
||||
it("round-trips a record through the real SurrealDB instance", {
|
||||
timeout: 180_000,
|
||||
}, async () => {
|
||||
if (adapter === null) throw new Error("fixture not ready");
|
||||
const store = new SurrealImportStore({ db: adapter });
|
||||
|
||||
const record = makeRecord("rt1", {
|
||||
metadata: { source: "integration" },
|
||||
errors: [{ row: 1, message: "preflight" }],
|
||||
});
|
||||
await store.create(record);
|
||||
|
||||
const fetched = await store.get("rt1");
|
||||
expect(fetched).toBeDefined();
|
||||
expect(fetched?.ownerId).toBe("alice");
|
||||
expect(fetched?.metadata).toEqual({ source: "integration" });
|
||||
expect(fetched?.errors).toEqual([{ row: 1, message: "preflight" }]);
|
||||
|
||||
await store.update("rt1", {
|
||||
status: "completed",
|
||||
successCount: 1,
|
||||
completedAt: new Date().toISOString(),
|
||||
});
|
||||
const updated = await store.get("rt1");
|
||||
expect(updated?.status).toBe("completed");
|
||||
expect(updated?.successCount).toBe(1);
|
||||
expect(updated?.completedAt).toBeDefined();
|
||||
|
||||
const owned = await store.list({ ownerId: "alice" });
|
||||
expect(owned.some((r) => r.id === "rt1")).toBe(true);
|
||||
|
||||
const completed = await store.list({ status: "completed", limit: 5 });
|
||||
expect(completed.some((r) => r.id === "rt1")).toBe(true);
|
||||
|
||||
expect(await store.delete("rt1")).toBe(true);
|
||||
expect(await store.get("rt1")).toBeUndefined();
|
||||
expect(await store.delete("rt1")).toBe(false);
|
||||
});
|
||||
|
||||
it("update on missing id surfaces the canonical message against a real instance", {
|
||||
timeout: 60_000,
|
||||
}, async () => {
|
||||
if (adapter === null) throw new Error("fixture not ready");
|
||||
const store = new SurrealImportStore({ db: adapter });
|
||||
await expect(
|
||||
store.update("never-existed", { status: "completed" })
|
||||
).rejects.toThrow(/Job "never-existed" not found/);
|
||||
});
|
||||
|
||||
it("migration enables canonical writes and rejects invalid status", {
|
||||
timeout: 60_000,
|
||||
}, async () => {
|
||||
if (adapter === null) throw new Error("fixture not ready");
|
||||
const store = new SurrealImportStore({ db: adapter });
|
||||
// Canonical write succeeds.
|
||||
await store.create(makeRecord("mig-canon"));
|
||||
expect((await store.get("mig-canon"))?.status).toBe("created");
|
||||
|
||||
// Direct SurQL with a bogus status MUST be rejected by the
|
||||
// SCHEMAFULL constraint — this proves the migration is in
|
||||
// effect.
|
||||
await expect(
|
||||
adapter.query(
|
||||
`CREATE import_job:mig-bogus SET owner_id = 'alice', status = 'bogus', total_rows = 0, processed_rows = 0, success_count = 0, error_count = 0, errors = [], created_at = time::now(), updated_at = time::now()`
|
||||
)
|
||||
).rejects.toBeDefined();
|
||||
|
||||
await store.delete("mig-canon");
|
||||
});
|
||||
});
|
||||
12
packages/api-stack/api/api-plugin-import/tsconfig.json
Normal file
12
packages/api-stack/api/api-plugin-import/tsconfig.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": ".",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts", "tests/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
12
packages/api-stack/api/api-plugin-import/tsconfig.lib.json
Normal file
12
packages/api-stack/api/api-plugin-import/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
37
packages/api-stack/api/api-plugin-import/vite.config.ts
Normal file
37
packages/api-stack/api/api-plugin-import/vite.config.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/api-plugin-import",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
/^@suaveplan\//,
|
||||
/^@opentelemetry\//,
|
||||
/^@hono\//,
|
||||
"hono",
|
||||
"tslib",
|
||||
"zod",
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
14
packages/api-stack/api/api-plugin-import/vitest.config.ts
Normal file
14
packages/api-stack/api/api-plugin-import/vitest.config.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { createTestConfig } from "@suaveplan/testing/config";
|
||||
|
||||
export default createTestConfig({
|
||||
testTimeout: 10_000,
|
||||
// Integration tests live under `tests/integration/**` and require a
|
||||
// real SurrealDB testcontainer; they ship behind the separate
|
||||
// `test:integration` script (`vitest.integration.config.ts`) so the
|
||||
// standard pre-publish gate does not require Docker.
|
||||
additionalExcludes: ["tests/integration/**"],
|
||||
// Coverage measures only the in-process unit suite. The integration
|
||||
// round-trip exercises the same code paths through a real database
|
||||
// and would double-count branches on the SurrealImportStore.
|
||||
additionalCoverageExcludes: ["tests/**"],
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
/**
|
||||
* Integration test config — exercises `SurrealImportStore` against a
|
||||
* real SurrealDB testcontainer via `@suaveplan/testing-integration-surrealdb`.
|
||||
*
|
||||
* Run with `bun run test:integration` (Docker required). The standard
|
||||
* `test` / `test:coverage` scripts use `vitest.config.ts` and exclude
|
||||
* `tests/integration/**` so the pre-publish coverage gate stays
|
||||
* runnable without Docker.
|
||||
*/
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
include: ["tests/integration/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
testTimeout: 60_000,
|
||||
hookTimeout: 180_000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user