Initial Commit
This commit is contained in:
101
packages/api-stack/api/README.md
Normal file
101
packages/api-stack/api/README.md
Normal file
@@ -0,0 +1,101 @@
|
||||
# `api`
|
||||
|
||||
API orchestration, plugin lifecycle, routing, workers, and tenant resolution.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Packages in `api`
|
||||
|
||||
89 packages.
|
||||
|
||||
| Package | Description |
|
||||
|---|---|
|
||||
| [`@suaveplan/api`](./api/README.md) | The thin Hono-based application orchestrator for the Suaveplan API stack. Composes the four sibling packages — `@suaveplan/api-plugin`, `@suaveplan/api-tenant`… |
|
||||
| [`@suaveplan/api-2pc`](./api-2pc/README.md) | Two-phase commit coordinator for the Suaveplan API plugin system. Drives Postgres PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK PREPARED across registered p… |
|
||||
| [`@suaveplan/api-audit-file`](./api-audit-file/README.md) | File-rotating, HMAC-signed audit sink for `@suaveplan/api-host`. Implements `RequestAuditSink` from `@suaveplan/api-host/request-pipeline`: writes one JSON lin… |
|
||||
| [`@suaveplan/api-audit-stdout`](./api-audit-stdout/README.md) | Stdout audit sink for `@suaveplan/api-host`. Writes JSON-per-line audit records to a `LineWriter` (default `process.stdout`) with an HMAC-SHA-256 signature app… |
|
||||
| [`@suaveplan/api-auth`](./api-auth/README.md) | **$2** for the Suaveplan API plugin system. Defines the abstract session-resolver shape (`Request → Principal \| null`) used by the host on every request, plus… |
|
||||
| [`@suaveplan/api-auth-anonymous`](./api-auth-anonymous/README.md) | Stage-4 trivial session resolver for `@suaveplan/api`. Always returns `null` for every request. Use this as the tail of resolver chains for public endpoints, a… |
|
||||
| [`@suaveplan/api-auth-bearer-static`](./api-auth-bearer-static/README.md) | Stage-4 service-to-service `SessionResolver` adapter for the `@suaveplan/api` plugin system. Matches an inbound `Authorization: Bearer <token>` header against… |
|
||||
| [`@suaveplan/api-auth-better-auth`](./api-auth-better-auth/README.md) | Stage-6 Better-Auth adapter for `@suaveplan/api`. Ships two **independently-wireable** capability surfaces — `createBetterAuthSession` (implements `auth.sessio… |
|
||||
| [`@suaveplan/api-auth-jwt`](./api-auth-jwt/README.md) | Stage-4 concrete `SessionResolver` for the `@suaveplan/api` plugin system. Verifies a JWT presented on the `Authorization: Bearer <jwt>` header (or a configure… |
|
||||
| [`@suaveplan/api-auth-provider`](./api-auth-provider/README.md) | Abstract account-lifecycle contract for the Suaveplan API plugin system (contracts). This package is OPTIONAL: hosted-IdP deployments (Clerk, Auth0, Cognito) i… |
|
||||
| [`@suaveplan/api-auth-rolled`](./api-auth-rolled/README.md) | Stage-8.12 rolled session-token resolver for `@suaveplan/api`. Issues RS256-signed JWTs with rotating `kid` slots, verifies tokens through the rotator's active… |
|
||||
| [`@suaveplan/api-auth-static-principal`](./api-auth-static-principal/README.md) | Fixed-`Principal` session resolver for the `@suaveplan/api` plugin system. Returns the configured `Principal` verbatim on every request — no token decoding, no… |
|
||||
| [`@suaveplan/api-db-adapter`](./api-db-adapter/README.md) | Stage-4 bridge from the `@suaveplan/db` engine adapters to the plugin `AdminDatabaseLike` / `TenantScopedDatabaseLike` surfaces exposed by `@suaveplan/api-type… |
|
||||
| [`@suaveplan/api-db-sqlite-memory`](./api-db-sqlite-memory/README.md) | Stage-5 in-memory SQLite database adapter for the `@suaveplan/api` plugin host testing kit. Wraps `bun:sqlite` (in `:memory:` mode by default, or a file path f… |
|
||||
| [`@suaveplan/api-db-worker-proxy`](./api-db-worker-proxy/README.md) | Worker-side proxy for the api `db` capability. Marshals `AdminDatabaseLike` and `TenantScopedDatabaseLike` calls — including transactions and cursor streams —… |
|
||||
| [`@suaveplan/api-events-in-memory`](./api-events-in-memory/README.md) | In-memory events bus adapter for the `@suaveplan/api` testing kit. Wraps `@suaveplan/events`' production `EventBus` with a per-topic recording layer so tests c… |
|
||||
| [`@suaveplan/api-events-worker-proxy`](./api-events-worker-proxy/README.md) | Worker-tier proxy over a main-process `@suaveplan/events` `EventBus`. |
|
||||
| [`@suaveplan/api-graphql`](./api-graphql/README.md) | Stage-1 abstract GraphQL contract for the Suaveplan API plugin system. |
|
||||
| [`@suaveplan/api-graphql-apollo`](./api-graphql-apollo/README.md) | Stage-7 Apollo Server GraphQL engine adapter for the Suaveplan API plugin system. Implements the `GraphqlEngine` contract from `@suaveplan/api-graphql` and pas… |
|
||||
| [`@suaveplan/api-graphql-ui-altair`](./api-graphql-ui-altair/README.md) | Altair GraphQL UI plugin for the Suaveplan API plugin system. Structural sibling of `@suaveplan/api-graphql-ui-graphiql`. |
|
||||
| [`@suaveplan/api-graphql-ui-graphiql`](./api-graphql-ui-graphiql/README.md) | GraphiQL UI mount for the Suaveplan API plugin system. Stage 7 of the umbrella plan. |
|
||||
| [`@suaveplan/api-graphql-yoga`](./api-graphql-yoga/README.md) | Stage-7 GraphQL Yoga engine adapter. **Default GraphQL engine** for the Suaveplan API plugin system per design doc §10.5 — Bun-friendly, fastest for our usage… |
|
||||
| [`@suaveplan/api-host`](./api-host/README.md) | Host orchestration core for the Suaveplan API plugin system. Provides the stable error-code registry (HOST/PLUGIN/RELOAD/CAPABILITY/ADMIN/LEDGER), the author-facing definePlugin DX, the runtime CapabilityRegistry, the LifecycleDriver + ReloadDriver + PluginLedger + admin Hono sub-router, the ErrorBoundary, the §6 11-step RequestPipeline, the top-level createHost(config) entry point composing every internal driver, and the HostInstruments registry covering the §12.3 metric surface. The ./testing subpath is reserved for test utilities. |
|
||||
| [`@suaveplan/api-jobs-worker-proxy`](./api-jobs-worker-proxy/README.md) | Bidirectional `MessagePort` proxy that lets a worker-isolated API plugin participate in the host's main-process job scheduler. The worker calls `registerCron`… |
|
||||
| [`@suaveplan/api-lifecycle`](./api-lifecycle/README.md) | Stage-1 contract package for the Suaveplan API plugin system: the plugin-lifecycle finite state machine definition, the legal-transition table, the side-effect… |
|
||||
| [`@suaveplan/api-loader`](./api-loader/README.md) | Stage-1 contract package for the Suaveplan API plugin system. |
|
||||
| [`@suaveplan/api-logger-worker-proxy`](./api-logger-worker-proxy/README.md) | Stage-3 worker → main proxy for `@suaveplan/logger`. Lets a worker-isolated API plugin call a normal `LoggerLike` surface while the records cross a `MessagePor… |
|
||||
| [`@suaveplan/api-manifest`](./api-manifest/README.md) | Stage-1 contract package of the Suaveplan API plugin system: Zod schemas, parser, cross-field validators, dependency-graph resolver, capability resolver, and c… |
|
||||
| [`@suaveplan/api-migrations`](./api-migrations/README.md) | Database-agnostic migration runner for the Suaveplan API. Ships a narrow `MigrationAdapter` surface (six methods), three shipped adapters (`surrealql`, `postgr… |
|
||||
| [`@suaveplan/api-openapi`](./api-openapi/README.md) | OpenAPI 3.1 registry, typed `RouteBuilder`, schema registry, and document merger for the Suaveplan API. Single source of truth that wires into Hono via `OpenAP… |
|
||||
| [`@suaveplan/api-openapi-ui-redoc`](./api-openapi-ui-redoc/README.md) | Stage-8 ReDoc UI adapter for the Suaveplan api-host. Mounts the ReDoc HTML shell at `/docs/ui/redoc` (or any caller-chosen path) via `createRedocUiHandler` and… |
|
||||
| [`@suaveplan/api-openapi-ui-scalar`](./api-openapi-ui-scalar/README.md) | Scalar UI adapter for the Suaveplan OpenAPI plugin system. |
|
||||
| [`@suaveplan/api-openapi-ui-swagger`](./api-openapi-ui-swagger/README.md) | Swagger UI adapter for the `/docs/ui/swagger` endpoint of the Suaveplan API plugin system (Stage 8.8). Returns a pure HTTP handler whose response is a `text/ht… |
|
||||
| [`@suaveplan/api-plugin`](./api-plugin/README.md) | Plugin-system foundation for the Suaveplan API. Universal — runs unchanged in Bun, Node, and the browser. |
|
||||
| [`@suaveplan/api-plugin-audit`](./api-plugin-audit/README.md) | Hono plugin that wraps `@suaveplan/audit-logging` as a write-through audit trail for `@suaveplan/api`. Installs a global per-request audit middleware (method /… |
|
||||
| [`@suaveplan/api-plugin-auth`](./api-plugin-auth/README.md) | Better-Auth-first authentication plugin for `@suaveplan/api`. The canonical exemplar of the 3-way plug-and-play contract every `@suaveplan/api-plugin-*` packag… |
|
||||
| [`@suaveplan/api-plugin-campaigns`](./api-plugin-campaigns/README.md) | Campaigns + suppression + delivery-webhooks HTTP plugin for `@suaveplan/api`. Mounts the `/campaigns`, `/suppression`, and `/webhooks` route trees on a Hono `O… |
|
||||
| [`@suaveplan/api-plugin-cms`](./api-plugin-cms/README.md) | Headless content-management primitives for `@suaveplan/api`. Ships an in-memory `CmsEngine` that manages flexible content models, content entries with a `draft… |
|
||||
| [`@suaveplan/api-plugin-database`](./api-plugin-database/README.md) | SurrealDB-backed database plugin for `@suaveplan/api`. Pools `@suaveplan/surrealdb`'s `SurrealAdapter` (implementing `@suaveplan/db`'s `DatabaseAdapter`), runs… |
|
||||
| [`@suaveplan/api-plugin-event-bus`](./api-plugin-event-bus/README.md) | Typed event-bus plugin for `@suaveplan/api`. Wraps `@suaveplan/events` (pub/sub, async dispatch, interceptors, bounded-queue backpressure, idempotent listener… |
|
||||
| [`@suaveplan/api-plugin-events`](./api-plugin-events/README.md) | Event-management plugin for `@suaveplan/api`. Ships a SurrealDB-backed `EventRepository` + `RsvpRepository` pair, ten Hono OpenAPI routes (list / get / create… |
|
||||
| [`@suaveplan/api-plugin-example`](./api-plugin-example/README.md) | Reference plugin for the `@suaveplan/api-host` plugin system. The package is a load-bearing integration target, not a tutorial: every contract surface the host… |
|
||||
| [`@suaveplan/api-plugin-export`](./api-plugin-export/README.md) | Data-export plugin for `@suaveplan/api`. Turns any registered resource into a CSV (RFC 4180), JSON, or XLSX (SpreadsheetML XML) blob through three self-contain… |
|
||||
| [`@suaveplan/api-plugin-file-upload`](./api-plugin-file-upload/README.md) | Multipart and chunked file-upload plugin for `@suaveplan/api`. Ships six OpenAPI-typed Hono routes, a pluggable storage + session-store contract, SHA-256 hash… |
|
||||
| [`@suaveplan/api-plugin-graphql`](./api-plugin-graphql/README.md) | GraphQL description + resolver-composition layer for `@suaveplan/api`. Server-tier — runs in Bun and Node. |
|
||||
| [`@suaveplan/api-plugin-import`](./api-plugin-import/README.md) | Tabular-data ingest plugin for `@suaveplan/api`. Parses CSV / JSON / TSV-XLSX payloads, applies declarative column mappings + per-field validation + computed f… |
|
||||
| [`@suaveplan/api-plugin-notifications`](./api-plugin-notifications/README.md) | SurrealDB-backed notifications + push plugin for `@suaveplan/api`. Ships a `NotificationRepository` + `DeviceTokenRepository` pair, an `ApnService` with a plug… |
|
||||
| [`@suaveplan/api-plugin-payment`](./api-plugin-payment/README.md) | Payment-lifecycle plugin for `@suaveplan/api`. Wraps `@suaveplan/payment` + a caller-injected `PaymentProvider` (Stripe / Adyen / PayPal / Square / Mollie / cu… |
|
||||
| [`@suaveplan/api-plugin-rate-limit`](./api-plugin-rate-limit/README.md) | Rate-limit plugin for `@suaveplan/api`. Wraps `@suaveplan/rate-limiter` as a `definePlugin` descriptor exposing per-route + per-tenant configuration, parameter… |
|
||||
| [`@suaveplan/api-plugin-search`](./api-plugin-search/README.md) | Full-text search plugin for `@suaveplan/api`. Brings TF-weighted relevance scoring, four built-in text analyzers (`standard`, `simple`, `whitespace`, `keyword`… |
|
||||
| [`@suaveplan/api-plugin-site`](./api-plugin-site/README.md) | Multi-site / multi-domain management plugin for `@suaveplan/api`. Stores `site` records (name, slug, theme, settings, push config) in SurrealDB, resolves incom… |
|
||||
| [`@suaveplan/api-plugin-sse`](./api-plugin-sse/README.md) | Server-Sent Events plugin for `@suaveplan/api`. Registers a typed `sseService` in the host `ServiceRegistry`, mounts a Hono streaming handler at `GET /sse/:cha… |
|
||||
| [`@suaveplan/api-plugin-tenant`](./api-plugin-tenant/README.md) | Multi-tenancy plugin for `@suaveplan/api`. Wraps `@suaveplan/api-tenant`'s resolver chain (header / subdomain / domain / path / JWT / query / cookie) as a firs… |
|
||||
| [`@suaveplan/api-plugin-user`](./api-plugin-user/README.md) | User-profile CRUD plugin for `@suaveplan/api`. Mounts the canonical `/users/me` GET/PATCH/DELETE surface on top of `@suaveplan/api-plugin-auth`, registers a ty… |
|
||||
| [`@suaveplan/api-plugin-vault`](./api-plugin-vault/README.md) | Per-user vault HTTP plugin for `@suaveplan/api`. Third instance of the 3-way plug-and-play contract every `@suaveplan/api-plugin-*` package follows: the plugin… |
|
||||
| [`@suaveplan/api-plugin-webhook`](./api-plugin-webhook/README.md) | Signed outbound webhook delivery plugin for `@suaveplan/api`. Wraps `@suaveplan/webhook-client` behind a `definePlugin` descriptor, registers a typed `webhookS… |
|
||||
| [`@suaveplan/api-rate-limit`](./api-rate-limit/README.md) | Per-route / per-principal / per-tenant / per-IP token-bucket rate-limit middleware for `@suaveplan/api`. Implements the `RequestRateLimiter` contract consumed… |
|
||||
| [`@suaveplan/api-rbac`](./api-rbac/README.md) | Scope + rule-based authorisation middleware for the Suaveplan API plugin system. Produces a framework-agnostic `Middleware` that consumes the resolved `Princip… |
|
||||
| [`@suaveplan/api-router`](./api-router/README.md) | Stage-1 contract package for the Suaveplan API plugin system. |
|
||||
| [`@suaveplan/api-router-hono`](./api-router-hono/README.md) | Stage-4 concrete Hono router adapter for `@suaveplan/api`. Implements the abstract `Router` contract from `@suaveplan/api-router` on top of a Hono `Hono` insta… |
|
||||
| [`@suaveplan/api-router-in-memory`](./api-router-in-memory/README.md) | Stage-5 in-memory router adapter for `@suaveplan/api`. Implements both the abstract `Router` contract from `@suaveplan/api-router` AND the `PipelineRouter` con… |
|
||||
| [`@suaveplan/api-router-worker-bridge`](./api-router-worker-bridge/README.md) | Stage-3 HTTP bridge that marshals Fetch `Request` and `Response` objects across the worker boundary. Pairs a main-side `dispatch` with a worker- side handler o… |
|
||||
| [`@suaveplan/api-saga`](./api-saga/README.md) | Saga orchestrator for the Suaveplan API plugin system. Drives manifest-declared multi-plugin workflows in declared order with reverse-order compensation on fai… |
|
||||
| [`@suaveplan/api-sandbox-permissions`](./api-sandbox-permissions/README.md) | Permission model for sandboxed-tier API plugins. Centralises the `permissions.fs[]` / `permissions.fetch[]` allowlists, the `permissions.budgets` enforcement,… |
|
||||
| [`@suaveplan/api-sandbox-vm`](./api-sandbox-vm/README.md) | `vm.Context`-based sandbox host for Suaveplan API plugins running in the sandboxed tier. The package owns the `vm.Context`+frozen-globals half of the design do… |
|
||||
| [`@suaveplan/api-sandbox-vm-host-bridge`](./api-sandbox-vm-host-bridge/README.md) | Host-integration bridge that lets a `VmSandboxHandle` from `@suaveplan/api-sandbox-vm` install into `@suaveplan/api-host` as a first-class plugin. |
|
||||
| [`@suaveplan/api-sandbox-wasm`](./api-sandbox-wasm/README.md) | Stage-8 WebAssembly sandbox host. Instantiates a sandboxed-tier plugin WASM module against a WASI snapshot preview1 implementation derived from the plugin's `P… |
|
||||
| [`@suaveplan/api-sandbox-wasm-host-bridge`](./api-sandbox-wasm-host-bridge/README.md) | Host-integration layer that wraps a `@suaveplan/api-sandbox-wasm` `WasmSandboxHandle` as a `@suaveplan/api-host` `PluginModule`. Sandboxed-tier plugins compile… |
|
||||
| [`@suaveplan/api-schemas`](./api-schemas/README.md) | Shared Zod schemas describing Suaveplan API request/response shapes. The single source of truth for server plugins, React data hooks, and generated OpenAPI doc… |
|
||||
| [`@suaveplan/api-secrets`](./api-secrets/README.md) | Abstract Secrets contract for the Suaveplan API plugin system — per-plugin secret access typed against `manifest.secrets[]`, canonical redaction marker, frozen… |
|
||||
| [`@suaveplan/api-secrets-bun`](./api-secrets-bun/README.md) | Stage-8 `SecretsLike` adapter backed by `Bun.secrets` (macOS Keychain, Linux libsecret, Windows Credential Manager) with per-plugin allow-list enforcement, OS-… |
|
||||
| [`@suaveplan/api-secrets-env`](./api-secrets-env/README.md) | Stage-4 `SecretsLike` adapter backed by `process.env` / `Bun.env` with per-plugin allowed-keys enforcement. |
|
||||
| [`@suaveplan/api-secrets-vault`](./api-secrets-vault/README.md) | Stage-8 `SecretsLike` adapter backed by `@suaveplan/vault`. The vault is passed in already unlocked by the host's startup sequence; the adapter never sees the… |
|
||||
| [`@suaveplan/api-secrets-worker-proxy`](./api-secrets-worker-proxy/README.md) | Stage-3 worker → main `SecretsLike` proxy for the API plugin system. |
|
||||
| [`@suaveplan/api-sse`](./api-sse/README.md) | Abstract Server-Sent Events contracts for `@suaveplan/api`. Stage-1 contract package: defines the engine-side `SseConnection`, `SseBroker`, `ReplayBuffer` (Las… |
|
||||
| [`@suaveplan/api-sse-bun`](./api-sse-bun/README.md) | `Bun.serve`-backed Server-Sent Events engine for the `@suaveplan/api` plugin system. Implements the abstract `@suaveplan/api-sse` contract and passes `runSseEn… |
|
||||
| [`@suaveplan/api-tenant`](./api-tenant/README.md) | Multi-tenant resolution + scoping for the Suaveplan API. Server-tier — runs on Bun and Node, depends on `hono` and `node:async_hooks`. |
|
||||
| [`@suaveplan/api-tenant-provider`](./api-tenant-provider/README.md) | Abstract `TenantProvider` contract for the Suaveplan API plugin system. Enumerates the universe of tenants known to a host so per-tenant cron / queued jobs (ma… |
|
||||
| [`@suaveplan/api-tenant-provider-memory`](./api-tenant-provider-memory/README.md) | In-memory reference implementation of the `@suaveplan/api-tenant-provider` `TenantProvider` contract. Backed by an insertion-ordered `Map<TenantId, TenantRecor… |
|
||||
| [`@suaveplan/api-tenant-resolver-header`](./api-tenant-resolver-header/README.md) | Stage-8 tenant resolver. Implements the `TenantResolver` contract from `@suaveplan/api-types/tenant-resolver` by reading the tenant identifier from a configura… |
|
||||
| [`@suaveplan/api-tenant-resolver-jwt`](./api-tenant-resolver-jwt/README.md) | Stage-4 tenant resolver for the Suaveplan API plugin system. Reads the already-resolved `Principal.tenantId` (attached by an earlier `api-auth-*` chain step) a… |
|
||||
| [`@suaveplan/api-tenant-resolver-subdomain`](./api-tenant-resolver-subdomain/README.md) | Stage-8 tenant resolver for the Suaveplan API plugin system. Reads the inbound `Host` header (and optionally a CIDR-trusted `X-Forwarded-Host`), strips a confi… |
|
||||
| [`@suaveplan/api-types`](./api-types/README.md) | Root contract-types package for the Suaveplan API plugin system. Defines every cross-cutting type the foundation packages (`api-manifest`, `api-router`, `api-h… |
|
||||
| [`@suaveplan/api-worker-host`](./api-worker-host/README.md) | Worker-side runtime that hosts exactly one Suaveplan API plugin inside an isolated `Worker` thread. The main-process counterpart is `@suaveplan/api-host`; this… |
|
||||
| [`@suaveplan/api-worker-rpc`](./api-worker-rpc/README.md) | MessagePort-based RPC protocol for worker-isolated API plugins. Provides the shared protocol — Zod-validated wire frames, monotonic correlation ids, AbortSigna… |
|
||||
| [`@suaveplan/api-ws`](./api-ws/README.md) | Abstract WebSocket engine contract for the Suaveplan API foundation — Connection, ConnectionRegistry, SubscriptionBus, WsBroker, reconnect-token envelope, backpressure + heartbeat policies, cross-tenant capability gate, and a runWsEngineConformance harness. Stage-1 contracts only; concrete engines (api-ws-bun) and brokers (api-ws-broker-redis / -in-memory) plug behind these interfaces. |
|
||||
| [`@suaveplan/api-ws-broker-in-memory`](./api-ws-broker-in-memory/README.md) | Single-host in-process implementation of the `@suaveplan/api-ws` `WsBroker` contract. Suitable for production single-host deployments AND as the default broker… |
|
||||
| [`@suaveplan/api-ws-broker-redis`](./api-ws-broker-redis/README.md) | Cross-host Redis pub/sub broker for `@suaveplan/api-ws` and `@suaveplan/api-sse`. |
|
||||
| [`@suaveplan/api-ws-bun`](./api-ws-bun/README.md) | Bun.serve-backed WebSocket engine for the `@suaveplan/api-ws` Stage-1 abstract contract. The engine composes the per-connection handshake pipeline, registry, s… |
|
||||
<!-- end:auto-docs -->
|
||||
1
packages/api-stack/api/_intro.md
Normal file
1
packages/api-stack/api/_intro.md
Normal file
@@ -0,0 +1 @@
|
||||
API orchestration, plugin lifecycle, routing, workers, and tenant resolution.
|
||||
109
packages/api-stack/api/api-2pc/README.md
Normal file
109
packages/api-stack/api/api-2pc/README.md
Normal file
@@ -0,0 +1,109 @@
|
||||
# @suaveplan/api-2pc - Two-phase commit coordinator for the Suaveplan API plugin system.
|
||||
|
||||
Two-phase commit coordinator for the Suaveplan API plugin system. Drives Postgres PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK PREPARED across registered participants with a host-owned ledger in suaveplan_2pc_transactions. Refuses non-preparable engines as writers (SQLite, SurrealDB) but accepts them as read-only voters. Default timeoutMs 5000, hard ceiling 30000, default outstanding-prepared cap 256, all operator-tunable. Ships boot-time orphan resolution via pg_prepared_xacts, the full TWOPC_* error namespace, OTel transaction/prepare/commit/rollback spans plus prepare_duration_ms / commit_duration_ms histograms / transactions_total counter / outstanding_prepared gauge, and run2pcParticipantConformance on the ./testing subpath. Server-tier — Bun + Node.
|
||||
|
||||
> **Postgres-only writer surface.** SQLite and SurrealDB cannot honour
|
||||
> `PREPARE TRANSACTION` and are refused as writers at install time.
|
||||
> They MAY participate as read-only voters that contribute a
|
||||
> `commit`/`abort` vote based on their own consistency check, but the
|
||||
> coordinator never sends them prepare / commit / rollback SQL.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [coordinator](./src/coordinator/coordinator.md) — Top-level orchestration for the 2PC protocol. `createTwoPcCoordinator` builds a fully-wired coordinator that holds a pa…
|
||||
- [coordinator](./src/coordinator/coordinator.md) — Top-level orchestration for the 2PC protocol. `createTwoPcCoordinator` builds a fully-wired coordinator that holds a pa…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — This module owns the `TWOPC_*` error-code namespace and the `TwoPcError` concrete class for the `@suaveplan/api-2pc` pa…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — This module owns the `TWOPC_*` error-code namespace and the `TwoPcError` concrete class for the `@suaveplan/api-2pc` pa…
|
||||
- [instruments](./src/instruments/instruments.md) — Builds the OTel instrument registry the 2PC coordinator emits against on every transaction. Each instrument's name foll…
|
||||
- [instruments](./src/instruments/instruments.md) — Builds the OTel instrument registry the 2PC coordinator emits against on every transaction. Each instrument's name foll…
|
||||
- [outstanding-cap](./src/limits/outstanding-cap.md) — Admission control for in-flight 2PC transactions. The coordinator holds a fixed-size budget of "outstanding prepared tr…
|
||||
- [timeout-ceiling](./src/limits/timeout-ceiling.md) — Enforces the hard 30 000 ms ceiling on caller-supplied `timeoutMs` values. The check fires at the public entry point of…
|
||||
- [options](./src/options/options.md) — Defines the boundary-validated option shape consumed by `createTwoPcCoordinator`. Every tunable that affects coordinato…
|
||||
- [options](./src/options/options.md) — Defines the boundary-validated option shape consumed by `createTwoPcCoordinator`. Every tunable that affects coordinato…
|
||||
- [persistence](./src/persistence/persistence.md) — Persists every coordinator state transition into the `suaveplan_2pc_transactions` ledger table. The table is owned by `…
|
||||
- [persistence](./src/persistence/persistence.md) — Persists every coordinator state transition into the `suaveplan_2pc_transactions` ledger table. The table is owned by `…
|
||||
- [commit](./src/phases/commit.md) — Phase-2 commit for a single participant. Once every participant has returned `commit` from phase 1 and the coordinator…
|
||||
- [gid](./src/phases/gid.md) — Builds and parses the global transaction id (gid) string passed to Postgres' `PREPARE TRANSACTION '<gid>'`. The format…
|
||||
- [prepare](./src/phases/prepare.md) — Implements phase 1 of the 2PC protocol for a single participant. The coordinator iterates participants and calls this m…
|
||||
- [rollback](./src/phases/rollback.md) — Phase-2 rollback for a single participant. Used both by the coordinator's abort path (vote_abort or prepare failure) an…
|
||||
- [recovery](./src/recovery/recovery.md) — Boot-time orphan resolution for the 2PC coordinator. When a host crashes between phase 1 and phase 2, prepared transact…
|
||||
- [recovery](./src/recovery/recovery.md) — Boot-time orphan resolution for the 2PC coordinator. When a host crashes between phase 1 and phase 2, prepared transact…
|
||||
- [registry](./src/registry/participant-registry.md) — The participant registry is the install-time admission control between plugins and the 2PC coordinator. Plugins declare…
|
||||
- [participant-registry](./src/registry/participant-registry.md) — The participant registry is the install-time admission control between plugins and the 2PC coordinator. Plugins declare…
|
||||
- [testing](./src/testing/conformance.md) — Provides `run2pcParticipantConformance(factory)` — a runnable test suite that pins the contract every 2PC writer partic…
|
||||
- [conformance](./src/testing/conformance.md) — Provides `run2pcParticipantConformance(factory)` — a runnable test suite that pins the contract every 2PC writer partic…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Why
|
||||
|
||||
Some cross-database workflows demand atomic visibility — no partial
|
||||
state, no eventual convergence. Per the API Plugin Foundation design
|
||||
doc §9.2 the coordinator runs phase-1 prepare across every registered
|
||||
participant, persists the vote tally to `suaveplan_2pc_transactions`
|
||||
on the host's metadata DB, then issues `COMMIT PREPARED` or
|
||||
`ROLLBACK PREPARED` against every writer participant. Recovery on the
|
||||
next boot resolves any prepared transactions left over from a host
|
||||
crash.
|
||||
|
||||
## Modules
|
||||
|
||||
- `coordinator/` — `createTwoPcCoordinator` + `run2pc` factory and
|
||||
state machine.
|
||||
- `registry/` — `TwoPcParticipantRegistry` (admission control).
|
||||
- `phases/` — `preparePhaseOne`, `commitPhaseTwo`, `rollbackPhaseTwo`,
|
||||
`gid` helpers.
|
||||
- `persistence/` — `TwoPcLedgerStore` over the host-owned
|
||||
`suaveplan_2pc_transactions` table.
|
||||
- `recovery/` — boot-time orphan resolution via `pg_prepared_xacts`.
|
||||
- `limits/` — `OutstandingPreparedCap` (admission), `assertTimeoutBelowCeiling`
|
||||
(per-call ceiling).
|
||||
- `instruments/` — OTel meter/tracer bundle.
|
||||
- `options/` — Zod-validated coordinator options + defaults.
|
||||
- `error-codes/` — 10 `TWOPC_*` codes + `TwoPcError`.
|
||||
- `types/` — `Run2PcInput`, `Run2PcOutcome`, participant + state types.
|
||||
- `testing/` — `run2pcParticipantConformance(factory)` suite.
|
||||
|
||||
## Quick start
|
||||
|
||||
```ts
|
||||
import { createTwoPcCoordinator } from "@suaveplan/api-2pc";
|
||||
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-01" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () => crypto.randomUUID(),
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "billing", input: { invoiceId: "inv-1" } },
|
||||
{ id: "inventory", input: { sku: "abc-9" } },
|
||||
],
|
||||
});
|
||||
// outcome.state is "committed" | "aborted" | "in_doubt"
|
||||
```
|
||||
|
||||
## Defaults
|
||||
|
||||
- `timeoutMs`: **5000** (operator-tunable per-call up to the
|
||||
30000 ms ceiling).
|
||||
- `outstandingPreparedCap`: **256** (operator-tunable via host
|
||||
config).
|
||||
|
||||
## Error namespace
|
||||
|
||||
Every production code path throws `TwoPcError` with one of the ten
|
||||
`TWOPC_*` codes. See `src/error-codes/error-codes.md` for the full
|
||||
catalogue.
|
||||
|
||||
## Conformance
|
||||
|
||||
Every adapter implementing a writer participant MUST run
|
||||
`run2pcParticipantConformance({ create: ... })` in a
|
||||
`tests/conformance/run-2pc-participant.test.ts` file. The harness
|
||||
pins prepare / commit / rollback semantics and the idempotency
|
||||
contract.
|
||||
|
||||
123
packages/api-stack/api/api-2pc/package.json
Normal file
123
packages/api-stack/api/api-2pc/package.json
Normal file
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"name": "@suaveplan/api-2pc",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Two-phase commit coordinator for the Suaveplan API plugin foundation. Drives Postgres PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK PREPARED across registered participants with a host-owned ledger in suaveplan_2pc_transactions. Refuses non-preparable engines as writers (SQLite, SurrealDB) but accepts them as read-only voters. Default timeoutMs 5000, hard ceiling 30000, default outstanding-prepared cap 256, all operator-tunable. Ships boot-time orphan resolution via pg_prepared_xacts, the full TWOPC_* error namespace, OTel transaction/prepare/commit/rollback spans plus prepare_duration_ms / commit_duration_ms histograms / transactions_total counter / outstanding_prepared gauge, and run2pcParticipantConformance on the ./testing subpath. Server-tier — Bun + Node.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./coordinator": {
|
||||
"import": {
|
||||
"types": "./dist/coordinator/coordinator.d.ts",
|
||||
"default": "./dist/coordinator/coordinator.js"
|
||||
}
|
||||
},
|
||||
"./registry": {
|
||||
"import": {
|
||||
"types": "./dist/registry/participant-registry.d.ts",
|
||||
"default": "./dist/registry/participant-registry.js"
|
||||
}
|
||||
},
|
||||
"./phases": {
|
||||
"import": {
|
||||
"types": "./dist/phases/index.d.ts",
|
||||
"default": "./dist/phases/index.js"
|
||||
}
|
||||
},
|
||||
"./persistence": {
|
||||
"import": {
|
||||
"types": "./dist/persistence/persistence.d.ts",
|
||||
"default": "./dist/persistence/persistence.js"
|
||||
}
|
||||
},
|
||||
"./recovery": {
|
||||
"import": {
|
||||
"types": "./dist/recovery/recovery.d.ts",
|
||||
"default": "./dist/recovery/recovery.js"
|
||||
}
|
||||
},
|
||||
"./limits": {
|
||||
"import": {
|
||||
"types": "./dist/limits/index.d.ts",
|
||||
"default": "./dist/limits/index.js"
|
||||
}
|
||||
},
|
||||
"./instruments": {
|
||||
"import": {
|
||||
"types": "./dist/instruments/instruments.d.ts",
|
||||
"default": "./dist/instruments/instruments.js"
|
||||
}
|
||||
},
|
||||
"./error-codes": {
|
||||
"import": {
|
||||
"types": "./dist/error-codes/error-codes.d.ts",
|
||||
"default": "./dist/error-codes/error-codes.js"
|
||||
}
|
||||
},
|
||||
"./options": {
|
||||
"import": {
|
||||
"types": "./dist/options/options.d.ts",
|
||||
"default": "./dist/options/options.js"
|
||||
}
|
||||
},
|
||||
"./testing": {
|
||||
"import": {
|
||||
"types": "./dist/testing/conformance.d.ts",
|
||||
"default": "./dist/testing/conformance.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "bun test --timeout=30000 src/",
|
||||
"test:watch": "bun test --watch src/",
|
||||
"test:coverage": "bun test --coverage --coverage-reporter=lcov --timeout=30000 src/",
|
||||
"test:integration": "vitest run --config vitest.integration.config.ts --passWithNoTests",
|
||||
"lint": "bunx biome check src/ tests/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/ tests/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/ tests/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/ tests/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/api-types": "workspace:^",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@suaveplan/testing-integration": "workspace:^",
|
||||
"@suaveplan/testing-integration-postgres": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"postgres": "catalog:database",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": ["dist", "src", "README.md"]
|
||||
}
|
||||
150
packages/api-stack/api/api-2pc/src/coordinator/coordinator.md
Normal file
150
packages/api-stack/api/api-2pc/src/coordinator/coordinator.md
Normal file
@@ -0,0 +1,150 @@
|
||||
# `coordinator/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Top-level orchestration for the 2PC protocol. `createTwoPcCoordinator`
|
||||
builds a fully-wired coordinator that holds a participant registry, a
|
||||
ledger store, an admission counter, and an OTel instrument bundle.
|
||||
The `run2pc({ participants, timeoutMs? })` entry point owns the full
|
||||
state machine described in design doc §9.2:
|
||||
|
||||
1. Admission via the outstanding-prepared cap.
|
||||
2. Ledger insert at `voting`.
|
||||
3. Phase-1 prepare per participant, with the
|
||||
`2pc.prepare_duration_ms` histogram and a `2pc.prepare` span per
|
||||
participant.
|
||||
4. Either unanimous commit → `markAllVotedCommit` → phase-2
|
||||
`COMMIT PREPARED` per writer; OR any abort/error →
|
||||
`markAborting` → phase-2 `ROLLBACK PREPARED` per prepared writer.
|
||||
5. Terminal ledger stamp (`committed` / `aborted` / `in_doubt`).
|
||||
|
||||
The coordinator is the only path application code ever touches.
|
||||
Every smaller module (phases, persistence, recovery, limits) is
|
||||
internal plumbing this factory composes.
|
||||
|
||||
## Features
|
||||
|
||||
- Single public surface: `createTwoPcCoordinator(deps)` returns
|
||||
`{ run2pc, outstandingCount, options }`.
|
||||
- Default `timeoutMs` from options (5000), per-call override accepted
|
||||
up to the 30000 ceiling (enforced via
|
||||
`limits/timeout-ceiling`).
|
||||
- Admission cap (default 256) enforced via
|
||||
`limits/outstanding-cap`; over-cap calls fail with
|
||||
`TWOPC_OUTSTANDING_LIMIT_EXCEEDED` before any participant is
|
||||
contacted.
|
||||
- Hooks every documented OTel instrument (`2pc.transaction` span,
|
||||
`2pc.prepare` / `2pc.commit` / `2pc.rollback` child spans,
|
||||
`2pc.transactions_total` counter, `prepare`/`commit_duration_ms`
|
||||
histograms, `outstanding_prepared` gauge).
|
||||
- Ledger persistence is bidirectional — every state transition writes
|
||||
through `TwoPcLedgerStore`, so the recovery runner has a full audit
|
||||
trail on restart.
|
||||
- Abort handling: vote_abort, prepare_failed, and timeout all funnel
|
||||
to the rollback branch; rollback errors escalate to `in_doubt`
|
||||
rather than failing the rollback path itself.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { createTwoPcCoordinator } from "@suaveplan/api-2pc/coordinator";
|
||||
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-01" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () => crypto.randomUUID(),
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "billing", input: { invoiceId: "inv-1" } },
|
||||
{ id: "inventory", input: { sku: "abc-9" } },
|
||||
],
|
||||
});
|
||||
console.log(outcome.state); // "committed" | "aborted" | "in_doubt"
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Inject a custom instruments bundle to feed a hand-rolled
|
||||
MeterProvider, override the default timeout per call, and consume the
|
||||
per-participant vote summary on the outcome:
|
||||
|
||||
```ts
|
||||
import { createTwoPcCoordinator } from "@suaveplan/api-2pc/coordinator";
|
||||
import { createTwoPcInstruments } from "@suaveplan/api-2pc/instruments";
|
||||
|
||||
const instruments = createTwoPcInstruments({
|
||||
meterProvider,
|
||||
outstandingPreparedSnapshot: () => coordinator.outstandingCount(),
|
||||
});
|
||||
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: {
|
||||
hostInstanceId: "host-01",
|
||||
defaultTimeoutMs: 10_000,
|
||||
outstandingPreparedCap: 64,
|
||||
},
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () => crypto.randomUUID(),
|
||||
instruments,
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "billing", input: { invoiceId } },
|
||||
{ id: "audit", input: { event: "invoice.created" } },
|
||||
],
|
||||
timeoutMs: 2_500,
|
||||
});
|
||||
for (const v of outcome.votes) {
|
||||
auditLog.append({
|
||||
txId: outcome.txId,
|
||||
participant: v.participantId,
|
||||
vote: v.vote,
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface TwoPcCoordinatorDeps`
|
||||
- `options: TwoPcCoordinatorOptions`
|
||||
- `registry: TwoPcParticipantRegistry`
|
||||
- `ledgerDb: LedgerDb`
|
||||
- `clock: Clock`
|
||||
- `idGenerator: IdGenerator`
|
||||
- `instruments?: TwoPcInstruments`
|
||||
- `interface TwoPcCoordinator`
|
||||
- `run2pc(input: Run2PcInput): Promise<Run2PcOutcome>`
|
||||
- `outstandingCount(): number`
|
||||
- `options: ResolvedTwoPcCoordinatorOptions`
|
||||
- `createTwoPcCoordinator(deps): TwoPcCoordinator` — factory.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The function is structured as one large `try` / `finally` so cleanup
|
||||
(span end, deadline timer clear, admission release) runs regardless
|
||||
of which branch fires. The pattern keeps the happy path linear and
|
||||
the failure paths exhaustive without nested try blocks.
|
||||
|
||||
The "any commit failure → in_doubt" escalation is deliberate. Once
|
||||
the coordinator has issued `COMMIT PREPARED` against at least one
|
||||
participant, the only correct follow-up is to keep trying — but the
|
||||
hot path cannot block on that. Instead, the row is marked `in_doubt`,
|
||||
the OTel counter ticks the `in_doubt` outcome, and the recovery
|
||||
runner picks it up on the next boot.
|
||||
|
||||
Vote-abort and prepare-failed paths both call `ac.abort()` BEFORE
|
||||
running phase 2, so any prepared participants whose subsequent SQL
|
||||
hits the signal map to `TWOPC_TIMEOUT` rather than getting stuck. The
|
||||
deadline timer is cleared in the `finally` so a successful commit
|
||||
does not leave a dangling abort handle on the AbortController.
|
||||
@@ -0,0 +1,414 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { LedgerDb } from "../persistence/persistence.js";
|
||||
import { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { createTwoPcCoordinator } from "./coordinator.js";
|
||||
|
||||
let nextId = 0;
|
||||
const FROZEN_NOW = 1_700_000_000_000;
|
||||
const baseDeps = () => {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
return {
|
||||
ledgerDb: db,
|
||||
registry,
|
||||
clock: () => FROZEN_NOW,
|
||||
idGenerator: () => `tx-${++nextId}`,
|
||||
options: { hostInstanceId: "h" },
|
||||
};
|
||||
};
|
||||
|
||||
interface Spy {
|
||||
readonly id: string;
|
||||
readonly calls: string[];
|
||||
participant: RegisteredTwoPcParticipant;
|
||||
}
|
||||
|
||||
function buildSpy(
|
||||
id: string,
|
||||
opts: {
|
||||
vote?: "commit" | "abort";
|
||||
writeIntent?: boolean;
|
||||
voterMode?: "writer" | "read-only";
|
||||
sqlFail?: { stage: "prepare" | "commit" | "rollback"; err: Error };
|
||||
prepareThrows?: Error;
|
||||
delayMs?: number;
|
||||
} = {}
|
||||
): Spy {
|
||||
const calls: string[] = [];
|
||||
const participant: RegisteredTwoPcParticipant = {
|
||||
id,
|
||||
voterMode: opts.voterMode ?? "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.sqlFail) {
|
||||
const matchesStage =
|
||||
(opts.sqlFail.stage === "prepare" &&
|
||||
sql.startsWith("PREPARE")) ||
|
||||
(opts.sqlFail.stage === "commit" &&
|
||||
sql.startsWith("COMMIT")) ||
|
||||
(opts.sqlFail.stage === "rollback" &&
|
||||
sql.startsWith("ROLLBACK"));
|
||||
if (matchesStage) throw opts.sqlFail.err;
|
||||
}
|
||||
if (opts.delayMs) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, opts.delayMs)
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => {
|
||||
if (opts.prepareThrows) throw opts.prepareThrows;
|
||||
return {
|
||||
vote: opts.vote ?? "commit",
|
||||
writeIntent: opts.writeIntent ?? true,
|
||||
};
|
||||
},
|
||||
};
|
||||
return { id, calls, participant };
|
||||
}
|
||||
|
||||
describe("createTwoPcCoordinator", () => {
|
||||
it("unanimous commit issues COMMIT PREPARED on every participant", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b");
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
const outcome = await c.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
expect(outcome.state).toBe("committed");
|
||||
expect(outcome.votes).toEqual([
|
||||
{ participantId: "a", vote: "commit" },
|
||||
{ participantId: "b", vote: "commit" },
|
||||
]);
|
||||
expect(
|
||||
a.calls.filter((s) => s.startsWith("COMMIT PREPARED")).length
|
||||
).toBe(1);
|
||||
expect(
|
||||
b.calls.filter((s) => s.startsWith("COMMIT PREPARED")).length
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("any vote-abort rolls back everywhere with TWOPC_VOTE_ABORT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b", { vote: "abort" });
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.VOTE_ABORT);
|
||||
}
|
||||
expect(
|
||||
a.calls.filter((s) => s.startsWith("ROLLBACK PREPARED")).length
|
||||
).toBe(1);
|
||||
expect(
|
||||
b.calls.filter((s) => s.startsWith("ROLLBACK PREPARED")).length
|
||||
).toBe(0); // b voted abort, never prepared.
|
||||
});
|
||||
|
||||
it("deadline expiry during prepare triggers rollback with TWOPC_TIMEOUT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const slow = buildSpy("slow", {
|
||||
sqlFail: {
|
||||
stage: "prepare",
|
||||
err: new Error("connection aborted"),
|
||||
},
|
||||
});
|
||||
// Patch the run() implementation to abort the signal mid-prepare
|
||||
// — emulates the deadline elapsing while the SQL is in flight.
|
||||
const original = slow.participant.connection.run;
|
||||
let runs = 0;
|
||||
slow.participant = {
|
||||
...slow.participant,
|
||||
connection: {
|
||||
...slow.participant.connection,
|
||||
run: async (sql: string) => {
|
||||
runs += 1;
|
||||
if (runs === 1) {
|
||||
// Sleep long enough for the deadline (1ms) to fire.
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
return original(sql);
|
||||
},
|
||||
},
|
||||
};
|
||||
deps.registry.register(slow.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [{ id: "slow", input: {} }],
|
||||
timeoutMs: 1,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.TIMEOUT);
|
||||
}
|
||||
});
|
||||
|
||||
it("commit failure escalates to TWOPC_IN_DOUBT with the row marked in_doubt", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a", {
|
||||
sqlFail: { stage: "commit", err: new Error("commit failed") },
|
||||
});
|
||||
deps.registry.register(a.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
}
|
||||
});
|
||||
|
||||
it("rollback failure escalates to TWOPC_IN_DOUBT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b", { vote: "abort" });
|
||||
// a's rollback SQL fails (non-idempotency error)
|
||||
const originalA = a.participant.connection.run;
|
||||
a.participant = {
|
||||
...a.participant,
|
||||
connection: {
|
||||
...a.participant.connection,
|
||||
run: async (sql: string) => {
|
||||
if (sql.startsWith("ROLLBACK PREPARED"))
|
||||
throw new Error("connection dropped");
|
||||
return originalA(sql);
|
||||
},
|
||||
},
|
||||
};
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses unknown participant id with TWOPC_PREPARE_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "ghost", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects timeoutMs above the hard ceiling without contacting participants", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
deps.registry.register(a.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [{ id: "a", input: {} }],
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED
|
||||
);
|
||||
}
|
||||
expect(a.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("outstanding-prepared cap rejects 257th concurrent transaction", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
deps.options = { hostInstanceId: "h", outstandingPreparedCap: 2 };
|
||||
let resolveA: () => void = () => undefined;
|
||||
let resolveB: () => void = () => undefined;
|
||||
const aGate = new Promise<void>((r) => {
|
||||
resolveA = r;
|
||||
});
|
||||
const bGate = new Promise<void>((r) => {
|
||||
resolveB = r;
|
||||
});
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b");
|
||||
a.participant = {
|
||||
...a.participant,
|
||||
prepare: async () => {
|
||||
await aGate;
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
b.participant = {
|
||||
...b.participant,
|
||||
prepare: async () => {
|
||||
await bGate;
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
const p1 = c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
const p2 = c.run2pc({ participants: [{ id: "b", input: {} }] });
|
||||
// Third call should hit the cap and reject synchronously.
|
||||
await expect(
|
||||
c.run2pc({ participants: [{ id: "a", input: {} }] })
|
||||
).rejects.toMatchObject({
|
||||
twoPcCode: TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED,
|
||||
});
|
||||
resolveA();
|
||||
resolveB();
|
||||
await Promise.all([p1, p2]);
|
||||
// After draining, a fresh call admits.
|
||||
const fresh = buildSpy("fresh");
|
||||
deps.registry.register(fresh.participant);
|
||||
const r = await c.run2pc({
|
||||
participants: [{ id: "fresh", input: {} }],
|
||||
});
|
||||
expect(r.state).toBe("committed");
|
||||
});
|
||||
|
||||
it("exposes outstandingCount and resolved options", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const deps = baseDeps();
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
expect(c.outstandingCount()).toBe(0);
|
||||
expect(c.options.hostInstanceId).toBe("h");
|
||||
expect(c.options.defaultTimeoutMs).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("preserves non-TwoPcError throws from a participant prepare callback as is", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a", { prepareThrows: new Error("custom") });
|
||||
deps.registry.register(a.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
// The prepare phase wraps the cause into TWOPC_PREPARE_FAILED.
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rethrows non-TwoPcError exceptions raised by commitPhaseTwo", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
// Force commit to throw a plain Error (not a TwoPcError) by
|
||||
// patching the connection's run to raise a non-TwoPcError on
|
||||
// the COMMIT PREPARED call but wrap that exception in a way
|
||||
// commitPhaseTwo doesn't recognise — we patch commitPhaseTwo's
|
||||
// invocation by replacing the participant's run AFTER prepare.
|
||||
deps.registry.register(a.participant);
|
||||
const original = a.participant.connection.run;
|
||||
let calls = 0;
|
||||
a.participant.connection.run = (async (sql: string) => {
|
||||
calls += 1;
|
||||
if (sql.startsWith("COMMIT PREPARED")) {
|
||||
// Throw something that bypasses TwoPcError detection
|
||||
// inside commitPhaseTwo: the phase wraps any thrown
|
||||
// value into TWOPC_COMMIT_FAILED, so the coordinator
|
||||
// never sees a non-TwoPcError. Confirm the wrapping
|
||||
// happens (so the else-branch in the coordinator's
|
||||
// catch is unreachable by design and the test asserts
|
||||
// that the wrapping path is the one taken).
|
||||
throw new RangeError("raw");
|
||||
}
|
||||
return original.call(a.participant.connection, sql);
|
||||
}) as typeof original;
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
// commitPhaseTwo wraps any thrown value into TWOPC_COMMIT_FAILED,
|
||||
// which the coordinator collects into commitErrors and escalates
|
||||
// to TWOPC_IN_DOUBT.
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
expect(calls).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes a read-only voter in the vote tally without sending SQL", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const writer = buildSpy("writer");
|
||||
const reader = buildSpy("reader", {
|
||||
voterMode: "read-only",
|
||||
writeIntent: false,
|
||||
});
|
||||
deps.registry.register(writer.participant);
|
||||
deps.registry.register(reader.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
const outcome = await c.run2pc({
|
||||
participants: [
|
||||
{ id: "writer", input: {} },
|
||||
{ id: "reader", input: {} },
|
||||
],
|
||||
});
|
||||
expect(outcome.state).toBe("committed");
|
||||
expect(reader.calls).toEqual([]); // no SQL ever to the read-only voter
|
||||
});
|
||||
});
|
||||
281
packages/api-stack/api/api-2pc/src/coordinator/coordinator.ts
Normal file
281
packages/api-stack/api/api-2pc/src/coordinator/coordinator.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
import type { Clock, IdGenerator } from "@suaveplan/types";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import {
|
||||
createTwoPcInstruments,
|
||||
type TwoPcInstruments,
|
||||
} from "../instruments/instruments.js";
|
||||
import { OutstandingPreparedCap } from "../limits/outstanding-cap.js";
|
||||
import { assertTimeoutBelowCeiling } from "../limits/timeout-ceiling.js";
|
||||
import {
|
||||
type ResolvedTwoPcCoordinatorOptions,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
type TwoPcCoordinatorOptions,
|
||||
} from "../options/options.js";
|
||||
import type { LedgerDb } from "../persistence/persistence.js";
|
||||
import {
|
||||
type LedgerParticipantEntry,
|
||||
TwoPcLedgerStore,
|
||||
} from "../persistence/persistence.js";
|
||||
import {
|
||||
commitPhaseTwo,
|
||||
type ParticipantPrepareOutcome,
|
||||
preparePhaseOne,
|
||||
rollbackPhaseTwo,
|
||||
} from "../phases/index.js";
|
||||
import type { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
import type { Run2PcInput, Run2PcOutcome } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Adapter surface the coordinator depends on. Mirrors the foundation
|
||||
* pattern: dependencies passed in, not imported.
|
||||
*/
|
||||
export interface TwoPcCoordinatorDeps {
|
||||
readonly options: TwoPcCoordinatorOptions;
|
||||
readonly registry: TwoPcParticipantRegistry;
|
||||
readonly ledgerDb: LedgerDb;
|
||||
readonly clock: Clock;
|
||||
readonly idGenerator: IdGenerator;
|
||||
/**
|
||||
* Optional override; defaults to a fresh instance backed by the OTel
|
||||
* globals. Tests inject their own meter/tracer providers.
|
||||
*/
|
||||
readonly instruments?: TwoPcInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Public coordinator surface. `run2pc` is the only entry point;
|
||||
* `outstandingCount` and `getOptions` are exposed for diagnostics and
|
||||
* the OTel observable gauge.
|
||||
*/
|
||||
export interface TwoPcCoordinator {
|
||||
run2pc(input: Run2PcInput): Promise<Run2PcOutcome>;
|
||||
outstandingCount(): number;
|
||||
readonly options: ResolvedTwoPcCoordinatorOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory entry point. Builds a fully-wired coordinator from the
|
||||
* supplied dependencies. The returned object is the only surface
|
||||
* application code touches.
|
||||
*/
|
||||
export function createTwoPcCoordinator(
|
||||
deps: TwoPcCoordinatorDeps
|
||||
): TwoPcCoordinator {
|
||||
const options = resolveTwoPcCoordinatorOptions(deps.options);
|
||||
const ledger = new TwoPcLedgerStore(deps.ledgerDb);
|
||||
const cap = new OutstandingPreparedCap(options.outstandingPreparedCap);
|
||||
const instruments =
|
||||
deps.instruments ??
|
||||
createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => cap.snapshot(),
|
||||
});
|
||||
const { registry, clock, idGenerator } = deps;
|
||||
|
||||
async function run2pc(input: Run2PcInput): Promise<Run2PcOutcome> {
|
||||
const requestedTimeout = input.timeoutMs ?? options.defaultTimeoutMs;
|
||||
assertTimeoutBelowCeiling(requestedTimeout);
|
||||
cap.admit();
|
||||
const txId = idGenerator();
|
||||
const lookups = input.participants.map((p) => {
|
||||
const registered = registry.get(p.id);
|
||||
if (registered === undefined) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PREPARE_FAILED,
|
||||
`Unknown participant ${p.id}`,
|
||||
{ context: { participantId: p.id, txId } }
|
||||
);
|
||||
}
|
||||
return { spec: p, registered };
|
||||
});
|
||||
const participantEntries: LedgerParticipantEntry[] = lookups.map(
|
||||
({ registered }) => ({
|
||||
participantId: registered.id,
|
||||
voterMode: registered.voterMode,
|
||||
vote: null,
|
||||
})
|
||||
);
|
||||
|
||||
ledger.insert({
|
||||
id: txId,
|
||||
coordinator: options.hostInstanceId,
|
||||
participants: participantEntries,
|
||||
});
|
||||
|
||||
const txSpan = instruments.tracer.startSpan("2pc.transaction", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.count": lookups.length,
|
||||
"coordinator.host": options.hostInstanceId,
|
||||
},
|
||||
});
|
||||
const ac = new AbortController();
|
||||
const deadlineHandle = setTimeout(() => ac.abort(), requestedTimeout);
|
||||
|
||||
const prepareT0 = clock();
|
||||
const outcomes: ParticipantPrepareOutcome[] = [];
|
||||
let abortReason: TwoPcError | null = null;
|
||||
try {
|
||||
try {
|
||||
for (const { spec, registered } of lookups) {
|
||||
const o = await preparePhaseOne({
|
||||
hostInstanceId: options.hostInstanceId,
|
||||
txId,
|
||||
participant: registered,
|
||||
payload: spec.input,
|
||||
instruments,
|
||||
signal: ac.signal,
|
||||
});
|
||||
outcomes.push(o);
|
||||
if (o.vote === "abort") {
|
||||
abortReason = new TwoPcError(
|
||||
TWOPC_CODES.VOTE_ABORT,
|
||||
`Participant ${o.participantId} voted abort`,
|
||||
{
|
||||
context: {
|
||||
participantId: o.participantId,
|
||||
txId,
|
||||
},
|
||||
}
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) {
|
||||
abortReason = err;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const prepareDuration = clock() - prepareT0;
|
||||
const commitT0 = clock();
|
||||
let outcomeState: Run2PcOutcome["state"];
|
||||
let commitDuration = 0;
|
||||
|
||||
if (
|
||||
abortReason === null &&
|
||||
outcomes.every((o) => o.vote === "commit")
|
||||
) {
|
||||
ledger.markAllVotedCommit({
|
||||
id: txId,
|
||||
participants: participantEntries.map((e) => {
|
||||
const matched = outcomes.find(
|
||||
(o) => o.participantId === e.participantId
|
||||
);
|
||||
return {
|
||||
...e,
|
||||
vote: matched?.vote ?? null,
|
||||
};
|
||||
}),
|
||||
preparedAt: new Date(clock()).toISOString(),
|
||||
});
|
||||
const commitErrors: TwoPcError[] = [];
|
||||
for (const o of outcomes) {
|
||||
try {
|
||||
await commitPhaseTwo({
|
||||
txId,
|
||||
participant: registry.get(o.participantId)!,
|
||||
outcome: o,
|
||||
instruments,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) commitErrors.push(err);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
commitDuration = clock() - commitT0;
|
||||
if (commitErrors.length > 0) {
|
||||
ledger.markInDoubt(txId);
|
||||
outcomeState = "in_doubt";
|
||||
instruments.transactionsTotal.add(1, {
|
||||
outcome: "in_doubt",
|
||||
});
|
||||
instruments.commitDurationMs.record(commitDuration, {
|
||||
outcome: "in_doubt",
|
||||
});
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.IN_DOUBT,
|
||||
`Transaction ${txId} is in doubt (${commitErrors.length} commit failure(s))`,
|
||||
{
|
||||
cause: commitErrors[0],
|
||||
context: { txId, failed: commitErrors.length },
|
||||
}
|
||||
);
|
||||
}
|
||||
ledger.markCommitted(txId, new Date(clock()).toISOString());
|
||||
outcomeState = "committed";
|
||||
instruments.transactionsTotal.add(1, { outcome: "committed" });
|
||||
instruments.commitDurationMs.record(commitDuration, {
|
||||
outcome: "committed",
|
||||
});
|
||||
} else {
|
||||
// Rollback path. Abort the deadline so any pending prepares
|
||||
// hit the TIMEOUT path on their next SQL call.
|
||||
ac.abort();
|
||||
ledger.markAborting({
|
||||
id: txId,
|
||||
participants: participantEntries.map((e) => {
|
||||
const matched = outcomes.find(
|
||||
(o) => o.participantId === e.participantId
|
||||
);
|
||||
return { ...e, vote: matched?.vote ?? null };
|
||||
}),
|
||||
});
|
||||
const rollbackErrors: TwoPcError[] = [];
|
||||
for (const o of outcomes) {
|
||||
try {
|
||||
await rollbackPhaseTwo({
|
||||
txId,
|
||||
participant: registry.get(o.participantId)!,
|
||||
outcome: o,
|
||||
instruments,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) rollbackErrors.push(err);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
ledger.markAborted(txId, new Date(clock()).toISOString());
|
||||
outcomeState = "aborted";
|
||||
instruments.transactionsTotal.add(1, { outcome: "aborted" });
|
||||
if (rollbackErrors.length > 0) {
|
||||
ledger.markInDoubt(txId);
|
||||
instruments.transactionsTotal.add(1, {
|
||||
outcome: "in_doubt",
|
||||
});
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.IN_DOUBT,
|
||||
`Rollback partially failed for ${txId}`,
|
||||
{
|
||||
cause: rollbackErrors[0],
|
||||
context: { txId },
|
||||
}
|
||||
);
|
||||
}
|
||||
if (abortReason !== null) throw abortReason;
|
||||
}
|
||||
|
||||
return {
|
||||
txId,
|
||||
state: outcomeState,
|
||||
votes: outcomes.map((o) => ({
|
||||
participantId: o.participantId,
|
||||
vote: o.vote as TwoPcVote,
|
||||
})),
|
||||
prepareDurationMs: prepareDuration,
|
||||
commitDurationMs: commitDuration,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(deadlineHandle);
|
||||
txSpan.end();
|
||||
cap.release();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
run2pc,
|
||||
outstandingCount: () => cap.snapshot(),
|
||||
options,
|
||||
};
|
||||
}
|
||||
5
packages/api-stack/api/api-2pc/src/coordinator/index.ts
Normal file
5
packages/api-stack/api/api-2pc/src/coordinator/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
createTwoPcCoordinator,
|
||||
type TwoPcCoordinator,
|
||||
type TwoPcCoordinatorDeps,
|
||||
} from "./coordinator.js";
|
||||
@@ -0,0 +1,97 @@
|
||||
# `error-codes/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
This module owns the `TWOPC_*` error-code namespace and the `TwoPcError`
|
||||
concrete class for the `@suaveplan/api-2pc` package. The registry is the
|
||||
canonical contract every production code path in the coordinator throws
|
||||
against — observability dashboards, support runbooks, and integrating
|
||||
plugins all join on these stable string codes.
|
||||
|
||||
The codes are intentionally enumerated up-front rather than grown
|
||||
ad-hoc: a missing code is a spec deviation that a CI gate can detect,
|
||||
and the closed set lets operators write alerting rules without
|
||||
constantly chasing schema drift.
|
||||
|
||||
## Features
|
||||
|
||||
- Ten stable codes (`TIMEOUT`, `VOTE_ABORT`, `PREPARE_FAILED`,
|
||||
`COMMIT_FAILED`, `ROLLBACK_FAILED`, `IN_DOUBT`,
|
||||
`PARTICIPANT_NOT_PREPARABLE`, `READ_ONLY_PARTICIPANT_CANNOT_WRITE`,
|
||||
`OUTSTANDING_LIMIT_EXCEEDED`, `TIMEOUT_CEILING_EXCEEDED`).
|
||||
- Each code is exposed via `TWOPC_CODES.<NAME>` returning the prefixed
|
||||
string `"TWOPC.<NAME>"` produced by `defineCodes`.
|
||||
- `TwoPcError` extends `SuaveplanError` so cause-chain traversal,
|
||||
Zod-validated wire format, and the OTel span-event adapter from
|
||||
`@suaveplan/error` all apply unchanged.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT,
|
||||
"Phase-1 deadline elapsed",
|
||||
{ context: { txId, timeoutMs: 5000 } }
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Branching on error code at the boundary between coordinator and caller
|
||||
is the recommended consumer pattern. The narrowed `twoPcCode` field
|
||||
gives an exhaustive switch the compiler can check:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
|
||||
try {
|
||||
await run2pc({ participants, timeoutMs: 5000 }, ctx);
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) {
|
||||
switch (err.twoPcCode) {
|
||||
case TWOPC_CODES.TIMEOUT:
|
||||
metrics.incr("billing.2pc.timeout");
|
||||
return retryWithBackoff();
|
||||
case TWOPC_CODES.VOTE_ABORT:
|
||||
return surfaceUserError();
|
||||
case TWOPC_CODES.IN_DOUBT:
|
||||
pageOncall(err);
|
||||
return;
|
||||
default:
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `TWOPC_CODES` — frozen object `{ TIMEOUT: "TWOPC.TIMEOUT", ... }`.
|
||||
- `type TwoPcErrorCode` — string-literal union of every value in
|
||||
`TWOPC_CODES`.
|
||||
- `class TwoPcError extends SuaveplanError<TwoPcErrorCode>` — exposes
|
||||
`twoPcCode` as a same-typed accessor for ergonomic narrowing without
|
||||
re-asserting at every catch site.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`defineCodes` from `@suaveplan/error` produces the prefixed values at
|
||||
runtime, so every code string starts with `"TWOPC."`. This matches the
|
||||
prefix convention used by sibling api-stack packages (`HOST.*`,
|
||||
`PLUGIN.*`, `RELOAD.*`, `LEDGER.*`). The `TwoPcError` class carries a
|
||||
redundant `twoPcCode` field — duplicating `code` — purely to give
|
||||
callers a name they don't have to import the generic
|
||||
`SuaveplanError<C>` type to narrow against. The duplication is
|
||||
deliberate and harmless because the base class makes `code` readonly.
|
||||
|
||||
The registry is closed; adding a code requires an OpenSpec change so
|
||||
every consumer dashboard and runbook stays in sync. Coverage assertions
|
||||
ensure every code is actually thrown by a production path before this
|
||||
package can graduate to `1.0.0`.
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, expectTypeOf, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError, type TwoPcErrorCode } from "./error-codes.js";
|
||||
|
||||
describe("TWOPC_CODES", () => {
|
||||
it("exposes ten prefixed string codes", { timeout: 5000 }, () => {
|
||||
const values = Object.values(TWOPC_CODES);
|
||||
expect(values).toHaveLength(10);
|
||||
for (const v of values) {
|
||||
expect(v.startsWith("TWOPC.")).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes every required code identifier", { timeout: 5000 }, () => {
|
||||
expect(TWOPC_CODES.TIMEOUT).toBe("TWOPC.TIMEOUT");
|
||||
expect(TWOPC_CODES.VOTE_ABORT).toBe("TWOPC.VOTE_ABORT");
|
||||
expect(TWOPC_CODES.PREPARE_FAILED).toBe("TWOPC.PREPARE_FAILED");
|
||||
expect(TWOPC_CODES.COMMIT_FAILED).toBe("TWOPC.COMMIT_FAILED");
|
||||
expect(TWOPC_CODES.ROLLBACK_FAILED).toBe("TWOPC.ROLLBACK_FAILED");
|
||||
expect(TWOPC_CODES.IN_DOUBT).toBe("TWOPC.IN_DOUBT");
|
||||
expect(TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE).toBe(
|
||||
"TWOPC.PARTICIPANT_NOT_PREPARABLE"
|
||||
);
|
||||
expect(TWOPC_CODES.READ_ONLY_PARTICIPANT_CANNOT_WRITE).toBe(
|
||||
"TWOPC.READ_ONLY_PARTICIPANT_CANNOT_WRITE"
|
||||
);
|
||||
expect(TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED).toBe(
|
||||
"TWOPC.OUTSTANDING_LIMIT_EXCEEDED"
|
||||
);
|
||||
expect(TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED).toBe(
|
||||
"TWOPC.TIMEOUT_CEILING_EXCEEDED"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("TwoPcError", () => {
|
||||
it("carries the supplied code on both code and twoPcCode", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = new TwoPcError(TWOPC_CODES.TIMEOUT, "deadline");
|
||||
expect(err.code).toBe(TWOPC_CODES.TIMEOUT);
|
||||
expect(err.twoPcCode).toBe(TWOPC_CODES.TIMEOUT);
|
||||
expect(err.message).toBe("deadline");
|
||||
});
|
||||
|
||||
it("preserves the cause and context options", { timeout: 5000 }, () => {
|
||||
const cause = new Error("inner");
|
||||
const err = new TwoPcError(TWOPC_CODES.IN_DOUBT, "partial commit", {
|
||||
cause,
|
||||
context: { txId: "tx-1" },
|
||||
});
|
||||
expect(err.cause).toBe(cause);
|
||||
expect(err.context).toEqual({ txId: "tx-1" });
|
||||
});
|
||||
|
||||
it("narrows the code type to TwoPcErrorCode", { timeout: 5000 }, () => {
|
||||
const err = new TwoPcError(TWOPC_CODES.VOTE_ABORT, "abort");
|
||||
expectTypeOf(err.twoPcCode).toEqualTypeOf<TwoPcErrorCode>();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { SuaveplanErrorOptions } from "@suaveplan/error";
|
||||
import { defineCodes, SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
/**
|
||||
* Stable error-code registry for `@suaveplan/api-2pc`. The ten codes
|
||||
* cover every production failure mode the coordinator surfaces:
|
||||
*
|
||||
* - `TWOPC_TIMEOUT` — the prepare-phase deadline elapsed before all
|
||||
* participants returned a vote. The coordinator rolls back every
|
||||
* participant that already voted commit.
|
||||
* - `TWOPC_VOTE_ABORT` — at least one participant voted abort; the
|
||||
* coordinator rolled back every commit-voter.
|
||||
* - `TWOPC_PREPARE_FAILED` — a participant raised during phase 1
|
||||
* (network drop, SQL error, capability check failure). Treated as an
|
||||
* implicit abort vote: rollback runs everywhere a prepare succeeded.
|
||||
* - `TWOPC_COMMIT_FAILED` — at least one `COMMIT PREPARED` raised. The
|
||||
* transaction is `in_doubt`; the recovery runner is the only path
|
||||
* that can resolve it.
|
||||
* - `TWOPC_ROLLBACK_FAILED` — at least one `ROLLBACK PREPARED` raised.
|
||||
* Recovery resolves it on next boot.
|
||||
* - `TWOPC_IN_DOUBT` — terminal state surfacing a partially-committed
|
||||
* transaction whose follow-up requires the recovery runner.
|
||||
* - `TWOPC_PARTICIPANT_NOT_PREPARABLE` — install-time refusal of a
|
||||
* write participant whose engine does not advertise `db.preparable`.
|
||||
* SQLite / SurrealDB SHALL hit this when registered as writers.
|
||||
* - `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE` — a read-only voter's
|
||||
* prepare callback returned a write-intent payload; the coordinator
|
||||
* refused the transaction before phase 2.
|
||||
* - `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` — admission control refused a
|
||||
* new `run2pc` call because the host has 256 prepared transactions
|
||||
* already in flight. The cap is operator-tunable but the default
|
||||
* ships at 256 per design doc §14.
|
||||
* - `TWOPC_TIMEOUT_CEILING_EXCEEDED` — caller-supplied `timeoutMs`
|
||||
* exceeded the 30 000 ms hard ceiling. The call is refused at the
|
||||
* boundary (no participant is contacted).
|
||||
*
|
||||
* Consumers branch on the string value:
|
||||
*
|
||||
* ```ts
|
||||
* if (error.code === TWOPC_CODES.TIMEOUT) { ... }
|
||||
* ```
|
||||
*/
|
||||
export const TWOPC_CODES = defineCodes("TWOPC", [
|
||||
"TIMEOUT",
|
||||
"VOTE_ABORT",
|
||||
"PREPARE_FAILED",
|
||||
"COMMIT_FAILED",
|
||||
"ROLLBACK_FAILED",
|
||||
"IN_DOUBT",
|
||||
"PARTICIPANT_NOT_PREPARABLE",
|
||||
"READ_ONLY_PARTICIPANT_CANNOT_WRITE",
|
||||
"OUTSTANDING_LIMIT_EXCEEDED",
|
||||
"TIMEOUT_CEILING_EXCEEDED",
|
||||
] as const);
|
||||
|
||||
export type TwoPcErrorCode = (typeof TWOPC_CODES)[keyof typeof TWOPC_CODES];
|
||||
|
||||
/**
|
||||
* Concrete error class for the `api-2pc` package. Extends
|
||||
* {@link SuaveplanError} so `instanceof TwoPcError` narrows
|
||||
* `code` to {@link TwoPcErrorCode} at the type level.
|
||||
*/
|
||||
export class TwoPcError extends SuaveplanError<TwoPcErrorCode> {
|
||||
public readonly twoPcCode: TwoPcErrorCode;
|
||||
|
||||
public constructor(
|
||||
code: TwoPcErrorCode,
|
||||
message: string,
|
||||
options?: SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.twoPcCode = code;
|
||||
}
|
||||
}
|
||||
5
packages/api-stack/api/api-2pc/src/error-codes/index.ts
Normal file
5
packages/api-stack/api/api-2pc/src/error-codes/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
TWOPC_CODES,
|
||||
TwoPcError,
|
||||
type TwoPcErrorCode,
|
||||
} from "./error-codes.js";
|
||||
68
packages/api-stack/api/api-2pc/src/index.ts
Normal file
68
packages/api-stack/api/api-2pc/src/index.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
export {
|
||||
createTwoPcCoordinator,
|
||||
type TwoPcCoordinator,
|
||||
type TwoPcCoordinatorDeps,
|
||||
} from "./coordinator/coordinator.js";
|
||||
export {
|
||||
TWOPC_CODES,
|
||||
TwoPcError,
|
||||
type TwoPcErrorCode,
|
||||
} from "./error-codes/error-codes.js";
|
||||
export {
|
||||
createTwoPcInstruments,
|
||||
type TwoPcInstruments,
|
||||
type TwoPcInstrumentsConfig,
|
||||
type TwoPcOutcome,
|
||||
} from "./instruments/instruments.js";
|
||||
export {
|
||||
assertTimeoutBelowCeiling,
|
||||
OutstandingPreparedCap,
|
||||
} from "./limits/index.js";
|
||||
export {
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
type ResolvedTwoPcCoordinatorOptions,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
TIMEOUT_CEILING_MS,
|
||||
type TwoPcCoordinatorOptions,
|
||||
twoPcCoordinatorOptionsSchema,
|
||||
} from "./options/options.js";
|
||||
export {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
type LedgerStatement,
|
||||
type TwoPcLedgerRow,
|
||||
TwoPcLedgerStore,
|
||||
} from "./persistence/persistence.js";
|
||||
export {
|
||||
buildGid,
|
||||
type CommitPhaseInput,
|
||||
commitPhaseTwo,
|
||||
gidPrefixFor,
|
||||
isUnknownPreparedTransaction,
|
||||
type ParticipantPrepareOutcome,
|
||||
type PreparePhaseInput,
|
||||
parseGid,
|
||||
preparePhaseOne,
|
||||
type RollbackPhaseInput,
|
||||
rollbackPhaseTwo,
|
||||
} from "./phases/index.js";
|
||||
export {
|
||||
buildProbeFromRegistry,
|
||||
type RecoveryInput,
|
||||
type RecoveryProbe,
|
||||
type RecoveryReport,
|
||||
resolveRecovery,
|
||||
} from "./recovery/recovery.js";
|
||||
export {
|
||||
type PreparableCapabilityCheck,
|
||||
TwoPcParticipantRegistry,
|
||||
} from "./registry/participant-registry.js";
|
||||
export type {
|
||||
PreparableConnection,
|
||||
RegisteredTwoPcParticipant,
|
||||
Run2PcInput,
|
||||
Run2PcOutcome,
|
||||
TwoPcCoordinatorState,
|
||||
TwoPcPrepareResult,
|
||||
} from "./types/types.types.js";
|
||||
6
packages/api-stack/api/api-2pc/src/instruments/index.ts
Normal file
6
packages/api-stack/api/api-2pc/src/instruments/index.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
createTwoPcInstruments,
|
||||
type TwoPcInstruments,
|
||||
type TwoPcInstrumentsConfig,
|
||||
type TwoPcOutcome,
|
||||
} from "./instruments.js";
|
||||
105
packages/api-stack/api/api-2pc/src/instruments/instruments.md
Normal file
105
packages/api-stack/api/api-2pc/src/instruments/instruments.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# `instruments/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Builds the OTel instrument registry the 2PC coordinator emits against
|
||||
on every transaction. Each instrument's name follows design doc §9.2
|
||||
exactly so a missing entry is a spec deviation a CI gate could detect.
|
||||
Centralising creation keeps the coordinator code free of meter / tracer
|
||||
plumbing — every phase module simply records against the frozen
|
||||
bundle returned here.
|
||||
|
||||
The observable gauge wires into the coordinator's outstanding-prepared
|
||||
counter via a snapshot callback, so dashboards see the live admission
|
||||
state without the coordinator pushing on every change.
|
||||
|
||||
## Features
|
||||
|
||||
- One span: `2pc.transaction` per transaction lifetime; per-phase
|
||||
child spans `2pc.prepare`, `2pc.commit`, `2pc.rollback` (started by
|
||||
the phase modules themselves, using the tracer this bundle exposes).
|
||||
- One counter: `2pc.transactions_total` labelled `{ outcome }`.
|
||||
- Two histograms: `2pc.prepare_duration_ms`, `2pc.commit_duration_ms`
|
||||
labelled `{ outcome }`.
|
||||
- One observable gauge: `2pc.outstanding_prepared` wired to a
|
||||
snapshot callback supplied at construction.
|
||||
- Provider defaults to the OTel globals (no-op when no SDK is
|
||||
installed) so unit tests do not have to wire a provider.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { createTwoPcInstruments } from "@suaveplan/api-2pc/instruments";
|
||||
|
||||
const instruments = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => coordinator.outstandingCount(),
|
||||
});
|
||||
|
||||
instruments.transactionsTotal.add(1, { outcome: "committed" });
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Inject a real OTel SDK and a custom snapshot to feed a live dashboard:
|
||||
|
||||
```ts
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
import { MeterProvider } from "@opentelemetry/sdk-metrics";
|
||||
import { createTwoPcInstruments } from "@suaveplan/api-2pc/instruments";
|
||||
|
||||
const meterProvider = new MeterProvider({ /* readers/exporters */ });
|
||||
metrics.setGlobalMeterProvider(meterProvider);
|
||||
|
||||
const instruments = createTwoPcInstruments({
|
||||
meterProvider,
|
||||
tracerProvider: trace.getTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => coordinator.outstandingCount(),
|
||||
});
|
||||
|
||||
const span = instruments.tracer.startSpan("2pc.transaction", {
|
||||
attributes: { "tx.id": txId, participant_count: 2 },
|
||||
});
|
||||
try {
|
||||
// ...
|
||||
instruments.prepareDurationMs.record(prepareElapsedMs, {
|
||||
outcome: "committed",
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type TwoPcOutcome = "committed" | "aborted" | "in_doubt"` — values
|
||||
attached to the `outcome` attribute on every counter / histogram
|
||||
observation.
|
||||
- `interface TwoPcInstrumentsConfig`
|
||||
- `meterProvider?: MeterProvider`
|
||||
- `tracerProvider?: TracerProvider`
|
||||
- `outstandingPreparedSnapshot: () => number`
|
||||
- `interface TwoPcInstruments` — frozen bundle exposing `meter`,
|
||||
`tracer`, `transactionsTotal`, `prepareDurationMs`,
|
||||
`commitDurationMs`, `outstandingPreparedGauge`.
|
||||
- `createTwoPcInstruments(config): TwoPcInstruments` — factory.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The observable gauge's callback fires whenever the SDK collects metrics
|
||||
(typically on a fixed interval driven by the exporter). The snapshot
|
||||
returned by `outstandingPreparedSnapshot` is therefore an
|
||||
eventually-consistent value — operators must not treat it as a strict
|
||||
upper bound for admission decisions. The strict upper bound is enforced
|
||||
inside `limits/outstanding-cap.ts`; the gauge is purely a dashboard
|
||||
signal.
|
||||
|
||||
Histogram unit is set explicitly to `"ms"` per OTel naming convention.
|
||||
The two histograms are named after the phases they measure so a single
|
||||
dashboard panel can render side-by-side latency for prepare and commit
|
||||
without re-labelling — operators flagging long prepares typically
|
||||
care more about phase-1 latency because that is when row-level locks
|
||||
are held on every participant.
|
||||
@@ -0,0 +1,218 @@
|
||||
import type {
|
||||
BatchObservableCallback,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
ObservableCallback,
|
||||
ObservableGauge,
|
||||
Tracer,
|
||||
TracerProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { createTwoPcInstruments } from "./instruments.js";
|
||||
|
||||
interface FakeInstrument {
|
||||
readonly kind: "counter" | "histogram" | "gauge";
|
||||
readonly name: string;
|
||||
readonly options: Readonly<Record<string, unknown>>;
|
||||
addCalls: Array<{
|
||||
value: number;
|
||||
attrs?: Readonly<Record<string, unknown>>;
|
||||
}>;
|
||||
recordCalls: Array<{
|
||||
value: number;
|
||||
attrs?: Readonly<Record<string, unknown>>;
|
||||
}>;
|
||||
gaugeCallbacks: ObservableCallback[];
|
||||
}
|
||||
|
||||
interface CapturedMeter {
|
||||
readonly name: string;
|
||||
readonly instruments: FakeInstrument[];
|
||||
}
|
||||
|
||||
function buildFakeMeterProvider(): {
|
||||
provider: MeterProvider;
|
||||
meters: CapturedMeter[];
|
||||
} {
|
||||
const meters: CapturedMeter[] = [];
|
||||
const provider: MeterProvider = {
|
||||
getMeter(name: string): Meter {
|
||||
const captured: CapturedMeter = { name, instruments: [] };
|
||||
meters.push(captured);
|
||||
const meter: Meter = {
|
||||
createCounter(n, opts) {
|
||||
const inst: FakeInstrument = {
|
||||
kind: "counter",
|
||||
name: n,
|
||||
options: opts ?? {},
|
||||
addCalls: [],
|
||||
recordCalls: [],
|
||||
gaugeCallbacks: [],
|
||||
};
|
||||
captured.instruments.push(inst);
|
||||
return {
|
||||
add(v: number, a?: Readonly<Record<string, unknown>>) {
|
||||
inst.addCalls.push({ value: v, attrs: a });
|
||||
},
|
||||
} as ReturnType<Meter["createCounter"]>;
|
||||
},
|
||||
createHistogram(n, opts) {
|
||||
const inst: FakeInstrument = {
|
||||
kind: "histogram",
|
||||
name: n,
|
||||
options: opts ?? {},
|
||||
addCalls: [],
|
||||
recordCalls: [],
|
||||
gaugeCallbacks: [],
|
||||
};
|
||||
captured.instruments.push(inst);
|
||||
return {
|
||||
record(
|
||||
v: number,
|
||||
a?: Readonly<Record<string, unknown>>
|
||||
) {
|
||||
inst.recordCalls.push({ value: v, attrs: a });
|
||||
},
|
||||
} as ReturnType<Meter["createHistogram"]>;
|
||||
},
|
||||
createObservableGauge(n, opts) {
|
||||
const inst: FakeInstrument = {
|
||||
kind: "gauge",
|
||||
name: n,
|
||||
options: opts ?? {},
|
||||
addCalls: [],
|
||||
recordCalls: [],
|
||||
gaugeCallbacks: [],
|
||||
};
|
||||
captured.instruments.push(inst);
|
||||
const gauge: ObservableGauge = {
|
||||
addCallback(cb: ObservableCallback) {
|
||||
inst.gaugeCallbacks.push(cb);
|
||||
},
|
||||
removeCallback(_cb: ObservableCallback) {
|
||||
// no-op for tests
|
||||
},
|
||||
} as unknown as ObservableGauge;
|
||||
return gauge;
|
||||
},
|
||||
addBatchObservableCallback(_cb: BatchObservableCallback) {
|
||||
// no-op
|
||||
},
|
||||
removeBatchObservableCallback(_cb: BatchObservableCallback) {
|
||||
// no-op
|
||||
},
|
||||
} as unknown as Meter;
|
||||
return meter;
|
||||
},
|
||||
} as MeterProvider;
|
||||
return { provider, meters };
|
||||
}
|
||||
|
||||
function buildFakeTracerProvider(): TracerProvider {
|
||||
return {
|
||||
getTracer(_name: string): Tracer {
|
||||
return {
|
||||
startSpan() {
|
||||
return {
|
||||
setAttribute: () => undefined,
|
||||
setStatus: () => undefined,
|
||||
recordException: () => undefined,
|
||||
addEvent: () => undefined,
|
||||
end: () => undefined,
|
||||
spanContext: () => ({
|
||||
traceId: "0",
|
||||
spanId: "0",
|
||||
traceFlags: 0,
|
||||
}),
|
||||
isRecording: () => false,
|
||||
updateName: () => undefined,
|
||||
};
|
||||
},
|
||||
startActiveSpan: undefined as never,
|
||||
} as unknown as Tracer;
|
||||
},
|
||||
} as TracerProvider;
|
||||
}
|
||||
|
||||
describe("createTwoPcInstruments", () => {
|
||||
it("creates every documented instrument on the package meter", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const { provider, meters } = buildFakeMeterProvider();
|
||||
const instruments = createTwoPcInstruments({
|
||||
meterProvider: provider,
|
||||
tracerProvider: buildFakeTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => 7,
|
||||
});
|
||||
expect(meters[0]?.name).toBe("@suaveplan/api-2pc");
|
||||
const names = meters[0]?.instruments.map((i) => i.name);
|
||||
expect(names).toEqual([
|
||||
"2pc.transactions_total",
|
||||
"2pc.prepare_duration_ms",
|
||||
"2pc.commit_duration_ms",
|
||||
"2pc.outstanding_prepared",
|
||||
]);
|
||||
expect(instruments.transactionsTotal).toBeDefined();
|
||||
expect(instruments.prepareDurationMs).toBeDefined();
|
||||
expect(instruments.commitDurationMs).toBeDefined();
|
||||
expect(instruments.outstandingPreparedGauge).toBeDefined();
|
||||
});
|
||||
|
||||
it("uses ms units on the duration histograms", { timeout: 5000 }, () => {
|
||||
const { provider, meters } = buildFakeMeterProvider();
|
||||
createTwoPcInstruments({
|
||||
meterProvider: provider,
|
||||
tracerProvider: buildFakeTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
const histograms = meters[0]?.instruments.filter(
|
||||
(i) => i.kind === "histogram"
|
||||
);
|
||||
const byName = new Map(histograms?.map((h) => [h.name, h]));
|
||||
expect(byName.get("2pc.prepare_duration_ms")?.options).toEqual({
|
||||
unit: "ms",
|
||||
});
|
||||
expect(byName.get("2pc.commit_duration_ms")?.options).toEqual({
|
||||
unit: "ms",
|
||||
});
|
||||
});
|
||||
|
||||
it("the observable gauge callback reports the snapshot value", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const { provider, meters } = buildFakeMeterProvider();
|
||||
let snapshot = 3;
|
||||
createTwoPcInstruments({
|
||||
meterProvider: provider,
|
||||
tracerProvider: buildFakeTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => snapshot,
|
||||
});
|
||||
const gauge = meters[0]?.instruments.find((i) => i.kind === "gauge");
|
||||
const observations: number[] = [];
|
||||
const observer = {
|
||||
observe(value: number) {
|
||||
observations.push(value);
|
||||
},
|
||||
};
|
||||
for (const cb of gauge?.gaugeCallbacks ?? []) {
|
||||
cb(observer as Parameters<ObservableCallback>[0]);
|
||||
}
|
||||
snapshot = 9;
|
||||
for (const cb of gauge?.gaugeCallbacks ?? []) {
|
||||
cb(observer as Parameters<ObservableCallback>[0]);
|
||||
}
|
||||
expect(observations).toEqual([3, 9]);
|
||||
});
|
||||
|
||||
it("falls back to OTel globals when no provider supplied", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// The OTel API returns a no-op provider when no SDK is installed;
|
||||
// this exercises the default-branch wiring.
|
||||
const instruments = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
expect(instruments.transactionsTotal).toBeDefined();
|
||||
expect(instruments.tracer).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
import type {
|
||||
Counter,
|
||||
Histogram,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
ObservableGauge,
|
||||
Tracer,
|
||||
TracerProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
|
||||
/** Outcome attribute attached to every counter/histogram observation. */
|
||||
export type TwoPcOutcome = "committed" | "aborted" | "in_doubt";
|
||||
|
||||
/**
|
||||
* Config consumed by {@link createTwoPcInstruments}. The two
|
||||
* providers default to the OTel globals (which fall back to no-op
|
||||
* implementations when no SDK is installed) so the coordinator works
|
||||
* in test envs without ceremony.
|
||||
*/
|
||||
export interface TwoPcInstrumentsConfig {
|
||||
readonly meterProvider?: MeterProvider;
|
||||
readonly tracerProvider?: TracerProvider;
|
||||
/** Callback returning the current outstanding-prepared count for the gauge. */
|
||||
readonly outstandingPreparedSnapshot: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Frozen bundle of every OTel instrument the coordinator emits. The
|
||||
* names follow design doc §9.2:
|
||||
*
|
||||
* - `2pc.transaction` — span per transaction lifetime.
|
||||
* - `2pc.prepare` / `2pc.commit` / `2pc.rollback` — spans per
|
||||
* participant phase.
|
||||
* - `2pc.transactions_total` — counter labelled `{ outcome }`.
|
||||
* - `2pc.prepare_duration_ms` / `2pc.commit_duration_ms` — histograms
|
||||
* labelled `{ outcome }`.
|
||||
* - `2pc.outstanding_prepared` — observable gauge.
|
||||
*/
|
||||
export interface TwoPcInstruments {
|
||||
readonly meter: Meter;
|
||||
readonly tracer: Tracer;
|
||||
readonly transactionsTotal: Counter;
|
||||
readonly prepareDurationMs: Histogram;
|
||||
readonly commitDurationMs: Histogram;
|
||||
readonly outstandingPreparedGauge: ObservableGauge;
|
||||
}
|
||||
|
||||
const METER_NAME = "@suaveplan/api-2pc";
|
||||
|
||||
/**
|
||||
* Build the coordinator's instrument registry. Returns a frozen bundle
|
||||
* — every counter / histogram / gauge is constructed once so the
|
||||
* first measurement has zero allocation cost.
|
||||
*
|
||||
* The observable gauge is wired to `config.outstandingPreparedSnapshot`
|
||||
* at construction time. Operators who want a live admission-cap
|
||||
* dashboard register their SDK once at process boot and the
|
||||
* coordinator picks it up via the global provider.
|
||||
*/
|
||||
export function createTwoPcInstruments(
|
||||
config: TwoPcInstrumentsConfig
|
||||
): TwoPcInstruments {
|
||||
const meterProvider = config.meterProvider ?? metrics.getMeterProvider();
|
||||
const tracerProvider = config.tracerProvider ?? trace.getTracerProvider();
|
||||
const meter = meterProvider.getMeter(METER_NAME);
|
||||
const tracer = tracerProvider.getTracer(METER_NAME);
|
||||
const transactionsTotal = meter.createCounter("2pc.transactions_total");
|
||||
const prepareDurationMs = meter.createHistogram("2pc.prepare_duration_ms", {
|
||||
unit: "ms",
|
||||
});
|
||||
const commitDurationMs = meter.createHistogram("2pc.commit_duration_ms", {
|
||||
unit: "ms",
|
||||
});
|
||||
const outstandingPreparedGauge = meter.createObservableGauge(
|
||||
"2pc.outstanding_prepared"
|
||||
);
|
||||
outstandingPreparedGauge.addCallback((observer) => {
|
||||
observer.observe(config.outstandingPreparedSnapshot());
|
||||
});
|
||||
return Object.freeze({
|
||||
meter,
|
||||
tracer,
|
||||
transactionsTotal,
|
||||
prepareDurationMs,
|
||||
commitDurationMs,
|
||||
outstandingPreparedGauge,
|
||||
});
|
||||
}
|
||||
2
packages/api-stack/api/api-2pc/src/limits/index.ts
Normal file
2
packages/api-stack/api/api-2pc/src/limits/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export { OutstandingPreparedCap } from "./outstanding-cap.js";
|
||||
export { assertTimeoutBelowCeiling } from "./timeout-ceiling.js";
|
||||
104
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.md
Normal file
104
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# `limits/outstanding-cap`
|
||||
|
||||
## Purpose
|
||||
|
||||
Admission control for in-flight 2PC transactions. The coordinator
|
||||
holds a fixed-size budget of "outstanding prepared transactions"; once
|
||||
the budget is exhausted new `run2pc` calls refuse rather than queue.
|
||||
The default is 256 (design doc §14's open question, pinned by this
|
||||
package's spec) and operators can tune it via host config.
|
||||
|
||||
Refusing rather than queuing is intentional: Postgres holds row-level
|
||||
locks for every prepared transaction, so a backed-up coordinator
|
||||
would otherwise convert a transient overload into a global lock
|
||||
storm. Failing fast lets callers route to a degraded path
|
||||
(degraded-write, retry queue, etc.).
|
||||
|
||||
## Features
|
||||
|
||||
- `OutstandingPreparedCap` — synchronous counter with `admit()`,
|
||||
`release()`, `snapshot()`, `capacity()`.
|
||||
- Constructor validates that the cap is a positive integer (rejects
|
||||
zero, negatives, fractions, `NaN`).
|
||||
- `release()` is idempotent below zero — a buggy double-release in
|
||||
caller code cannot silently expand the real cap.
|
||||
- Synchronous-only — relies on the event-loop atomicity guarantee
|
||||
between `admit` / `release` pairs in JavaScript runtimes.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { OutstandingPreparedCap } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
const cap = new OutstandingPreparedCap(256);
|
||||
|
||||
cap.admit();
|
||||
try {
|
||||
// run the 2PC transaction
|
||||
} finally {
|
||||
cap.release();
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Wrap admission inside the coordinator's public entry to centralise the
|
||||
release in one `finally`. The snapshot fuels the OTel gauge:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
import { OutstandingPreparedCap } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
class MiniCoordinator {
|
||||
private readonly cap = new OutstandingPreparedCap(256);
|
||||
|
||||
snapshot() {
|
||||
return this.cap.snapshot();
|
||||
}
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
this.cap.admit();
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof TwoPcError &&
|
||||
err.twoPcCode === TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED
|
||||
) {
|
||||
// route to degraded path
|
||||
throw err;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.cap.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `class OutstandingPreparedCap`
|
||||
- `constructor(cap: number)` — throws
|
||||
`TWOPC_OUTSTANDING_LIMIT_EXCEEDED` on invalid `cap`.
|
||||
- `admit(): void` — throws
|
||||
`TWOPC_OUTSTANDING_LIMIT_EXCEEDED` when at capacity.
|
||||
- `release(): void` — idempotent below zero.
|
||||
- `snapshot(): number` — current outstanding count.
|
||||
- `capacity(): number` — configured cap.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Construction-time validation re-uses the same error code as the
|
||||
runtime cap-exceeded path. This is deliberate: operators tuning the
|
||||
cap via host config want one error string to alert on, regardless of
|
||||
whether the misconfiguration was a fractional value at startup or a
|
||||
real overload at steady state. The context object on the thrown
|
||||
`TwoPcError` distinguishes the two cases (`cap` only vs.
|
||||
`cap` + `outstanding`).
|
||||
|
||||
The class is intentionally tiny — no async, no internal queue, no
|
||||
external dependencies. Adding a queue here would re-introduce the
|
||||
lock-storm risk the cap exists to prevent.
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { OutstandingPreparedCap } from "./outstanding-cap.js";
|
||||
|
||||
describe("OutstandingPreparedCap", () => {
|
||||
it("admits up to the configured cap", { timeout: 5000 }, () => {
|
||||
const cap = new OutstandingPreparedCap(3);
|
||||
cap.admit();
|
||||
cap.admit();
|
||||
cap.admit();
|
||||
expect(cap.snapshot()).toBe(3);
|
||||
expect(cap.capacity()).toBe(3);
|
||||
});
|
||||
|
||||
it("rejects the (cap+1)th admission with TWOPC.OUTSTANDING_LIMIT_EXCEEDED", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cap = new OutstandingPreparedCap(2);
|
||||
cap.admit();
|
||||
cap.admit();
|
||||
try {
|
||||
cap.admit();
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("recovers as transactions release slots", { timeout: 5000 }, () => {
|
||||
const cap = new OutstandingPreparedCap(1);
|
||||
cap.admit();
|
||||
expect(() => cap.admit()).toThrow();
|
||||
cap.release();
|
||||
cap.admit();
|
||||
expect(cap.snapshot()).toBe(1);
|
||||
});
|
||||
|
||||
it("release below zero clamps at zero (defends against double-release)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cap = new OutstandingPreparedCap(2);
|
||||
cap.release();
|
||||
cap.release();
|
||||
expect(cap.snapshot()).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
0,
|
||||
-1,
|
||||
1.5,
|
||||
Number.NaN,
|
||||
])("constructor rejects non-positive-integer cap %p", (bad) => {
|
||||
expect(() => new OutstandingPreparedCap(bad as number)).toThrow();
|
||||
});
|
||||
});
|
||||
64
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.ts
Normal file
64
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
|
||||
/**
|
||||
* Admission counter for outstanding prepared transactions. Tracks how
|
||||
* many `run2pc` calls are between `admit()` and `release()`; new calls
|
||||
* past the cap raise `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` until an
|
||||
* in-flight transaction releases a slot.
|
||||
*
|
||||
* The implementation is single-threaded — `admit` and `release` are
|
||||
* synchronous balanced calls. Bun's event loop guarantees the call
|
||||
* pair is atomic relative to other tasks, so no mutex is required.
|
||||
*/
|
||||
export class OutstandingPreparedCap {
|
||||
private outstanding = 0;
|
||||
|
||||
public constructor(private readonly cap: number) {
|
||||
if (!Number.isInteger(cap) || cap <= 0) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED,
|
||||
`outstandingPreparedCap must be a positive integer; got ${cap}`,
|
||||
{ context: { cap } }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserve a slot. Throws
|
||||
* `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` when the cap is already at
|
||||
* capacity. On success the caller MUST eventually invoke
|
||||
* `release()` in a `finally` block.
|
||||
*/
|
||||
public admit(): void {
|
||||
if (this.outstanding >= this.cap) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED,
|
||||
`Outstanding prepared transactions at cap ${this.cap}`,
|
||||
{ context: { cap: this.cap, outstanding: this.outstanding } }
|
||||
);
|
||||
}
|
||||
this.outstanding += 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the slot reserved by a matching {@link admit}. Idempotent
|
||||
* against arithmetic underflow — repeated calls clamp at zero so
|
||||
* a buggy double-release in caller code does not allow the next
|
||||
* `admit` to silently exceed the real cap.
|
||||
*/
|
||||
public release(): void {
|
||||
if (this.outstanding > 0) {
|
||||
this.outstanding -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** Current outstanding count. Snapshot read for the OTel gauge. */
|
||||
public snapshot(): number {
|
||||
return this.outstanding;
|
||||
}
|
||||
|
||||
/** Configured cap. Diagnostics only. */
|
||||
public capacity(): number {
|
||||
return this.cap;
|
||||
}
|
||||
}
|
||||
77
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.md
Normal file
77
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.md
Normal file
@@ -0,0 +1,77 @@
|
||||
# `limits/timeout-ceiling`
|
||||
|
||||
## Purpose
|
||||
|
||||
Enforces the hard 30 000 ms ceiling on caller-supplied `timeoutMs`
|
||||
values. The check fires at the public entry point of `run2pc` so
|
||||
above-ceiling calls never contact a participant; the operational
|
||||
warning in design doc §9.2 is that long prepares hold row-level locks,
|
||||
so surfacing the misuse loudly is more useful than silently clamping.
|
||||
|
||||
This module is intentionally tiny — one function — because the
|
||||
behaviour is small but the gate is one of the most-asserted contracts
|
||||
in the spec.
|
||||
|
||||
## Features
|
||||
|
||||
- `assertTimeoutBelowCeiling(timeoutMs)` — throws
|
||||
`TWOPC_TIMEOUT_CEILING_EXCEEDED` for above-ceiling, non-positive,
|
||||
non-finite, or `NaN` inputs.
|
||||
- Re-uses the `TIMEOUT_CEILING_MS` constant from `options/` so the
|
||||
ceiling stays a single source of truth.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { assertTimeoutBelowCeiling } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
assertTimeoutBelowCeiling(5000); // ok
|
||||
assertTimeoutBelowCeiling(30001); // throws TWOPC_TIMEOUT_CEILING_EXCEEDED
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The function is intentionally side-effect-free so callers can wrap it
|
||||
in a per-tenant override path:
|
||||
|
||||
```ts
|
||||
import { TIMEOUT_CEILING_MS } from "@suaveplan/api-2pc/options";
|
||||
import { assertTimeoutBelowCeiling } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
function resolveTimeout(
|
||||
requested: number | undefined,
|
||||
tenantMax: number
|
||||
): number {
|
||||
const effective = requested ?? DEFAULT_TIMEOUT_MS;
|
||||
assertTimeoutBelowCeiling(effective);
|
||||
if (effective > tenantMax) {
|
||||
return tenantMax;
|
||||
}
|
||||
if (effective > TIMEOUT_CEILING_MS) {
|
||||
// unreachable given the assert above — included only to
|
||||
// illustrate that the ceiling acts as the strict upper bound.
|
||||
return TIMEOUT_CEILING_MS;
|
||||
}
|
||||
return effective;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `function assertTimeoutBelowCeiling(timeoutMs: number): void`
|
||||
- Throws `TwoPcError(TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED)` for any
|
||||
of: `NaN`, `Infinity`, `-Infinity`, values `<= 0`, values
|
||||
`> TIMEOUT_CEILING_MS`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The check rejects non-positive values for the same reason it rejects
|
||||
above-ceiling ones: an immediate or backwards-in-time deadline would
|
||||
collapse phase 1 to a single tick and the rollback path would fire
|
||||
before any participant could finish a real prepare. Treating that as
|
||||
the same misuse class as "30 minutes" is a deliberate ergonomic
|
||||
choice — operators get one error code to grep for.
|
||||
|
||||
`Number.isFinite` is used in preference to `Number.isNaN` because the
|
||||
former covers `Infinity` and `-Infinity` in addition to `NaN`. The
|
||||
explicit `<= 0` check then excludes zero and negative-finite values.
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { DEFAULT_TIMEOUT_MS, TIMEOUT_CEILING_MS } from "../options/options.js";
|
||||
import { assertTimeoutBelowCeiling } from "./timeout-ceiling.js";
|
||||
|
||||
function expectCeilingError(fn: () => void): TwoPcError {
|
||||
try {
|
||||
fn();
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
const e = err as TwoPcError;
|
||||
expect(e.twoPcCode).toBe(TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED);
|
||||
return e;
|
||||
}
|
||||
throw new Error("expected TWOPC_TIMEOUT_CEILING_EXCEEDED");
|
||||
}
|
||||
|
||||
describe("assertTimeoutBelowCeiling", () => {
|
||||
it("accepts the default timeoutMs (5000)", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
assertTimeoutBelowCeiling(DEFAULT_TIMEOUT_MS)
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts a value equal to the ceiling", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
assertTimeoutBelowCeiling(TIMEOUT_CEILING_MS)
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects timeoutMs above the ceiling", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() =>
|
||||
assertTimeoutBelowCeiling(TIMEOUT_CEILING_MS + 1)
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects zero", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() => assertTimeoutBelowCeiling(0));
|
||||
});
|
||||
|
||||
it("rejects negative values", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() => assertTimeoutBelowCeiling(-1));
|
||||
});
|
||||
|
||||
it("rejects NaN", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() => assertTimeoutBelowCeiling(Number.NaN));
|
||||
});
|
||||
|
||||
it("rejects Infinity", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() =>
|
||||
assertTimeoutBelowCeiling(Number.POSITIVE_INFINITY)
|
||||
);
|
||||
});
|
||||
});
|
||||
29
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.ts
Normal file
29
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { TIMEOUT_CEILING_MS } from "../options/options.js";
|
||||
|
||||
/**
|
||||
* Validate a caller-supplied `timeoutMs`. Per spec the coordinator
|
||||
* MUST refuse values above the hard ceiling (default 30 000 ms) at
|
||||
* call time — silently clamping would hide the misuse and let
|
||||
* long-held prepares regress production throughput.
|
||||
*
|
||||
* Negative or non-finite values are also refused — they would
|
||||
* collapse the deadline to "immediate" which is operationally
|
||||
* indistinguishable from a flaky participant.
|
||||
*/
|
||||
export function assertTimeoutBelowCeiling(timeoutMs: number): void {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED,
|
||||
`timeoutMs must be a positive finite number; got ${timeoutMs}`,
|
||||
{ context: { timeoutMs } }
|
||||
);
|
||||
}
|
||||
if (timeoutMs > TIMEOUT_CEILING_MS) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED,
|
||||
`timeoutMs ${timeoutMs} exceeds hard ceiling ${TIMEOUT_CEILING_MS}`,
|
||||
{ context: { timeoutMs, ceiling: TIMEOUT_CEILING_MS } }
|
||||
);
|
||||
}
|
||||
}
|
||||
9
packages/api-stack/api/api-2pc/src/options/index.ts
Normal file
9
packages/api-stack/api/api-2pc/src/options/index.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
type ResolvedTwoPcCoordinatorOptions,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
TIMEOUT_CEILING_MS,
|
||||
type TwoPcCoordinatorOptions,
|
||||
twoPcCoordinatorOptionsSchema,
|
||||
} from "./options.js";
|
||||
96
packages/api-stack/api/api-2pc/src/options/options.md
Normal file
96
packages/api-stack/api/api-2pc/src/options/options.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# `options/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the boundary-validated option shape consumed by
|
||||
`createTwoPcCoordinator`. Every tunable that affects coordinator
|
||||
behaviour (host instance id, outstanding-prepared cap, default phase-1
|
||||
timeout) lives here, with Zod validation at the seam and explicit
|
||||
defaults pinned to the design doc §9.2 numbers.
|
||||
|
||||
Centralising option resolution in one module keeps the coordinator
|
||||
free of conditional fallback logic — by the time it sees the resolved
|
||||
shape every field is concrete, so subsequent code never has to
|
||||
re-check `undefined`.
|
||||
|
||||
## Features
|
||||
|
||||
- `DEFAULT_TIMEOUT_MS = 5000` — phase-1 deadline default per design
|
||||
doc §9.2.
|
||||
- `TIMEOUT_CEILING_MS = 30000` — hard ceiling enforced at call time
|
||||
(the coordinator's `limits/timeout-ceiling` module imports this).
|
||||
- `DEFAULT_OUTSTANDING_PREPARED_CAP = 256` — admission-control default
|
||||
per design doc §14.
|
||||
- `twoPcCoordinatorOptionsSchema` — Zod schema, `strict()` so typos
|
||||
surface as Zod errors instead of being silently ignored.
|
||||
- `resolveTwoPcCoordinatorOptions(raw)` — validates and fills in
|
||||
defaults; returns the fully-populated `ResolvedTwoPcCoordinatorOptions`
|
||||
shape the coordinator consumes.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
} from "@suaveplan/api-2pc/options";
|
||||
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "host-01",
|
||||
});
|
||||
|
||||
console.log(resolved.defaultTimeoutMs === DEFAULT_TIMEOUT_MS); // true
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Override every default — useful for canary hosts where operators want
|
||||
a tighter admission cap and a longer prepare window:
|
||||
|
||||
```ts
|
||||
import { resolveTwoPcCoordinatorOptions } from "@suaveplan/api-2pc/options";
|
||||
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "canary-eu-west-1",
|
||||
outstandingPreparedCap: 64,
|
||||
defaultTimeoutMs: 15_000,
|
||||
});
|
||||
```
|
||||
|
||||
Calls supplying `defaultTimeoutMs` above `TIMEOUT_CEILING_MS` raise a
|
||||
Zod error at this boundary — the coordinator's runtime ceiling check
|
||||
in `limits/timeout-ceiling.ts` then surfaces the misuse on per-call
|
||||
overrides as well.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `const DEFAULT_TIMEOUT_MS: number` — 5000.
|
||||
- `const TIMEOUT_CEILING_MS: number` — 30000.
|
||||
- `const DEFAULT_OUTSTANDING_PREPARED_CAP: number` — 256.
|
||||
- `interface TwoPcCoordinatorOptions` — caller-supplied input shape.
|
||||
- `interface ResolvedTwoPcCoordinatorOptions` — internal shape after
|
||||
defaults.
|
||||
- `const twoPcCoordinatorOptionsSchema: z.ZodType` — strict-shape
|
||||
Zod schema.
|
||||
- `function resolveTwoPcCoordinatorOptions(raw): ResolvedTwoPcCoordinatorOptions`
|
||||
— validates and resolves defaults.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The schema uses `.strict()` so unknown keys fail validation rather than
|
||||
silently being dropped. The default for `defaultTimeoutMs` is applied
|
||||
post-parse with `?? DEFAULT_TIMEOUT_MS` rather than via
|
||||
`z.default(...)` so the schema continues to reject above-ceiling
|
||||
values regardless of whether the caller supplied the field — Zod's
|
||||
`default` would short-circuit `.max()` checks when the field is
|
||||
omitted, which is the wrong shape for a ceiling-enforced option.
|
||||
|
||||
The module deliberately does NOT re-throw Zod failures as
|
||||
`TwoPcError` — that's the responsibility of the coordinator's
|
||||
public entry points. Keeping this module pure of cross-package error
|
||||
mapping makes it reusable from tests that want a raw Zod surface for
|
||||
property-based shape testing.
|
||||
70
packages/api-stack/api/api-2pc/src/options/options.test.ts
Normal file
70
packages/api-stack/api/api-2pc/src/options/options.test.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
TIMEOUT_CEILING_MS,
|
||||
twoPcCoordinatorOptionsSchema,
|
||||
} from "./options.js";
|
||||
|
||||
describe("resolveTwoPcCoordinatorOptions", () => {
|
||||
it("applies all defaults when only required fields supplied", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "host-1",
|
||||
});
|
||||
expect(resolved.hostInstanceId).toBe("host-1");
|
||||
expect(resolved.defaultTimeoutMs).toBe(DEFAULT_TIMEOUT_MS);
|
||||
expect(resolved.outstandingPreparedCap).toBe(
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves explicit overrides", { timeout: 5000 }, () => {
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "h",
|
||||
defaultTimeoutMs: 12345,
|
||||
outstandingPreparedCap: 4,
|
||||
});
|
||||
expect(resolved.defaultTimeoutMs).toBe(12345);
|
||||
expect(resolved.outstandingPreparedCap).toBe(4);
|
||||
});
|
||||
|
||||
it("rejects defaultTimeoutMs above the ceiling at the schema boundary", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "h",
|
||||
defaultTimeoutMs: TIMEOUT_CEILING_MS + 1,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects an empty hostInstanceId", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
resolveTwoPcCoordinatorOptions({ hostInstanceId: "" })
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects unknown keys (strict schema)", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
twoPcCoordinatorOptionsSchema.parse({
|
||||
hostInstanceId: "h",
|
||||
bogus: 1,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects a non-integer outstandingPreparedCap", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "h",
|
||||
outstandingPreparedCap: 1.5,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
84
packages/api-stack/api/api-2pc/src/options/options.ts
Normal file
84
packages/api-stack/api/api-2pc/src/options/options.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
import z from "zod";
|
||||
|
||||
/**
|
||||
* Default phase-1 deadline. Design doc §9.2 anchors at 5 000 ms — long
|
||||
* enough for a real cross-database prepare, short enough that
|
||||
* row-level locks held by `PREPARE TRANSACTION` do not pile up.
|
||||
*/
|
||||
export const DEFAULT_TIMEOUT_MS = 5000;
|
||||
|
||||
/**
|
||||
* Hard ceiling on `timeoutMs`. Calls above this are refused at the
|
||||
* boundary rather than silently clamped — operators want to see the
|
||||
* misuse loudly because long prepares hold row-level locks.
|
||||
*/
|
||||
export const TIMEOUT_CEILING_MS = 30000;
|
||||
|
||||
/**
|
||||
* Default cap on outstanding prepared transactions per coordinator.
|
||||
* Design doc §14 left this open; this is the spec'd default and is
|
||||
* operator-tunable. New `run2pc` calls past the cap fail with
|
||||
* `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` until in-flight transactions
|
||||
* complete.
|
||||
*/
|
||||
export const DEFAULT_OUTSTANDING_PREPARED_CAP = 256;
|
||||
|
||||
/**
|
||||
* Caller-tunable options handed to `createTwoPcCoordinator`. Every
|
||||
* field is optional; defaults are applied by `resolveOptions`.
|
||||
*/
|
||||
export interface TwoPcCoordinatorOptions {
|
||||
/** Stable host id pre-tagged on every prepared transaction's gid. */
|
||||
readonly hostInstanceId: string;
|
||||
/**
|
||||
* Override of `DEFAULT_OUTSTANDING_PREPARED_CAP`. Must be `>= 1`.
|
||||
*/
|
||||
readonly outstandingPreparedCap?: number;
|
||||
/**
|
||||
* Override of `DEFAULT_TIMEOUT_MS`. Must satisfy
|
||||
* `1 <= value <= TIMEOUT_CEILING_MS`.
|
||||
*/
|
||||
readonly defaultTimeoutMs?: number;
|
||||
}
|
||||
|
||||
/** Strict-shape Zod schema for {@link TwoPcCoordinatorOptions}. */
|
||||
export const twoPcCoordinatorOptionsSchema = z
|
||||
.object({
|
||||
hostInstanceId: z.string().min(1),
|
||||
outstandingPreparedCap: z.number().int().positive().optional(),
|
||||
defaultTimeoutMs: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.max(TIMEOUT_CEILING_MS)
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
/**
|
||||
* Resolved option shape — every field present, defaults applied. The
|
||||
* coordinator only ever reads this shape; the optional fields exist on
|
||||
* the input shape only.
|
||||
*/
|
||||
export interface ResolvedTwoPcCoordinatorOptions {
|
||||
readonly hostInstanceId: string;
|
||||
readonly outstandingPreparedCap: number;
|
||||
readonly defaultTimeoutMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply defaults and Zod validation. Throws the raw `z.ZodError` so
|
||||
* callers can repackage it into a domain error if they prefer; the
|
||||
* coordinator boundary wraps this in a `TwoPcError` itself.
|
||||
*/
|
||||
export function resolveTwoPcCoordinatorOptions(
|
||||
raw: TwoPcCoordinatorOptions
|
||||
): ResolvedTwoPcCoordinatorOptions {
|
||||
const parsed = twoPcCoordinatorOptionsSchema.parse(raw);
|
||||
return {
|
||||
hostInstanceId: parsed.hostInstanceId,
|
||||
outstandingPreparedCap:
|
||||
parsed.outstandingPreparedCap ?? DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
defaultTimeoutMs: parsed.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
};
|
||||
}
|
||||
7
packages/api-stack/api/api-2pc/src/persistence/index.ts
Normal file
7
packages/api-stack/api/api-2pc/src/persistence/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
type LedgerStatement,
|
||||
type TwoPcLedgerRow,
|
||||
TwoPcLedgerStore,
|
||||
} from "./persistence.js";
|
||||
114
packages/api-stack/api/api-2pc/src/persistence/persistence.md
Normal file
114
packages/api-stack/api/api-2pc/src/persistence/persistence.md
Normal file
@@ -0,0 +1,114 @@
|
||||
# `persistence/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Persists every coordinator state transition into the
|
||||
`suaveplan_2pc_transactions` ledger table. The table is owned by
|
||||
`@suaveplan/api-host` (migration id 2, `saga-2pc-reserved`); this
|
||||
module only reads and writes rows so the host stays the single source
|
||||
of truth for schema evolution.
|
||||
|
||||
The ledger is what makes 2PC durable across coordinator restarts.
|
||||
Without it, a host that crashes mid-prepare would lose track of which
|
||||
participants were `PREPARE TRANSACTION`'d and could not safely choose
|
||||
between commit and rollback on the next boot.
|
||||
|
||||
## Features
|
||||
|
||||
- `TwoPcLedgerStore` — CRUD facade over `suaveplan_2pc_transactions`
|
||||
with statements for every state transition (`insert`,
|
||||
`markAllVotedCommit`, `markAborting`, `markCommitted`, `markAborted`,
|
||||
`markInDoubt`).
|
||||
- `findById` — single-row lookup for state inspection.
|
||||
- `findRecoverable` — scan returning every non-terminal row for the
|
||||
current host instance id. Drives `recovery/`.
|
||||
- `TwoPcLedgerStore.decodeParticipants` — static helper that parses the
|
||||
`participants_json` blob back into a typed array.
|
||||
- Structural `LedgerDb` / `LedgerStatement` interfaces so the store
|
||||
works with `bun:sqlite`, `@suaveplan/db-sqlite`'s thin wrapper, and
|
||||
`better-sqlite3` without any concrete dependency.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { Database } from "bun:sqlite";
|
||||
import { TwoPcLedgerStore } from "@suaveplan/api-2pc/persistence";
|
||||
|
||||
const db = new Database(":memory:");
|
||||
// Schema is created by api-host's migration runner — see migrations.ts
|
||||
// migration id 2 (`saga-2pc-reserved`). In tests, run the host runner
|
||||
// against the same database.
|
||||
|
||||
const store = new TwoPcLedgerStore(db as never);
|
||||
store.insert({
|
||||
id: "tx-1",
|
||||
coordinator: "host-instance-01",
|
||||
participants: [
|
||||
{ participantId: "billing", voterMode: "writer", vote: null },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Walk the recovery candidates and decode each blob — the typical shape
|
||||
used inside `recovery/recovery.ts`:
|
||||
|
||||
```ts
|
||||
import { TwoPcLedgerStore } from "@suaveplan/api-2pc/persistence";
|
||||
|
||||
function listRecoverable(store: TwoPcLedgerStore, hostId: string) {
|
||||
const rows = store.findRecoverable(hostId);
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
state: row.state,
|
||||
participants: TwoPcLedgerStore.decodeParticipants(row),
|
||||
}));
|
||||
}
|
||||
```
|
||||
|
||||
The static `decodeParticipants` is deliberately exposed so callers
|
||||
working with raw rows (e.g. ad-hoc admin scripts) do not have to
|
||||
recreate the JSON contract themselves.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface LedgerStatement` / `interface LedgerDb` — structural
|
||||
SQLite-shaped views.
|
||||
- `interface TwoPcLedgerRow` — exact row shape returned by `findById`
|
||||
/ `findRecoverable`.
|
||||
- `interface LedgerParticipantEntry` — per-participant entry in
|
||||
`participants_json`.
|
||||
- `class TwoPcLedgerStore`
|
||||
- `constructor(db: LedgerDb)`
|
||||
- `insert({ id, coordinator, participants })`
|
||||
- `markAllVotedCommit({ id, participants, preparedAt })`
|
||||
- `markAborting({ id, participants })`
|
||||
- `markCommitted(id, committedAt)`
|
||||
- `markAborted(id, abortedAt)`
|
||||
- `markInDoubt(id)`
|
||||
- `findById(id): TwoPcLedgerRow | null`
|
||||
- `findRecoverable(coordinator): readonly TwoPcLedgerRow[]`
|
||||
- `static decodeParticipants(row): readonly LedgerParticipantEntry[]`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Every UPDATE uses a single statement with the new `state` value
|
||||
matched against `TwoPcCoordinatorState`'s string-literal union — the
|
||||
`satisfies TwoPcCoordinatorState` annotation catches typos at compile
|
||||
time. The store does NOT enforce a state machine on top of the table;
|
||||
that responsibility belongs to the coordinator. Keeping the
|
||||
persistence layer state-machine-free lets recovery use the same store
|
||||
without conflating "what changed in this lifetime" with "what the row
|
||||
looked like after a crash".
|
||||
|
||||
The `participants_json` blob is stored as a JSON string rather than a
|
||||
side table — the coordinator never queries by participant id, only by
|
||||
transaction id, so a JSON blob has zero indexing cost and survives
|
||||
schema evolution painlessly. `decodeParticipants` returns an empty
|
||||
array when the blob is not an array — defensive against rows
|
||||
hand-written by ad-hoc admin scripts.
|
||||
@@ -0,0 +1,174 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
TwoPcLedgerStore,
|
||||
} from "./persistence.js";
|
||||
|
||||
function makeDb(): LedgerDb {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
return db;
|
||||
}
|
||||
|
||||
const SAMPLE_PARTICIPANTS: ReadonlyArray<LedgerParticipantEntry> = [
|
||||
{ participantId: "billing", voterMode: "writer", vote: null },
|
||||
{ participantId: "inventory", voterMode: "writer", vote: null },
|
||||
];
|
||||
|
||||
describe("TwoPcLedgerStore", () => {
|
||||
it("inserts a new row in voting state", { timeout: 5000 }, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-1",
|
||||
coordinator: "host-1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
const row = store.findById("tx-1");
|
||||
expect(row).not.toBeNull();
|
||||
expect(row?.state).toBe("voting");
|
||||
expect(row?.coordinator).toBe("host-1");
|
||||
expect(row?.preparedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("marks the row committing once all participants voted commit", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-2",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markAllVotedCommit({
|
||||
id: "tx-2",
|
||||
participants: SAMPLE_PARTICIPANTS.map((p) => ({
|
||||
...p,
|
||||
vote: "commit",
|
||||
})),
|
||||
preparedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
const row = store.findById("tx-2");
|
||||
expect(row?.state).toBe("committing");
|
||||
expect(row?.preparedAt).toBe("2026-05-26T00:00:00.000Z");
|
||||
const decoded = TwoPcLedgerStore.decodeParticipants(
|
||||
row as NonNullable<typeof row>
|
||||
);
|
||||
expect(decoded[0]?.vote).toBe("commit");
|
||||
});
|
||||
|
||||
it("transitions through aborting then committed and aborted final states", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-3",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markAborting({ id: "tx-3", participants: SAMPLE_PARTICIPANTS });
|
||||
expect(store.findById("tx-3")?.state).toBe("aborting");
|
||||
|
||||
store.markCommitted("tx-3", "2026-05-26T00:01:00.000Z");
|
||||
expect(store.findById("tx-3")?.committedAt).toBe(
|
||||
"2026-05-26T00:01:00.000Z"
|
||||
);
|
||||
|
||||
store.markAborted("tx-3", "2026-05-26T00:02:00.000Z");
|
||||
expect(store.findById("tx-3")?.abortedAt).toBe(
|
||||
"2026-05-26T00:02:00.000Z"
|
||||
);
|
||||
});
|
||||
|
||||
it("marks rows in-doubt", { timeout: 5000 }, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-4",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markInDoubt("tx-4");
|
||||
expect(store.findById("tx-4")?.state).toBe("in_doubt");
|
||||
});
|
||||
|
||||
it("findById returns null for missing rows", { timeout: 5000 }, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
expect(store.findById("never")).toBeNull();
|
||||
});
|
||||
|
||||
it("findRecoverable returns only non-terminal rows for matching coordinator", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "a",
|
||||
coordinator: "h1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.insert({
|
||||
id: "b",
|
||||
coordinator: "h1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markCommitted("b", "2026-05-26T00:00:00.000Z");
|
||||
store.insert({
|
||||
id: "c",
|
||||
coordinator: "h2",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.insert({
|
||||
id: "d",
|
||||
coordinator: "h1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markInDoubt("d");
|
||||
|
||||
const recoverable = store.findRecoverable("h1");
|
||||
const ids = recoverable.map((r) => r.id).sort();
|
||||
expect(ids).toEqual(["a", "d"]);
|
||||
});
|
||||
|
||||
it("decodeParticipants returns an empty array on non-array JSON", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const decoded = TwoPcLedgerStore.decodeParticipants({
|
||||
id: "x",
|
||||
coordinator: "h",
|
||||
state: "voting",
|
||||
participantsJson: '{"not":"array"}',
|
||||
preparedAt: null,
|
||||
committedAt: null,
|
||||
abortedAt: null,
|
||||
});
|
||||
expect(decoded).toEqual([]);
|
||||
});
|
||||
|
||||
it("decodeParticipants round-trips real entries", { timeout: 5000 }, () => {
|
||||
const decoded = TwoPcLedgerStore.decodeParticipants({
|
||||
id: "x",
|
||||
coordinator: "h",
|
||||
state: "committing",
|
||||
participantsJson: JSON.stringify([
|
||||
{
|
||||
participantId: "p",
|
||||
voterMode: "writer",
|
||||
vote: "commit",
|
||||
},
|
||||
]),
|
||||
preparedAt: null,
|
||||
committedAt: null,
|
||||
abortedAt: null,
|
||||
});
|
||||
expect(decoded).toHaveLength(1);
|
||||
expect(decoded[0]?.vote).toBe("commit");
|
||||
});
|
||||
});
|
||||
228
packages/api-stack/api/api-2pc/src/persistence/persistence.ts
Normal file
228
packages/api-stack/api/api-2pc/src/persistence/persistence.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
import type { TwoPcCoordinatorState } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Structural view of a SQLite-shaped statement returned by
|
||||
* `LedgerDb.prepare(sql)`. Matches `bun:sqlite`'s `Statement` surface
|
||||
* narrowly: only the methods this module needs are exposed.
|
||||
*/
|
||||
export interface LedgerStatement {
|
||||
run(...params: readonly unknown[]): { changes: number } | unknown;
|
||||
all<T = Record<string, unknown>>(
|
||||
...params: readonly unknown[]
|
||||
): readonly T[];
|
||||
get<T = Record<string, unknown>>(
|
||||
...params: readonly unknown[]
|
||||
): T | undefined | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural SQLite-like database handle. Compatible with `bun:sqlite`,
|
||||
* `@suaveplan/db-sqlite`'s thin wrapper, and `better-sqlite3`. The
|
||||
* coordinator only ever sees this interface — concrete drivers stay
|
||||
* outside the package boundary.
|
||||
*/
|
||||
export interface LedgerDb {
|
||||
prepare(sql: string): LedgerStatement;
|
||||
/** Execute a single DDL or DML statement with no parameters. */
|
||||
run(sql: string): unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* One row of `suaveplan_2pc_transactions`. The schema is owned by
|
||||
* `@suaveplan/api-host` (see `plugin-ledger/migrations.ts` migration
|
||||
* id 2, `saga-2pc-reserved`). This row shape mirrors the DDL fields
|
||||
* verbatim — keep them in lockstep when the host schema evolves.
|
||||
*/
|
||||
export interface TwoPcLedgerRow {
|
||||
readonly id: string;
|
||||
readonly coordinator: string;
|
||||
readonly state: TwoPcCoordinatorState;
|
||||
readonly participantsJson: string;
|
||||
readonly preparedAt: string | null;
|
||||
readonly committedAt: string | null;
|
||||
readonly abortedAt: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-participant entry inside the `participants_json` blob. Captures
|
||||
* the vote and the voter mode so recovery can re-derive whether to
|
||||
* send commit/rollback SQL on boot.
|
||||
*/
|
||||
export interface LedgerParticipantEntry {
|
||||
readonly participantId: string;
|
||||
readonly voterMode: "writer" | "read-only";
|
||||
readonly vote: TwoPcVote | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistence facade over `suaveplan_2pc_transactions`. The schema is
|
||||
* created by `api-host`'s migration runner; this module only reads and
|
||||
* writes rows.
|
||||
*/
|
||||
export class TwoPcLedgerStore {
|
||||
public constructor(private readonly db: LedgerDb) {}
|
||||
|
||||
/** Insert a new row in the `voting` state. */
|
||||
public insert(input: {
|
||||
id: string;
|
||||
coordinator: string;
|
||||
participants: ReadonlyArray<LedgerParticipantEntry>;
|
||||
}): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO suaveplan_2pc_transactions
|
||||
(id, coordinator, state, participants_json, prepared_at, committed_at, aborted_at)
|
||||
VALUES (?, ?, ?, ?, NULL, NULL, NULL)`
|
||||
)
|
||||
.run(
|
||||
input.id,
|
||||
input.coordinator,
|
||||
"voting" satisfies TwoPcCoordinatorState,
|
||||
JSON.stringify(input.participants)
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark the row as `committing` and stamp `prepared_at`. */
|
||||
public markAllVotedCommit(input: {
|
||||
id: string;
|
||||
participants: ReadonlyArray<LedgerParticipantEntry>;
|
||||
preparedAt: string;
|
||||
}): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, participants_json = ?, prepared_at = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(
|
||||
"committing" satisfies TwoPcCoordinatorState,
|
||||
JSON.stringify(input.participants),
|
||||
input.preparedAt,
|
||||
input.id
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark the row as `aborting` after a vote_abort or timeout. */
|
||||
public markAborting(input: {
|
||||
id: string;
|
||||
participants: ReadonlyArray<LedgerParticipantEntry>;
|
||||
}): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, participants_json = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(
|
||||
"aborting" satisfies TwoPcCoordinatorState,
|
||||
JSON.stringify(input.participants),
|
||||
input.id
|
||||
);
|
||||
}
|
||||
|
||||
/** Final-state stamp once every participant acknowledged commit. */
|
||||
public markCommitted(id: string, committedAt: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, committed_at = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run("committed" satisfies TwoPcCoordinatorState, committedAt, id);
|
||||
}
|
||||
|
||||
/** Final-state stamp once every participant acknowledged rollback. */
|
||||
public markAborted(id: string, abortedAt: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, aborted_at = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run("aborted" satisfies TwoPcCoordinatorState, abortedAt, id);
|
||||
}
|
||||
|
||||
/** Mark a transaction as in-doubt (commit-prepared / rollback-prepared SQL failed). */
|
||||
public markInDoubt(id: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run("in_doubt" satisfies TwoPcCoordinatorState, id);
|
||||
}
|
||||
|
||||
/** Lookup a single row by id. Returns `null` when none exists. */
|
||||
public findById(id: string): TwoPcLedgerRow | null {
|
||||
const row = this.db
|
||||
.prepare(
|
||||
`SELECT id, coordinator, state, participants_json,
|
||||
prepared_at, committed_at, aborted_at
|
||||
FROM suaveplan_2pc_transactions WHERE id = ?`
|
||||
)
|
||||
.get<{
|
||||
id: string;
|
||||
coordinator: string;
|
||||
state: TwoPcCoordinatorState;
|
||||
participants_json: string;
|
||||
prepared_at: string | null;
|
||||
committed_at: string | null;
|
||||
aborted_at: string | null;
|
||||
}>(id);
|
||||
if (row === null || row === undefined) return null;
|
||||
return {
|
||||
id: row.id,
|
||||
coordinator: row.coordinator,
|
||||
state: row.state,
|
||||
participantsJson: row.participants_json,
|
||||
preparedAt: row.prepared_at,
|
||||
committedAt: row.committed_at,
|
||||
abortedAt: row.aborted_at,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Recovery scan — every row whose state is non-terminal AND whose
|
||||
* coordinator matches the provided host instance id. These rows
|
||||
* are the candidates the recovery runner must resolve on boot.
|
||||
*/
|
||||
public findRecoverable(coordinator: string): readonly TwoPcLedgerRow[] {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT id, coordinator, state, participants_json,
|
||||
prepared_at, committed_at, aborted_at
|
||||
FROM suaveplan_2pc_transactions
|
||||
WHERE coordinator = ?
|
||||
AND state IN ('voting','committing','aborting','in_doubt')`
|
||||
)
|
||||
.all<{
|
||||
id: string;
|
||||
coordinator: string;
|
||||
state: TwoPcCoordinatorState;
|
||||
participants_json: string;
|
||||
prepared_at: string | null;
|
||||
committed_at: string | null;
|
||||
aborted_at: string | null;
|
||||
}>(coordinator);
|
||||
return rows.map((r) => ({
|
||||
id: r.id,
|
||||
coordinator: r.coordinator,
|
||||
state: r.state,
|
||||
participantsJson: r.participants_json,
|
||||
preparedAt: r.prepared_at,
|
||||
committedAt: r.committed_at,
|
||||
abortedAt: r.aborted_at,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Decode the `participants_json` blob back into an array. */
|
||||
public static decodeParticipants(
|
||||
row: TwoPcLedgerRow
|
||||
): ReadonlyArray<LedgerParticipantEntry> {
|
||||
const parsed = JSON.parse(row.participantsJson) as unknown;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed as ReadonlyArray<LedgerParticipantEntry>;
|
||||
}
|
||||
}
|
||||
94
packages/api-stack/api/api-2pc/src/phases/commit.md
Normal file
94
packages/api-stack/api/api-2pc/src/phases/commit.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# `phases/commit`
|
||||
|
||||
## Purpose
|
||||
|
||||
Phase-2 commit for a single participant. Once every participant has
|
||||
returned `commit` from phase 1 and the coordinator has persisted the
|
||||
`all-voted-commit` state to the ledger, this module emits
|
||||
`COMMIT PREPARED '<gid>'` against each writer participant and records
|
||||
`2pc.commit_duration_ms`.
|
||||
|
||||
Phase 2 is intentionally unbounded by timeout — the protocol cannot
|
||||
tolerate giving up on a commit-prepared mid-flight, because doing so
|
||||
would leave the system in `in_doubt` state and require the recovery
|
||||
runner to resolve it on the next boot.
|
||||
|
||||
## Features
|
||||
|
||||
- Skips participants that did not actually prepare (`outcome.prepared
|
||||
=== false`) — read-only voters and abort-voters never see commit
|
||||
SQL.
|
||||
- Emits a `2pc.commit` span with `tx.id`, `participant.id`, and `gid`
|
||||
attributes.
|
||||
- Records `2pc.commit_duration_ms` tagged with `outcome: "committed"`
|
||||
on success.
|
||||
- Failure preserves the underlying cause on a thrown
|
||||
`TwoPcError(TWOPC_COMMIT_FAILED)`; the coordinator escalates to
|
||||
`TWOPC_IN_DOUBT` once it has tried every participant.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { commitPhaseTwo } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
await commitPhaseTwo({
|
||||
txId,
|
||||
participant,
|
||||
outcome: preparedOutcome,
|
||||
instruments,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The coordinator collects per-participant failures rather than
|
||||
short-circuiting — every participant gets attempted exactly once so
|
||||
operators can see the full picture in the ledger:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
import { commitPhaseTwo } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const failures: TwoPcError[] = [];
|
||||
for (const outcome of preparedOutcomes) {
|
||||
if (!outcome.prepared) continue;
|
||||
try {
|
||||
await commitPhaseTwo({
|
||||
txId,
|
||||
participant: registry.get(outcome.participantId)!,
|
||||
outcome,
|
||||
instruments,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) failures.push(err);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
// ledger.markInDoubt(txId); recovery resolves on next boot
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface CommitPhaseInput`
|
||||
- `txId: string`
|
||||
- `participant: RegisteredTwoPcParticipant`
|
||||
- `outcome: ParticipantPrepareOutcome` — the phase-1 result.
|
||||
- `instruments: TwoPcInstruments`
|
||||
- `function commitPhaseTwo(input): Promise<void>`
|
||||
- no-op for `outcome.prepared === false`
|
||||
- throws `TwoPcError(TWOPC_COMMIT_FAILED)` on SQL failure.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The gid stored on the prepare outcome is the single source of truth —
|
||||
this module never rebuilds it. Reusing the prepare-time value
|
||||
defends against the unlikely case where a host-config change between
|
||||
phases would have produced a different gid: the SQL the coordinator
|
||||
needs to send is exactly the one it prepared with.
|
||||
|
||||
The SQL escape (`replace(/'/g, "''")`) matches the prepare-phase
|
||||
treatment so a gid containing the unlikely quote character would
|
||||
still produce valid SQL. The escape is cheap; the defence-in-depth
|
||||
costs nothing on the happy path.
|
||||
98
packages/api-stack/api/api-2pc/src/phases/commit.test.ts
Normal file
98
packages/api-stack/api/api-2pc/src/phases/commit.test.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { createTwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { commitPhaseTwo } from "./commit.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
|
||||
const INSTRUMENTS = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
|
||||
function makeParticipant(opts: { fail?: Error } = {}) {
|
||||
const calls: string[] = [];
|
||||
const p: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.fail) throw opts.fail;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
return { participant: p, calls };
|
||||
}
|
||||
|
||||
const PREPARED_OUTCOME: ParticipantPrepareOutcome = {
|
||||
participantId: "billing",
|
||||
vote: "commit",
|
||||
prepared: true,
|
||||
gid: "2pc-h-t-billing",
|
||||
};
|
||||
|
||||
describe("commitPhaseTwo", () => {
|
||||
it("issues COMMIT PREPARED against a prepared writer", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: PREPARED_OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls[0]).toBe("COMMIT PREPARED '2pc-h-t-billing'");
|
||||
});
|
||||
|
||||
it("is a no-op when outcome.prepared === false", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: { ...PREPARED_OUTCOME, prepared: false, gid: "" },
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("wraps a SQL throw into TWOPC.COMMIT_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error("commit failed");
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
try {
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: PREPARED_OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.COMMIT_FAILED
|
||||
);
|
||||
expect((err as TwoPcError).cause).toBe(cause);
|
||||
}
|
||||
});
|
||||
|
||||
it("escapes single quotes inside the gid before interpolation", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: { ...PREPARED_OUTCOME, gid: "a'b" },
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls[0]).toBe("COMMIT PREPARED 'a''b'");
|
||||
});
|
||||
});
|
||||
66
packages/api-stack/api/api-2pc/src/phases/commit.ts
Normal file
66
packages/api-stack/api/api-2pc/src/phases/commit.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { TwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
|
||||
/**
|
||||
* Input to {@link commitPhaseTwo}. The participant entry is consulted
|
||||
* for the connection; the outcome carries the gid.
|
||||
*/
|
||||
export interface CommitPhaseInput {
|
||||
readonly txId: string;
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
readonly outcome: ParticipantPrepareOutcome;
|
||||
readonly instruments: TwoPcInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 2 commit for a single participant. Issues
|
||||
* `COMMIT PREPARED '<gid>'` and records the commit duration.
|
||||
*
|
||||
* Read-only voters and writers that did not actually prepare (vote
|
||||
* was `abort`) are skipped — the caller only routes prepared writers
|
||||
* through this function.
|
||||
*
|
||||
* Failures throw `TWOPC_COMMIT_FAILED` with the cause preserved. The
|
||||
* coordinator escalates to `TWOPC_IN_DOUBT` once it has tried every
|
||||
* participant.
|
||||
*/
|
||||
export async function commitPhaseTwo(input: CommitPhaseInput): Promise<void> {
|
||||
const { txId, participant, outcome, instruments } = input;
|
||||
if (!outcome.prepared) return;
|
||||
const span = instruments.tracer.startSpan("2pc.commit", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.id": participant.id,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
});
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`COMMIT PREPARED '${outcome.gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (cause) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.COMMIT_FAILED,
|
||||
`COMMIT PREPARED failed for ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
participantId: participant.id,
|
||||
txId,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
instruments.commitDurationMs.record(Date.now() - t0, {
|
||||
outcome: "committed",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
85
packages/api-stack/api/api-2pc/src/phases/gid.md
Normal file
85
packages/api-stack/api/api-2pc/src/phases/gid.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# `phases/gid`
|
||||
|
||||
## Purpose
|
||||
|
||||
Builds and parses the global transaction id (gid) string passed to
|
||||
Postgres' `PREPARE TRANSACTION '<gid>'`. The format embeds the host
|
||||
instance id so the recovery scanner can filter `pg_prepared_xacts` to
|
||||
only the rows this coordinator owns — without that anchor, two
|
||||
coordinators sharing a Postgres host would clobber each other's
|
||||
recovery decisions.
|
||||
|
||||
The module is intentionally pure: no SQL, no client. Centralising the
|
||||
gid format here makes the recovery filter and the per-participant
|
||||
prepare path share a single source of truth.
|
||||
|
||||
## Features
|
||||
|
||||
- `buildGid(hostInstanceId, txId, participantId)` — produces
|
||||
`2pc-<host>-<tx>-<participant>`.
|
||||
- `parseGid(gid, hostInstanceId)` — extracts `{ txId, participantId }`
|
||||
for gids written by the supplied host; returns `null` otherwise so
|
||||
recovery skips foreign prepared transactions silently. The
|
||||
hostInstanceId is supplied by the caller (the recovery runner)
|
||||
because the host id itself may contain hyphens and the parser cannot
|
||||
recover it unambiguously from the gid alone.
|
||||
- `gidPrefixFor(hostInstanceId)` — the prefix the recovery SQL filter
|
||||
uses; centralised so SQL and parser never drift.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { buildGid, parseGid } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const gid = buildGid("host-01", "tx-abc", "billing");
|
||||
// "2pc-host-01-tx-abc-billing"
|
||||
|
||||
const parts = parseGid(gid, "host-01");
|
||||
// { txId: "tx-abc", participantId: "billing" }
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Recovery uses the prefix filter to ignore prepared transactions
|
||||
written by other components sharing the database:
|
||||
|
||||
```ts
|
||||
import { gidPrefixFor, parseGid } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const prefix = gidPrefixFor("host-01");
|
||||
const rows = await db.run<{ gid: string }>(
|
||||
`SELECT gid FROM pg_prepared_xacts WHERE gid LIKE $1`,
|
||||
[`${prefix}%`]
|
||||
);
|
||||
for (const row of rows) {
|
||||
const parts = parseGid(row.gid, "host-01");
|
||||
if (parts === null) continue; // foreign prepared transaction
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `buildGid(hostInstanceId: string, txId: string, participantId: string): string`
|
||||
- `parseGid(gid: string, hostInstanceId: string): { txId, participantId } | null`
|
||||
- `gidPrefixFor(hostInstanceId: string): string`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`parseGid` takes the caller-supplied `hostInstanceId` and strips the
|
||||
prefix `2pc-<host>-` before extracting `txId` and `participantId`.
|
||||
The host instance id is supplied explicitly because hyphens are
|
||||
allowed inside it (production hosts often look like `host-eu-west-1`)
|
||||
and the parser cannot recover the boundary from the gid alone. The
|
||||
recovery runner always knows its own host id, so this constraint is
|
||||
ergonomic in practice.
|
||||
|
||||
Participant ids are constrained by the registry (and by host config)
|
||||
to not contain hyphens, so the LAST hyphen after the prefix
|
||||
unambiguously delimits the `participantId` from the `txId`.
|
||||
|
||||
The function returns `null` rather than throwing on unrecognised gids
|
||||
because the recovery scan iterates over rows that may include prepared
|
||||
transactions written by code outside this package's control — Postgres
|
||||
has no participant-tag column, only the gid string itself, so the
|
||||
shared-database scenario is a legitimate concern.
|
||||
58
packages/api-stack/api/api-2pc/src/phases/gid.test.ts
Normal file
58
packages/api-stack/api/api-2pc/src/phases/gid.test.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { buildGid, gidPrefixFor, parseGid } from "./gid.js";
|
||||
|
||||
describe("buildGid", () => {
|
||||
it("produces the documented prefix-host-tx-participant format", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(buildGid("h1", "tx-abc", "billing")).toBe(
|
||||
"2pc-h1-tx-abc-billing"
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves hyphens in the host instance id", { timeout: 5000 }, () => {
|
||||
expect(buildGid("host-01", "tx", "billing")).toBe(
|
||||
"2pc-host-01-tx-billing"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseGid", () => {
|
||||
it("round-trips a freshly-built gid", { timeout: 5000 }, () => {
|
||||
const gid = buildGid("host-01", "tx-uuid-with-hyphens", "billing");
|
||||
const parts = parseGid(gid, "host-01");
|
||||
expect(parts).toEqual({
|
||||
txId: "tx-uuid-with-hyphens",
|
||||
participantId: "billing",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null for gids without the 2pc- prefix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseGid("foreign-gid", "h")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for gids written by another host", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const gid = buildGid("host-other", "tx", "billing");
|
||||
expect(parseGid(gid, "host-mine")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when there is no participant suffix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseGid("2pc-h-onlyhost", "h")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null on an empty txId", { timeout: 5000 }, () => {
|
||||
expect(parseGid("2pc-h--billing", "h")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("gidPrefixFor", () => {
|
||||
it("returns the recovery-filter prefix", { timeout: 5000 }, () => {
|
||||
expect(gidPrefixFor("host-01")).toBe("2pc-host-01-");
|
||||
});
|
||||
});
|
||||
64
packages/api-stack/api/api-2pc/src/phases/gid.ts
Normal file
64
packages/api-stack/api/api-2pc/src/phases/gid.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Global transaction id helpers. Postgres' `PREPARE TRANSACTION '<gid>'`
|
||||
* accepts a string up to ~200 bytes. The coordinator's gid format is:
|
||||
*
|
||||
* `2pc-<hostInstanceId>-<txId>-<participantId>`
|
||||
*
|
||||
* — the host instance id is the recovery anchor (every prepared row
|
||||
* tagged with the host that wrote it; recovery filters on this prefix),
|
||||
* the txId is the coordinator-owned transaction id, the participantId
|
||||
* disambiguates per-participant rows for hosts that participate as more
|
||||
* than one logical actor.
|
||||
*
|
||||
* Embedded single quotes in any field are escaped by SQL-escaping the
|
||||
* gid at the prepared-statement layer — the coordinator never builds
|
||||
* the SQL with string concatenation against this value.
|
||||
*/
|
||||
|
||||
const PREFIX = "2pc";
|
||||
|
||||
/** Build the gid string for a given transaction + participant. */
|
||||
export function buildGid(
|
||||
hostInstanceId: string,
|
||||
txId: string,
|
||||
participantId: string
|
||||
): string {
|
||||
return `${PREFIX}-${hostInstanceId}-${txId}-${participantId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a gid back into its `txId` and `participantId` parts, given
|
||||
* the host instance id of the current coordinator. Returns `null`
|
||||
* when the gid does not match the expected prefix (foreign prepared
|
||||
* transactions sharing the host) — recovery skips those silently.
|
||||
*
|
||||
* `participantId` is constrained by the registry to never contain
|
||||
* hyphens, so the last hyphen unambiguously delimits it from
|
||||
* `txId`.
|
||||
*/
|
||||
export function parseGid(
|
||||
gid: string,
|
||||
hostInstanceId: string
|
||||
): {
|
||||
readonly txId: string;
|
||||
readonly participantId: string;
|
||||
} | null {
|
||||
const prefix = gidPrefixFor(hostInstanceId);
|
||||
if (!gid.startsWith(prefix)) return null;
|
||||
const remainder = gid.slice(prefix.length);
|
||||
const lastHyphen = remainder.lastIndexOf("-");
|
||||
if (lastHyphen === -1) return null;
|
||||
const participantId = remainder.slice(lastHyphen + 1);
|
||||
const txId = remainder.slice(0, lastHyphen);
|
||||
if (txId.length === 0 || participantId.length === 0) return null;
|
||||
return { txId, participantId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Predicate matching the prefix the recovery scan uses against
|
||||
* `pg_prepared_xacts`. Centralised so the SQL filter and parsing
|
||||
* logic never drift.
|
||||
*/
|
||||
export function gidPrefixFor(hostInstanceId: string): string {
|
||||
return `${PREFIX}-${hostInstanceId}-`;
|
||||
}
|
||||
15
packages/api-stack/api/api-2pc/src/phases/index.ts
Normal file
15
packages/api-stack/api/api-2pc/src/phases/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
export {
|
||||
type CommitPhaseInput,
|
||||
commitPhaseTwo,
|
||||
} from "./commit.js";
|
||||
export { buildGid, gidPrefixFor, parseGid } from "./gid.js";
|
||||
export {
|
||||
type ParticipantPrepareOutcome,
|
||||
type PreparePhaseInput,
|
||||
preparePhaseOne,
|
||||
} from "./prepare.js";
|
||||
export {
|
||||
isUnknownPreparedTransaction,
|
||||
type RollbackPhaseInput,
|
||||
rollbackPhaseTwo,
|
||||
} from "./rollback.js";
|
||||
105
packages/api-stack/api/api-2pc/src/phases/prepare.md
Normal file
105
packages/api-stack/api/api-2pc/src/phases/prepare.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# `phases/prepare`
|
||||
|
||||
## Purpose
|
||||
|
||||
Implements phase 1 of the 2PC protocol for a single participant. The
|
||||
coordinator iterates participants and calls this module for each;
|
||||
unanimous commit votes trigger phase 2, any abort or prepare failure
|
||||
short-circuits to rollback. This module is the only place the
|
||||
`PREPARE TRANSACTION '<gid>'` SQL is emitted, so the gid format
|
||||
(`phases/gid`) and the SQL emission point stay co-located.
|
||||
|
||||
Phase 1 is the latency-sensitive half of 2PC: every participant holds
|
||||
row-level locks for the duration. The module records
|
||||
`2pc.prepare_duration_ms` so operators can spot creeping lock holds
|
||||
before they cause throughput regressions.
|
||||
|
||||
## Features
|
||||
|
||||
- Runs the participant's `prepare` callback, then the
|
||||
`PREPARE TRANSACTION` SQL when the vote is `commit` AND the
|
||||
participant is a writer.
|
||||
- Skips the SQL for read-only voters — they participate in the vote
|
||||
only.
|
||||
- Refuses a read-only voter that returned `writeIntent: true` with
|
||||
`TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`.
|
||||
- Honours an `AbortSignal` for the phase-1 deadline; aborted SQL is
|
||||
remapped to `TWOPC_TIMEOUT`.
|
||||
- Emits a `2pc.prepare` span around the whole call and records a
|
||||
`2pc.prepare_duration_ms` observation tagged with the outcome.
|
||||
- Failure cases preserve the underlying cause on the thrown
|
||||
`TwoPcError`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { preparePhaseOne } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "host-01",
|
||||
txId,
|
||||
participant,
|
||||
payload: { invoiceId: "inv-1" },
|
||||
instruments,
|
||||
signal: ac.signal,
|
||||
});
|
||||
if (outcome.vote === "abort") {
|
||||
// run rollback for every participant that voted commit
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The coordinator runs this in a loop. Read-only voters interleave
|
||||
freely with writers — the outcome's `prepared` flag tells phase 2
|
||||
whether to send commit/rollback SQL:
|
||||
|
||||
```ts
|
||||
import { preparePhaseOne } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const outcomes: ParticipantPrepareOutcome[] = [];
|
||||
for (const p of registry.list()) {
|
||||
const o = await preparePhaseOne({
|
||||
hostInstanceId,
|
||||
txId,
|
||||
participant: p,
|
||||
payload: payloadFor(p),
|
||||
instruments,
|
||||
signal,
|
||||
});
|
||||
outcomes.push(o);
|
||||
if (o.vote === "abort") break; // short-circuit
|
||||
}
|
||||
|
||||
// phase 2: commit only those with prepared === true
|
||||
const writers = outcomes.filter((o) => o.prepared);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface ParticipantPrepareOutcome` — `{ participantId, vote,
|
||||
prepared, gid }`.
|
||||
- `interface PreparePhaseInput` — `{ hostInstanceId, txId,
|
||||
participant, payload, instruments, signal }`.
|
||||
- `function preparePhaseOne(input): Promise<ParticipantPrepareOutcome>`
|
||||
- throws `TWOPC_PREPARE_FAILED` when the participant callback or
|
||||
SQL raises (signal not aborted),
|
||||
- throws `TWOPC_TIMEOUT` when the signal aborted around the SQL,
|
||||
- throws `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE` when a read-only
|
||||
voter returned `writeIntent: true`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The SQL escape strategy is conservative: gids are built from the host
|
||||
instance id, transaction id, and participant id — all three are
|
||||
already validated at host config / id-generator / registry time — but
|
||||
the function still calls `.replace(/'/g, "''")` before interpolating
|
||||
into the `PREPARE TRANSACTION` statement. The runtime cost is zero on
|
||||
the happy path and the defence-in-depth is cheap. The coordinator
|
||||
never accepts user-supplied SQL fragments.
|
||||
|
||||
The span is started before the prepare callback so the participant's
|
||||
own work shows up in the trace as a child of the `2pc.prepare` span if
|
||||
the participant opens its own spans against the same tracer. The
|
||||
`finally`-block `span.end()` guarantees the span is closed regardless
|
||||
of which branch fires.
|
||||
252
packages/api-stack/api/api-2pc/src/phases/prepare.test.ts
Normal file
252
packages/api-stack/api/api-2pc/src/phases/prepare.test.ts
Normal file
@@ -0,0 +1,252 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { createTwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { preparePhaseOne } from "./prepare.js";
|
||||
|
||||
const INSTRUMENTS = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
|
||||
function makeConn(opts: { fail?: Error } = {}) {
|
||||
const calls: string[] = [];
|
||||
return {
|
||||
calls,
|
||||
connection: {
|
||||
engine: "postgres" as const,
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.fail) throw opts.fail;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeWriter(
|
||||
id: string,
|
||||
prepareImpl: RegisteredTwoPcParticipant["prepare"],
|
||||
connOpts: { fail?: Error } = {}
|
||||
): RegisteredTwoPcParticipant & { calls: string[] } {
|
||||
const c = makeConn(connOpts);
|
||||
return {
|
||||
id,
|
||||
voterMode: "writer",
|
||||
connection: c.connection,
|
||||
prepare: prepareImpl,
|
||||
calls: c.calls,
|
||||
};
|
||||
}
|
||||
|
||||
describe("preparePhaseOne", () => {
|
||||
it("emits PREPARE TRANSACTION on commit-voted writer and returns prepared=true", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p = makeWriter("billing", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}));
|
||||
const ac = new AbortController();
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: ac.signal,
|
||||
});
|
||||
expect(outcome.vote).toBe("commit");
|
||||
expect(outcome.prepared).toBe(true);
|
||||
expect(outcome.gid).toBe("2pc-h-t-billing");
|
||||
expect(p.calls[0]).toBe("PREPARE TRANSACTION '2pc-h-t-billing'");
|
||||
});
|
||||
|
||||
it("returns vote=abort without emitting SQL when participant votes abort", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p = makeWriter("billing", async () => ({
|
||||
vote: "abort",
|
||||
writeIntent: false,
|
||||
}));
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
expect(outcome.vote).toBe("abort");
|
||||
expect(outcome.prepared).toBe(false);
|
||||
expect(p.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns prepared=false for a read-only voter that voted commit", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p: RegisteredTwoPcParticipant & { calls: string[] } = {
|
||||
...makeWriter("audit", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: false,
|
||||
})),
|
||||
voterMode: "read-only",
|
||||
};
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
expect(outcome.vote).toBe("commit");
|
||||
expect(outcome.prepared).toBe(false);
|
||||
expect(p.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses a read-only voter that returns writeIntent=true", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p: RegisteredTwoPcParticipant & { calls: string[] } = {
|
||||
...makeWriter("rogue", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
})),
|
||||
voterMode: "read-only",
|
||||
};
|
||||
await expect(
|
||||
preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
twoPcCode: TWOPC_CODES.READ_ONLY_PARTICIPANT_CANNOT_WRITE,
|
||||
});
|
||||
});
|
||||
|
||||
it("wraps a prepare-callback throw into TWOPC.PREPARE_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error("boom");
|
||||
const p = makeWriter("billing", async () => {
|
||||
throw cause;
|
||||
});
|
||||
try {
|
||||
await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
expect((err as TwoPcError).cause).toBe(cause);
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps a SQL throw into TWOPC.PREPARE_FAILED when not aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const sqlCause = new Error("sql failed");
|
||||
const p = makeWriter(
|
||||
"billing",
|
||||
async () => ({ vote: "commit", writeIntent: true }),
|
||||
{ fail: sqlCause }
|
||||
);
|
||||
try {
|
||||
await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("maps a SQL throw to TWOPC.TIMEOUT when the signal is aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
const p = makeWriter(
|
||||
"billing",
|
||||
async () => ({ vote: "commit", writeIntent: true }),
|
||||
{ fail: new Error("connection terminated") }
|
||||
);
|
||||
// Abort BEFORE the PREPARE TRANSACTION runs.
|
||||
const original = p.connection.run;
|
||||
p.connection.run = async (sql: string) => {
|
||||
ac.abort();
|
||||
return original(sql);
|
||||
};
|
||||
try {
|
||||
await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: ac.signal,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.TIMEOUT);
|
||||
}
|
||||
});
|
||||
|
||||
it("raises TWOPC.TIMEOUT immediately when the signal was already aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const p = makeWriter("billing", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}));
|
||||
await expect(
|
||||
preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: ac.signal,
|
||||
})
|
||||
).rejects.toMatchObject({ twoPcCode: TWOPC_CODES.TIMEOUT });
|
||||
});
|
||||
|
||||
it("escapes single quotes in the gid before interpolation", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p = makeWriter("bill'ing", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}));
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
expect(outcome.gid).toBe("2pc-h-t-bill'ing");
|
||||
expect(p.calls[0]).toBe("PREPARE TRANSACTION '2pc-h-t-bill''ing'");
|
||||
});
|
||||
});
|
||||
160
packages/api-stack/api/api-2pc/src/phases/prepare.ts
Normal file
160
packages/api-stack/api/api-2pc/src/phases/prepare.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { TwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type {
|
||||
RegisteredTwoPcParticipant,
|
||||
TwoPcPrepareResult,
|
||||
} from "../types/types.types.js";
|
||||
import { buildGid } from "./gid.js";
|
||||
|
||||
/**
|
||||
* Outcome of phase-1 prepare for a single participant. `prepared`
|
||||
* tracks whether `PREPARE TRANSACTION '<gid>'` actually ran — the
|
||||
* rollback phase consults this to decide which participants need
|
||||
* `ROLLBACK PREPARED`.
|
||||
*/
|
||||
export interface ParticipantPrepareOutcome {
|
||||
readonly participantId: string;
|
||||
readonly vote: TwoPcVote;
|
||||
readonly prepared: boolean;
|
||||
/** SQL gid passed to PREPARE TRANSACTION — empty when no SQL ran. */
|
||||
readonly gid: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Input passed by the coordinator to a phase-1 run.
|
||||
*/
|
||||
export interface PreparePhaseInput {
|
||||
readonly hostInstanceId: string;
|
||||
readonly txId: string;
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
readonly payload: Readonly<Record<string, unknown>>;
|
||||
readonly instruments: TwoPcInstruments;
|
||||
/** Caller-driven AbortSignal — wired to the phase-1 deadline. */
|
||||
readonly signal: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run phase-1 for one participant. Three things happen, in order:
|
||||
*
|
||||
* 1. The participant's `prepare` callback runs its business work.
|
||||
* 2. If the participant voted `commit` AND is a writer, the coordinator
|
||||
* issues `PREPARE TRANSACTION '<gid>'` against the participant's
|
||||
* connection. Read-only voters skip this step.
|
||||
* 3. The vote and prepared-flag are returned to the coordinator.
|
||||
*
|
||||
* Failures map to {@link TWOPC_CODES} as follows:
|
||||
*
|
||||
* - read-only voter returned `writeIntent: true` →
|
||||
* `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`.
|
||||
* - prepare callback threw OR `PREPARE TRANSACTION` SQL threw →
|
||||
* `TWOPC_PREPARE_FAILED` (the cause is preserved).
|
||||
* - signal aborted during the SQL call → `TWOPC_TIMEOUT`.
|
||||
*/
|
||||
export async function preparePhaseOne(
|
||||
input: PreparePhaseInput
|
||||
): Promise<ParticipantPrepareOutcome> {
|
||||
const { hostInstanceId, txId, participant, payload, instruments, signal } =
|
||||
input;
|
||||
const span = instruments.tracer.startSpan("2pc.prepare", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.id": participant.id,
|
||||
"voter.mode": participant.voterMode,
|
||||
},
|
||||
});
|
||||
const t0 = Date.now();
|
||||
let outcome: TwoPcPrepareResult;
|
||||
try {
|
||||
try {
|
||||
outcome = await participant.prepare(txId, payload);
|
||||
} catch (cause) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PREPARE_FAILED,
|
||||
`Participant ${participant.id} prepare callback raised`,
|
||||
{
|
||||
cause,
|
||||
context: { participantId: participant.id, txId },
|
||||
}
|
||||
);
|
||||
}
|
||||
if (outcome.vote === "abort") {
|
||||
instruments.prepareDurationMs.record(Date.now() - t0, {
|
||||
outcome: "aborted",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
return {
|
||||
participantId: participant.id,
|
||||
vote: "abort",
|
||||
prepared: false,
|
||||
gid: "",
|
||||
};
|
||||
}
|
||||
if (
|
||||
participant.voterMode === "read-only" &&
|
||||
outcome.writeIntent === true
|
||||
) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.READ_ONLY_PARTICIPANT_CANNOT_WRITE,
|
||||
`Read-only voter ${participant.id} returned writeIntent=true`,
|
||||
{ context: { participantId: participant.id, txId } }
|
||||
);
|
||||
}
|
||||
if (participant.voterMode === "read-only") {
|
||||
instruments.prepareDurationMs.record(Date.now() - t0, {
|
||||
outcome: "committed",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
return {
|
||||
participantId: participant.id,
|
||||
vote: "commit",
|
||||
prepared: false,
|
||||
gid: "",
|
||||
};
|
||||
}
|
||||
if (signal.aborted) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT,
|
||||
`Phase-1 aborted before PREPARE TRANSACTION for ${participant.id}`,
|
||||
{ context: { participantId: participant.id, txId } }
|
||||
);
|
||||
}
|
||||
const gid = buildGid(hostInstanceId, txId, participant.id);
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (cause) {
|
||||
if (signal.aborted) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT,
|
||||
`Phase-1 deadline expired while preparing ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: { participantId: participant.id, txId },
|
||||
}
|
||||
);
|
||||
}
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PREPARE_FAILED,
|
||||
`PREPARE TRANSACTION failed for ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: { participantId: participant.id, txId, gid },
|
||||
}
|
||||
);
|
||||
}
|
||||
instruments.prepareDurationMs.record(Date.now() - t0, {
|
||||
outcome: "committed",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
return {
|
||||
participantId: participant.id,
|
||||
vote: "commit",
|
||||
prepared: true,
|
||||
gid,
|
||||
};
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
92
packages/api-stack/api/api-2pc/src/phases/rollback.md
Normal file
92
packages/api-stack/api/api-2pc/src/phases/rollback.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# `phases/rollback`
|
||||
|
||||
## Purpose
|
||||
|
||||
Phase-2 rollback for a single participant. Used both by the
|
||||
coordinator's abort path (vote_abort or prepare failure) and by the
|
||||
recovery runner (orphan with no `all-voted-commit` ledger row).
|
||||
|
||||
The function is idempotent at the protocol level: Postgres returns
|
||||
`42704` (`undefined_object`, "prepared transaction does not exist")
|
||||
for a gid that was already rolled back, and this module catches that
|
||||
specific error and treats it as a no-op. That property is what makes
|
||||
recovery safe to re-run on every boot.
|
||||
|
||||
## Features
|
||||
|
||||
- Emits `ROLLBACK PREPARED '<gid>'` against the participant's
|
||||
connection.
|
||||
- Idempotent against Postgres `42704` — a second rollback against the
|
||||
same gid is a no-op.
|
||||
- Skips participants that did not actually prepare (`outcome.prepared
|
||||
=== false`).
|
||||
- Emits a `2pc.rollback` span with `tx.id`, `participant.id`, and
|
||||
`gid` attributes.
|
||||
- Failure preserves the underlying cause on a thrown
|
||||
`TwoPcError(TWOPC_ROLLBACK_FAILED)`.
|
||||
- Exports `isUnknownPreparedTransaction(cause)` so the recovery
|
||||
runner can apply the same idempotency check without duplicating the
|
||||
SQLSTATE sniff.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { rollbackPhaseTwo } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
await rollbackPhaseTwo({
|
||||
txId,
|
||||
participant,
|
||||
outcome: preparedOutcome,
|
||||
instruments,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Recovery uses the same idempotency check directly to short-circuit
|
||||
when the row was concurrently resolved:
|
||||
|
||||
```ts
|
||||
import {
|
||||
isUnknownPreparedTransaction,
|
||||
rollbackPhaseTwo,
|
||||
} from "@suaveplan/api-2pc/phases";
|
||||
|
||||
try {
|
||||
await rollbackPhaseTwo({ txId, participant, outcome, instruments });
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof TwoPcError &&
|
||||
err.twoPcCode === TWOPC_CODES.ROLLBACK_FAILED &&
|
||||
isUnknownPreparedTransaction(err.cause)
|
||||
) {
|
||||
// race: another process already rolled this back — treat as success
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface RollbackPhaseInput` — same fields as `CommitPhaseInput`.
|
||||
- `function rollbackPhaseTwo(input): Promise<void>`
|
||||
- no-op for `outcome.prepared === false`
|
||||
- throws `TwoPcError(TWOPC_ROLLBACK_FAILED)` on non-idempotency SQL
|
||||
failure.
|
||||
- `function isUnknownPreparedTransaction(cause: unknown): boolean`
|
||||
- matches SQLSTATE `42704` AND the substring `"does not exist"` as
|
||||
a fallback for drivers that omit the SQLSTATE.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The idempotency check sniffs both `cause.code` and `cause.message`
|
||||
because the two postgres clients in use (`postgres`/porsager and
|
||||
`pg.Client`) surface the SQLSTATE differently. Using both lets the
|
||||
module work against either driver without a wrapper layer.
|
||||
|
||||
The function is split into a small public `rollbackPhaseTwo` and an
|
||||
exported `isUnknownPreparedTransaction` helper so recovery can re-use
|
||||
the idempotency predicate without re-implementing it. Recovery sees
|
||||
this exact race regularly: the coordinator and the recovery runner
|
||||
both target the same prepared gid when a host restarts mid-rollback.
|
||||
145
packages/api-stack/api/api-2pc/src/phases/rollback.test.ts
Normal file
145
packages/api-stack/api/api-2pc/src/phases/rollback.test.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { createTwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
import { isUnknownPreparedTransaction, rollbackPhaseTwo } from "./rollback.js";
|
||||
|
||||
const INSTRUMENTS = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
|
||||
const OUTCOME: ParticipantPrepareOutcome = {
|
||||
participantId: "billing",
|
||||
vote: "commit",
|
||||
prepared: true,
|
||||
gid: "2pc-h-t-billing",
|
||||
};
|
||||
|
||||
function makeParticipant(opts: { fail?: unknown } = {}) {
|
||||
const calls: string[] = [];
|
||||
const p: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.fail !== undefined) throw opts.fail;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
return { participant: p, calls };
|
||||
}
|
||||
|
||||
describe("rollbackPhaseTwo", () => {
|
||||
it("issues ROLLBACK PREPARED against a prepared writer", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls[0]).toBe("ROLLBACK PREPARED '2pc-h-t-billing'");
|
||||
});
|
||||
|
||||
it("is a no-op when outcome.prepared === false", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: { ...OUTCOME, prepared: false, gid: "" },
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats 42704 (undefined_object) as a no-op (idempotency)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = Object.assign(new Error("does not exist"), {
|
||||
code: "42704",
|
||||
});
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
await expect(
|
||||
rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("treats 'does not exist' message-only errors as a no-op", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error(
|
||||
"prepared transaction with identifier 'x' does not exist"
|
||||
);
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
await expect(
|
||||
rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("wraps a non-idempotency SQL throw into TWOPC.ROLLBACK_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error("connection broken");
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
try {
|
||||
await rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.ROLLBACK_FAILED
|
||||
);
|
||||
expect((err as TwoPcError).cause).toBe(cause);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("isUnknownPreparedTransaction", () => {
|
||||
it("returns true for SQLSTATE 42704", { timeout: 5000 }, () => {
|
||||
expect(isUnknownPreparedTransaction({ code: "42704" })).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for messages containing 'does not exist'", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
isUnknownPreparedTransaction({ message: "tx X does not exist" })
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for null / undefined / primitives", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(isUnknownPreparedTransaction(null)).toBe(false);
|
||||
expect(isUnknownPreparedTransaction(undefined)).toBe(false);
|
||||
expect(isUnknownPreparedTransaction(42)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for unrelated errors", { timeout: 5000 }, () => {
|
||||
expect(isUnknownPreparedTransaction(new Error("other"))).toBe(false);
|
||||
});
|
||||
});
|
||||
85
packages/api-stack/api/api-2pc/src/phases/rollback.ts
Normal file
85
packages/api-stack/api/api-2pc/src/phases/rollback.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { TwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
|
||||
/**
|
||||
* Input to {@link rollbackPhaseTwo}. Mirrors the commit-phase input
|
||||
* shape so the coordinator can swap between them without remapping.
|
||||
*/
|
||||
export interface RollbackPhaseInput {
|
||||
readonly txId: string;
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
readonly outcome: ParticipantPrepareOutcome;
|
||||
readonly instruments: TwoPcInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase-2 rollback for a single participant. Issues
|
||||
* `ROLLBACK PREPARED '<gid>'`. Idempotent at the protocol level —
|
||||
* Postgres returns an error for a gid that does not exist in
|
||||
* `pg_prepared_xacts`, which the coordinator catches and treats as a
|
||||
* no-op (the row was already rolled back, e.g. by recovery).
|
||||
*
|
||||
* Failures other than "no such prepared transaction" throw
|
||||
* `TWOPC_ROLLBACK_FAILED` with the cause preserved.
|
||||
*/
|
||||
export async function rollbackPhaseTwo(
|
||||
input: RollbackPhaseInput
|
||||
): Promise<void> {
|
||||
const { txId, participant, outcome, instruments } = input;
|
||||
if (!outcome.prepared) return;
|
||||
const span = instruments.tracer.startSpan("2pc.rollback", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.id": participant.id,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
});
|
||||
try {
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${outcome.gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (cause) {
|
||||
if (isUnknownPreparedTransaction(cause)) {
|
||||
return; // idempotent — already rolled back
|
||||
}
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.ROLLBACK_FAILED,
|
||||
`ROLLBACK PREPARED failed for ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
participantId: participant.id,
|
||||
txId,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect Postgres' "prepared transaction with identifier ... does not
|
||||
* exist" error. The driver-level cause includes the SQLSTATE `42704`
|
||||
* (undefined_object) — we sniff the message and the code defensively
|
||||
* so the check works against both `postgres` (porsager/postgres) and
|
||||
* `pg.Client`. The fallback to message-substring lookup catches the
|
||||
* uncommon case where a driver does not surface the SQLSTATE.
|
||||
*/
|
||||
export function isUnknownPreparedTransaction(cause: unknown): boolean {
|
||||
if (cause === null || typeof cause !== "object") return false;
|
||||
const errObj = cause as { code?: unknown; message?: unknown };
|
||||
if (errObj.code === "42704") return true;
|
||||
if (
|
||||
typeof errObj.message === "string" &&
|
||||
errObj.message.includes("does not exist")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
7
packages/api-stack/api/api-2pc/src/recovery/index.ts
Normal file
7
packages/api-stack/api/api-2pc/src/recovery/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
buildProbeFromRegistry,
|
||||
type RecoveryInput,
|
||||
type RecoveryProbe,
|
||||
type RecoveryReport,
|
||||
resolveRecovery,
|
||||
} from "./recovery.js";
|
||||
133
packages/api-stack/api/api-2pc/src/recovery/recovery.md
Normal file
133
packages/api-stack/api/api-2pc/src/recovery/recovery.md
Normal file
@@ -0,0 +1,133 @@
|
||||
# `recovery/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Boot-time orphan resolution for the 2PC coordinator. When a host
|
||||
crashes between phase 1 and phase 2, prepared transactions remain
|
||||
held in `pg_prepared_xacts` on every writer participant. The recovery
|
||||
runner sweeps those rows on the next boot, consults the host-owned
|
||||
ledger, and either commits or rolls back each one — guaranteeing the
|
||||
"once prepared, eventually resolved" half of the 2PC durability
|
||||
property.
|
||||
|
||||
The runner is the only path that should ever resolve an in-doubt
|
||||
transaction. The coordinator's hot path NEVER auto-recovers; it
|
||||
escalates to `TWOPC_IN_DOUBT` and lets the next-boot recovery sweep
|
||||
handle it. That separation keeps the hot path's failure handling
|
||||
small and the recovery path testable in isolation.
|
||||
|
||||
## Features
|
||||
|
||||
- `resolveRecovery({ hostInstanceId, ledger, probes, onResolved })` —
|
||||
the entry point. Returns a `RecoveryReport` enumerating committed,
|
||||
rolled-back, skipped, and errored gids.
|
||||
- Idempotent — re-running the function on a host with no outstanding
|
||||
prepared transactions is a no-op (the inner `queryOrphans` returns
|
||||
an empty list).
|
||||
- Treats Postgres `42704` ("does not exist") as success during
|
||||
rollback — covers the race where another process already resolved
|
||||
the orphan.
|
||||
- Marks rows as `in_doubt` on unresolvable errors so the next sweep
|
||||
can retry; errored gids are surfaced on the report rather than
|
||||
thrown.
|
||||
- `buildProbeFromRegistry(registry)` — convenience adapter that
|
||||
spins up one probe per writer participant.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
buildProbeFromRegistry,
|
||||
resolveRecovery,
|
||||
} from "@suaveplan/api-2pc/recovery";
|
||||
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "host-01",
|
||||
ledger,
|
||||
probes: buildProbeFromRegistry(registry),
|
||||
});
|
||||
|
||||
console.log(
|
||||
`committed=${report.committed.length} ` +
|
||||
`rolledBack=${report.rolledBack.length} ` +
|
||||
`errors=${report.errors.length}`
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Wire a custom probe to feed an alternate identifier convention or a
|
||||
replicated-read connection — useful when the recovery sweep should
|
||||
hit a read replica to avoid contending with the live writer:
|
||||
|
||||
```ts
|
||||
import { gidPrefixFor } from "@suaveplan/api-2pc/phases";
|
||||
import { resolveRecovery, type RecoveryProbe } from "@suaveplan/api-2pc/recovery";
|
||||
|
||||
const probes: RecoveryProbe[] = [
|
||||
{
|
||||
participantId: "billing",
|
||||
async queryOrphans(prefix) {
|
||||
const rows = await readReplica.unsafe<{ gid: string }[]>(
|
||||
`SELECT gid FROM pg_prepared_xacts WHERE gid LIKE $1`,
|
||||
[`${prefix}%`]
|
||||
);
|
||||
return rows.map((r) => r.gid);
|
||||
},
|
||||
async commit(gid) {
|
||||
await primary.unsafe(`COMMIT PREPARED $1`, [gid]);
|
||||
},
|
||||
async rollback(gid) {
|
||||
await primary.unsafe(`ROLLBACK PREPARED $1`, [gid]);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "host-01",
|
||||
ledger,
|
||||
probes,
|
||||
onResolved: (e) => log.info(e),
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface RecoveryProbe`
|
||||
- `participantId: string`
|
||||
- `queryOrphans(prefix: string): Promise<readonly string[]>`
|
||||
- `commit(gid: string): Promise<void>`
|
||||
- `rollback(gid: string): Promise<void>`
|
||||
- `interface RecoveryReport`
|
||||
- `committed: readonly string[]`
|
||||
- `rolledBack: readonly string[]`
|
||||
- `skipped: readonly string[]`
|
||||
- `errors: readonly { gid; error: TwoPcError }[]`
|
||||
- `interface RecoveryInput`
|
||||
- `hostInstanceId, ledger, probes, onResolved?`
|
||||
- `function resolveRecovery(input): Promise<RecoveryReport>`
|
||||
- `function buildProbeFromRegistry(registry): readonly RecoveryProbe[]`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The ledger lookup uses the gid's parsed `txId` as the primary key —
|
||||
the gid format from `phases/gid.ts` is what makes that mapping
|
||||
unambiguous. Rows whose ledger state is `committing` OR `committed`
|
||||
trigger `COMMIT PREPARED`; everything else (including missing rows)
|
||||
triggers `ROLLBACK PREPARED`. That asymmetric default ("rollback by
|
||||
default") is the safe choice — committing a row whose ledger does
|
||||
not explicitly say "all voted commit" would violate the design
|
||||
guarantee.
|
||||
|
||||
The `42704` swallow during rollback is what makes the runner safe to
|
||||
re-execute: if the previous sweep crashed AFTER rolling back the
|
||||
prepared transaction but BEFORE updating the ledger, the second
|
||||
sweep will see the orphan listed in `pg_prepared_xacts` (or not, if
|
||||
truly gone) and will not throw on the "does not exist" follow-up.
|
||||
Errors that cannot be classified as idempotency are captured per-gid
|
||||
and surfaced on the report — operators page-on `errors.length > 0`,
|
||||
not on a generic throw.
|
||||
308
packages/api-stack/api/api-2pc/src/recovery/recovery.test.ts
Normal file
308
packages/api-stack/api/api-2pc/src/recovery/recovery.test.ts
Normal file
@@ -0,0 +1,308 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES } from "../error-codes/error-codes.js";
|
||||
import {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
TwoPcLedgerStore,
|
||||
} from "../persistence/persistence.js";
|
||||
import { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import {
|
||||
buildProbeFromRegistry,
|
||||
type RecoveryProbe,
|
||||
resolveRecovery,
|
||||
} from "./recovery.js";
|
||||
|
||||
function makeLedger(): TwoPcLedgerStore {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
return new TwoPcLedgerStore(db);
|
||||
}
|
||||
|
||||
const SAMPLE: ReadonlyArray<LedgerParticipantEntry> = [
|
||||
{ participantId: "billing", voterMode: "writer", vote: "commit" },
|
||||
];
|
||||
|
||||
function makeProbe(opts: {
|
||||
orphans?: readonly string[];
|
||||
failCommit?: Error;
|
||||
failRollback?: Error;
|
||||
}): RecoveryProbe & { committed: string[]; rolledBack: string[] } {
|
||||
const committed: string[] = [];
|
||||
const rolledBack: string[] = [];
|
||||
return {
|
||||
participantId: "billing",
|
||||
committed,
|
||||
rolledBack,
|
||||
async queryOrphans(_prefix) {
|
||||
return opts.orphans ?? [];
|
||||
},
|
||||
async commit(gid) {
|
||||
if (opts.failCommit) throw opts.failCommit;
|
||||
committed.push(gid);
|
||||
},
|
||||
async rollback(gid) {
|
||||
if (opts.failRollback) throw opts.failRollback;
|
||||
rolledBack.push(gid);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("resolveRecovery", () => {
|
||||
it("commits orphans whose ledger state is committing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-1",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
ledger.markAllVotedCommit({
|
||||
id: "tx-1",
|
||||
participants: SAMPLE,
|
||||
preparedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
const probe = makeProbe({ orphans: ["2pc-h-tx-1-billing"] });
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.committed).toEqual(["2pc-h-tx-1-billing"]);
|
||||
expect(report.rolledBack).toEqual([]);
|
||||
expect(probe.committed).toEqual(["2pc-h-tx-1-billing"]);
|
||||
expect(ledger.findById("tx-1")?.state).toBe("committed");
|
||||
});
|
||||
|
||||
it("rolls back orphans whose ledger row is missing (safe default)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const probe = makeProbe({ orphans: ["2pc-h-tx-2-billing"] });
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.rolledBack).toEqual(["2pc-h-tx-2-billing"]);
|
||||
expect(report.committed).toEqual([]);
|
||||
expect(probe.rolledBack).toEqual(["2pc-h-tx-2-billing"]);
|
||||
});
|
||||
|
||||
it("rolls back orphans whose ledger state is voting (not yet all-voted-commit)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-3",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
const probe = makeProbe({ orphans: ["2pc-h-tx-3-billing"] });
|
||||
await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(ledger.findById("tx-3")?.state).toBe("aborted");
|
||||
});
|
||||
|
||||
it("treats Postgres 42704 during rollback as success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const cause = Object.assign(new Error("does not exist"), {
|
||||
code: "42704",
|
||||
});
|
||||
const probe = makeProbe({
|
||||
orphans: ["2pc-h-tx-4-billing"],
|
||||
failRollback: cause,
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.errors).toEqual([]);
|
||||
expect(report.rolledBack).toEqual(["2pc-h-tx-4-billing"]);
|
||||
});
|
||||
|
||||
it("skips foreign gids that don't match the host prefix", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const probe = makeProbe({
|
||||
orphans: ["2pc-other-host-tx-5-billing", "foreign-gid"],
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.skipped).toContain("2pc-other-host-tx-5-billing");
|
||||
expect(report.skipped).toContain("foreign-gid");
|
||||
expect(report.committed).toEqual([]);
|
||||
expect(report.rolledBack).toEqual([]);
|
||||
});
|
||||
|
||||
it("marks irrecoverable orphans in_doubt and surfaces them on the report", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-6",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
ledger.markAllVotedCommit({
|
||||
id: "tx-6",
|
||||
participants: SAMPLE,
|
||||
preparedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
const cause = new Error("connection broken");
|
||||
const probe = makeProbe({
|
||||
orphans: ["2pc-h-tx-6-billing"],
|
||||
failCommit: cause,
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.errors).toHaveLength(1);
|
||||
expect(report.errors[0]?.error.twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
expect(ledger.findById("tx-6")?.state).toBe("in_doubt");
|
||||
});
|
||||
|
||||
it("invokes onResolved for every outcome", { timeout: 5000 }, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-7",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
const events: Array<{ gid: string; outcome: string }> = [];
|
||||
await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [makeProbe({ orphans: ["2pc-h-tx-7-billing", "foreign"] })],
|
||||
onResolved: (e) => events.push({ ...e }),
|
||||
});
|
||||
expect(events.length).toBe(2);
|
||||
expect(events.some((e) => e.outcome === "rolledBack")).toBe(true);
|
||||
expect(events.some((e) => e.outcome === "skipped")).toBe(true);
|
||||
});
|
||||
|
||||
it("re-running recovery on a clean state is a no-op", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const probe = makeProbe({ orphans: [] });
|
||||
const r1 = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
const r2 = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(r1.committed.length).toBe(0);
|
||||
expect(r1.rolledBack.length).toBe(0);
|
||||
expect(r2.committed.length).toBe(0);
|
||||
expect(r2.rolledBack.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildProbeFromRegistry", () => {
|
||||
it("emits one probe per writer participant; skips read-only voters", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
const writer: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
const reader: RegisteredTwoPcParticipant = {
|
||||
id: "audit",
|
||||
voterMode: "read-only",
|
||||
connection: {
|
||||
engine: "surrealdb",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: false }),
|
||||
};
|
||||
registry.register(writer);
|
||||
registry.register(reader);
|
||||
const probes = buildProbeFromRegistry(registry);
|
||||
expect(probes).toHaveLength(1);
|
||||
expect(probes[0]?.participantId).toBe("billing");
|
||||
});
|
||||
|
||||
it("emitted probe runs the documented pg_prepared_xacts query", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const calls: string[] = [];
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (sql.includes("SELECT gid"))
|
||||
return [{ gid: "2pc-h-tx-1-billing" }];
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
});
|
||||
const probes = buildProbeFromRegistry(registry);
|
||||
const orphans = await probes[0]?.queryOrphans("2pc-h-");
|
||||
expect(orphans).toEqual(["2pc-h-tx-1-billing"]);
|
||||
expect(calls[0]).toContain("SELECT gid FROM pg_prepared_xacts");
|
||||
expect(calls[0]).toContain("LIKE '2pc-h-%'");
|
||||
await probes[0]?.commit("2pc-h-tx-1-billing");
|
||||
await probes[0]?.rollback("2pc-h-tx-1-billing");
|
||||
expect(calls).toEqual([
|
||||
"SELECT gid FROM pg_prepared_xacts WHERE gid LIKE '2pc-h-%'",
|
||||
"COMMIT PREPARED '2pc-h-tx-1-billing'",
|
||||
"ROLLBACK PREPARED '2pc-h-tx-1-billing'",
|
||||
]);
|
||||
});
|
||||
|
||||
it("queryOrphans handles a probe that returns undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
});
|
||||
const probes = buildProbeFromRegistry(registry);
|
||||
const orphans = await probes[0]?.queryOrphans("2pc-h-");
|
||||
expect(orphans).toEqual([]);
|
||||
});
|
||||
});
|
||||
196
packages/api-stack/api/api-2pc/src/recovery/recovery.ts
Normal file
196
packages/api-stack/api/api-2pc/src/recovery/recovery.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type {
|
||||
TwoPcLedgerRow,
|
||||
TwoPcLedgerStore,
|
||||
} from "../persistence/persistence.js";
|
||||
import { gidPrefixFor, parseGid } from "../phases/gid.js";
|
||||
import { isUnknownPreparedTransaction } from "../phases/rollback.js";
|
||||
import type { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
|
||||
/**
|
||||
* Per-participant probe surface — pings a single Postgres connection
|
||||
* for prepared transactions matching the host's gid prefix. The
|
||||
* coordinator iterates registered writers and asks each one for its
|
||||
* orphans on boot.
|
||||
*/
|
||||
export interface RecoveryProbe {
|
||||
readonly participantId: string;
|
||||
queryOrphans(prefix: string): Promise<readonly string[]>;
|
||||
commit(gid: string): Promise<void>;
|
||||
rollback(gid: string): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of running recovery. `committed` and `rolledBack` carry the
|
||||
* exact gids resolved on this boot; `skipped` carries gids the
|
||||
* coordinator could not parse (foreign prepared transactions sharing
|
||||
* the host).
|
||||
*/
|
||||
export interface RecoveryReport {
|
||||
readonly committed: readonly string[];
|
||||
readonly rolledBack: readonly string[];
|
||||
readonly skipped: readonly string[];
|
||||
readonly errors: ReadonlyArray<{
|
||||
readonly gid: string;
|
||||
readonly error: TwoPcError;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Input to {@link resolveRecovery}.
|
||||
*/
|
||||
export interface RecoveryInput {
|
||||
readonly hostInstanceId: string;
|
||||
readonly ledger: TwoPcLedgerStore;
|
||||
/**
|
||||
* Per-participant Postgres probes. The recovery runner does not
|
||||
* care which participants are registered — it sweeps every probe.
|
||||
*/
|
||||
readonly probes: ReadonlyArray<RecoveryProbe>;
|
||||
/**
|
||||
* Optional callback fired for every resolved gid; useful for
|
||||
* progress logging in tests and ops dashboards.
|
||||
*/
|
||||
readonly onResolved?: (event: {
|
||||
readonly gid: string;
|
||||
readonly outcome: "committed" | "rolledBack" | "skipped";
|
||||
}) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Boot-time orphan resolution. Per design doc §9.2 paragraph 3:
|
||||
*
|
||||
* 1. Query every participant's `pg_prepared_xacts` for gids matching
|
||||
* this host's prefix.
|
||||
* 2. For each orphan: if the ledger row records `all-voted-commit`
|
||||
* (state `committing` or `committed`) issue `COMMIT PREPARED`;
|
||||
* otherwise issue `ROLLBACK PREPARED`. Missing ledger row → safe
|
||||
* default `ROLLBACK PREPARED` (no commit without explicit log).
|
||||
* 3. Update the ledger row to a terminal state. Idempotent — running
|
||||
* recovery twice in a row is a no-op once orphans are resolved.
|
||||
*
|
||||
* Errors are collected per-gid rather than thrown — the runner makes
|
||||
* progress on every recoverable orphan and surfaces unresolved ones
|
||||
* to the caller for follow-up.
|
||||
*/
|
||||
export async function resolveRecovery(
|
||||
input: RecoveryInput
|
||||
): Promise<RecoveryReport> {
|
||||
const { hostInstanceId, ledger, probes, onResolved } = input;
|
||||
const prefix = gidPrefixFor(hostInstanceId);
|
||||
const committed: string[] = [];
|
||||
const rolledBack: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
const errors: Array<{ gid: string; error: TwoPcError }> = [];
|
||||
const nowIso = () => new Date().toISOString();
|
||||
|
||||
for (const probe of probes) {
|
||||
const orphans = await probe.queryOrphans(prefix);
|
||||
for (const gid of orphans) {
|
||||
const parts = parseGid(gid, hostInstanceId);
|
||||
if (parts === null) {
|
||||
skipped.push(gid);
|
||||
onResolved?.({ gid, outcome: "skipped" });
|
||||
continue;
|
||||
}
|
||||
const row = ledger.findById(parts.txId);
|
||||
const shouldCommit =
|
||||
row !== null &&
|
||||
(row.state === "committing" || row.state === "committed");
|
||||
try {
|
||||
if (shouldCommit) {
|
||||
await probe.commit(gid);
|
||||
finaliseLedgerForCommit(ledger, row, nowIso);
|
||||
committed.push(gid);
|
||||
onResolved?.({ gid, outcome: "committed" });
|
||||
} else {
|
||||
try {
|
||||
await probe.rollback(gid);
|
||||
} catch (cause) {
|
||||
if (!isUnknownPreparedTransaction(cause)) throw cause;
|
||||
}
|
||||
finaliseLedgerForAbort(ledger, row, nowIso);
|
||||
rolledBack.push(gid);
|
||||
onResolved?.({ gid, outcome: "rolledBack" });
|
||||
}
|
||||
} catch (cause) {
|
||||
if (row !== null) ledger.markInDoubt(row.id);
|
||||
errors.push({
|
||||
gid,
|
||||
error: new TwoPcError(
|
||||
TWOPC_CODES.IN_DOUBT,
|
||||
`Recovery could not resolve orphan ${gid}`,
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
gid,
|
||||
participantId: probe.participantId,
|
||||
},
|
||||
}
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { committed, rolledBack, skipped, errors };
|
||||
}
|
||||
|
||||
function finaliseLedgerForCommit(
|
||||
ledger: TwoPcLedgerStore,
|
||||
row: TwoPcLedgerRow | null,
|
||||
nowIso: () => string
|
||||
): void {
|
||||
if (row === null) return;
|
||||
if (row.state === "committed") return;
|
||||
ledger.markCommitted(row.id, nowIso());
|
||||
}
|
||||
|
||||
function finaliseLedgerForAbort(
|
||||
ledger: TwoPcLedgerStore,
|
||||
row: TwoPcLedgerRow | null,
|
||||
nowIso: () => string
|
||||
): void {
|
||||
if (row === null) return;
|
||||
if (row.state === "aborted") return;
|
||||
ledger.markAborted(row.id, nowIso());
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience adapter: builds a {@link RecoveryProbe} backed by a
|
||||
* registered participant. The participant's connection is used for
|
||||
* the `pg_prepared_xacts` scan AND the COMMIT/ROLLBACK PREPARED SQL;
|
||||
* the coordinator never asks a participant to manipulate another
|
||||
* participant's prepared transactions.
|
||||
*/
|
||||
export function buildProbeFromRegistry(
|
||||
registry: TwoPcParticipantRegistry
|
||||
): readonly RecoveryProbe[] {
|
||||
const probes: RecoveryProbe[] = [];
|
||||
for (const p of registry.list()) {
|
||||
if (p.voterMode === "read-only") continue;
|
||||
const conn = p.connection;
|
||||
probes.push({
|
||||
participantId: p.id,
|
||||
async queryOrphans(prefix) {
|
||||
const rows = (await conn.run<ReadonlyArray<{ gid: string }>>(
|
||||
`SELECT gid FROM pg_prepared_xacts WHERE gid LIKE ${escapeLiteral(`${prefix}%`)}`
|
||||
)) as ReadonlyArray<{ gid: string }> | undefined;
|
||||
return (rows ?? []).map((r) => r.gid);
|
||||
},
|
||||
async commit(gid) {
|
||||
await conn.run(`COMMIT PREPARED '${gid.replace(/'/g, "''")}'`);
|
||||
},
|
||||
async rollback(gid) {
|
||||
await conn.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
return probes;
|
||||
}
|
||||
|
||||
function escapeLiteral(s: string): string {
|
||||
return `'${s.replace(/'/g, "''")}'`;
|
||||
}
|
||||
4
packages/api-stack/api/api-2pc/src/registry/index.ts
Normal file
4
packages/api-stack/api/api-2pc/src/registry/index.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
type PreparableCapabilityCheck,
|
||||
TwoPcParticipantRegistry,
|
||||
} from "./participant-registry.js";
|
||||
@@ -0,0 +1,111 @@
|
||||
# `registry/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
The participant registry is the install-time admission control between
|
||||
plugins and the 2PC coordinator. Plugins declare
|
||||
`twoPhaseCommit.participates: true` in their manifest; the host
|
||||
registers them with this registry; the registry refuses any writer
|
||||
whose bound database engine cannot honour `PREPARE TRANSACTION`.
|
||||
|
||||
This is where the design doc §9.2 rule "SQLite and SurrealDB MUST NOT
|
||||
participate as writers" is enforced. The capability check is injected
|
||||
as a callback so the registry stays decoupled from
|
||||
`@suaveplan/api-host`'s concrete `CapabilityRegistry`.
|
||||
|
||||
## Features
|
||||
|
||||
- `TwoPcParticipantRegistry.register(participant)` — refuses writers
|
||||
without `db.preparable`; accepts read-only voters regardless of
|
||||
engine.
|
||||
- `get(id)` / `list()` / `size()` / `unregister(id)` — standard
|
||||
registry surface.
|
||||
- Injection seam: `PreparableCapabilityCheck` is a plain
|
||||
`(id: string) => boolean` callback — zero coupling to api-host.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { TwoPcParticipantRegistry } from "@suaveplan/api-2pc/registry";
|
||||
|
||||
const isPreparable = (pluginId: string) =>
|
||||
pluginId === "billing" || pluginId === "inventory";
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(isPreparable);
|
||||
registry.register({
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: { engine: "postgres", run: pg.unsafe.bind(pg) },
|
||||
prepare: billingPrepareCallback,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Refusing a SQLite-backed plugin as a writer happens at register-time
|
||||
— wrap the registration in a try/catch to surface the failure to the
|
||||
plugin loader without crashing the host boot:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
import { TwoPcParticipantRegistry } from "@suaveplan/api-2pc/registry";
|
||||
|
||||
function safeRegisterParticipants(
|
||||
registry: TwoPcParticipantRegistry,
|
||||
participants: ReadonlyArray<RegisteredTwoPcParticipant>
|
||||
) {
|
||||
const failures: { id: string; reason: string }[] = [];
|
||||
for (const p of participants) {
|
||||
try {
|
||||
registry.register(p);
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof TwoPcError &&
|
||||
err.twoPcCode === TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE
|
||||
) {
|
||||
failures.push({ id: p.id, reason: err.message });
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
```
|
||||
|
||||
The same plugin can then re-register as a `read-only` voter — the
|
||||
registry accepts read-only voters regardless of engine, because the
|
||||
coordinator never sends them prepare/commit SQL.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type PreparableCapabilityCheck = (participantId: string) => boolean`
|
||||
— capability resolver injected at construction.
|
||||
- `class TwoPcParticipantRegistry`
|
||||
- `constructor(isPreparable: PreparableCapabilityCheck)`
|
||||
- `register(participant): void` — throws
|
||||
`TWOPC_PARTICIPANT_NOT_PREPARABLE` on writer refusal.
|
||||
- `get(id): RegisteredTwoPcParticipant | undefined`
|
||||
- `list(): readonly RegisteredTwoPcParticipant[]`
|
||||
- `size(): number`
|
||||
- `unregister(id): boolean`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The registry refuses a writer at `register`-time rather than at
|
||||
`run2pc`-time. This is deliberate — admission failures should surface
|
||||
during plugin install, not when the first 2PC transaction is attempted
|
||||
in production. The capability check fires only for writers; read-only
|
||||
voters are accepted unconditionally because the coordinator drops their
|
||||
connection from phase-2 SQL traffic. The second-half guard against
|
||||
read-only voters that try to write a payload lives in
|
||||
`phases/prepare.ts` (raising `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`
|
||||
when the participant returns `writeIntent: true`).
|
||||
|
||||
`unregister` exists for hot-reload paths: when a plugin is removed
|
||||
from the running host, the registry drops its participant entry so
|
||||
subsequent `run2pc` calls cannot accidentally route to it.
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { TwoPcParticipantRegistry } from "./participant-registry.js";
|
||||
|
||||
const NOOP_PREPARE = async () => ({
|
||||
vote: "commit" as const,
|
||||
writeIntent: false,
|
||||
});
|
||||
|
||||
function makeParticipant(
|
||||
id: string,
|
||||
overrides: Partial<RegisteredTwoPcParticipant> = {}
|
||||
): RegisteredTwoPcParticipant {
|
||||
return {
|
||||
id,
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: NOOP_PREPARE,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("TwoPcParticipantRegistry", () => {
|
||||
it("registers preparable writers", { timeout: 5000 }, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => true);
|
||||
r.register(makeParticipant("billing"));
|
||||
expect(r.size()).toBe(1);
|
||||
expect(r.get("billing")?.id).toBe("billing");
|
||||
});
|
||||
|
||||
it("rejects writer whose engine is not preparable with TWOPC.PARTICIPANT_NOT_PREPARABLE", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => false);
|
||||
try {
|
||||
r.register(
|
||||
makeParticipant("sqlite-plugin", {
|
||||
connection: {
|
||||
engine: "sqlite",
|
||||
run: async () => undefined,
|
||||
},
|
||||
})
|
||||
);
|
||||
throw new Error("should have thrown");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts read-only voter regardless of engine", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => false);
|
||||
r.register(
|
||||
makeParticipant("audit", {
|
||||
voterMode: "read-only",
|
||||
connection: {
|
||||
engine: "surrealdb",
|
||||
run: async () => undefined,
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(r.size()).toBe(1);
|
||||
expect(r.get("audit")?.voterMode).toBe("read-only");
|
||||
});
|
||||
|
||||
it("lists every registered participant", { timeout: 5000 }, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => true);
|
||||
r.register(makeParticipant("a"));
|
||||
r.register(makeParticipant("b"));
|
||||
expect(
|
||||
r
|
||||
.list()
|
||||
.map((p) => p.id)
|
||||
.sort()
|
||||
).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("unregister removes the entry and reports outcome", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => true);
|
||||
r.register(makeParticipant("c"));
|
||||
expect(r.unregister("c")).toBe(true);
|
||||
expect(r.unregister("c")).toBe(false);
|
||||
expect(r.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Capability assertion handed to the registry. The host's capability
|
||||
* registry implements this — the registry only needs to know whether
|
||||
* a specific participant's engine is `db.preparable`. Modelling this
|
||||
* as a structural callback keeps the registry free of any direct
|
||||
* dependency on `@suaveplan/api-host`.
|
||||
*/
|
||||
export type PreparableCapabilityCheck = (participantId: string) => boolean;
|
||||
|
||||
/**
|
||||
* Registry of participants the coordinator can hand a transaction to.
|
||||
* Refuses writers whose bound engine is not `db.preparable` (SQLite,
|
||||
* SurrealDB). Read-only voters are allowed regardless of engine but
|
||||
* are refused later if their prepare callback returns
|
||||
* `writeIntent: true` — that refusal lives in `phases/prepare.ts`.
|
||||
*/
|
||||
export class TwoPcParticipantRegistry {
|
||||
private readonly entries = new Map<string, RegisteredTwoPcParticipant>();
|
||||
|
||||
public constructor(
|
||||
private readonly isPreparable: PreparableCapabilityCheck
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Add a participant. Throws
|
||||
* `TWOPC_PARTICIPANT_NOT_PREPARABLE` when a writer's engine does
|
||||
* not advertise the `db.preparable` capability.
|
||||
*/
|
||||
public register(participant: RegisteredTwoPcParticipant): void {
|
||||
if (
|
||||
participant.voterMode === "writer" &&
|
||||
!this.isPreparable(participant.id)
|
||||
) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE,
|
||||
`Participant ${participant.id} is not preparable (engine: ${participant.connection.engine})`,
|
||||
{
|
||||
context: {
|
||||
participantId: participant.id,
|
||||
engine: participant.connection.engine,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
this.entries.set(participant.id, participant);
|
||||
}
|
||||
|
||||
/** Lookup by id; returns `undefined` when none registered. */
|
||||
public get(id: string): RegisteredTwoPcParticipant | undefined {
|
||||
return this.entries.get(id);
|
||||
}
|
||||
|
||||
/** Snapshot of every registered participant. */
|
||||
public list(): readonly RegisteredTwoPcParticipant[] {
|
||||
return Array.from(this.entries.values());
|
||||
}
|
||||
|
||||
/** Total registered participants — useful for diagnostic assertions. */
|
||||
public size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
/** Remove a participant; returns `true` when one was removed. */
|
||||
public unregister(id: string): boolean {
|
||||
return this.entries.delete(id);
|
||||
}
|
||||
}
|
||||
100
packages/api-stack/api/api-2pc/src/testing/conformance.md
Normal file
100
packages/api-stack/api/api-2pc/src/testing/conformance.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# `testing/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides `run2pcParticipantConformance(factory)` — a runnable test
|
||||
suite that pins the contract every 2PC writer participant MUST honour.
|
||||
Adapter authors drop a single import + invocation into a
|
||||
`tests/conformance/run-2pc-participant.test.ts` file in their package;
|
||||
the harness handles the rest.
|
||||
|
||||
Shipping the harness on this package's `./testing` subpath keeps
|
||||
participants accountable to the same prepare/commit/rollback semantics
|
||||
the coordinator depends on at runtime. Without this harness, drift
|
||||
between participant implementations would only surface during
|
||||
integration testing — too late.
|
||||
|
||||
## Features
|
||||
|
||||
- Six pinned invariants (engine identifier, voter mode, prepare-then-
|
||||
prepare-SQL, commit-prepared, rollback-prepared, idempotency-on-
|
||||
second-rollback).
|
||||
- Per-scenario factory: every test starts from a fresh participant +
|
||||
teardown so the suite is isolation-safe under parallel runs.
|
||||
- Uses `@suaveplan/testing/runner` — works under both Vitest and
|
||||
Bun:test without re-importing.
|
||||
- Generates unique gids per scenario so re-runs do not collide.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
// In an adapter package's tests/conformance/run-2pc-participant.test.ts
|
||||
import { run2pcParticipantConformance } from "@suaveplan/api-2pc/testing";
|
||||
import { createMyParticipant, teardownMyParticipant } from "../helpers";
|
||||
|
||||
run2pcParticipantConformance({
|
||||
async create() {
|
||||
const participant = await createMyParticipant();
|
||||
return {
|
||||
participant,
|
||||
teardown: () => teardownMyParticipant(participant),
|
||||
};
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The harness exposes no surface beyond `run2pcParticipantConformance` —
|
||||
deliberately. Adapter-specific assertions belong in adapter test
|
||||
files, not in the conformance harness. If your adapter needs to
|
||||
exercise behaviour the suite does not cover (e.g. cancellation,
|
||||
custom serialization), add a sibling test file:
|
||||
|
||||
```ts
|
||||
import { describe, it } from "@suaveplan/testing/runner";
|
||||
import { run2pcParticipantConformance } from "@suaveplan/api-2pc/testing";
|
||||
|
||||
describe("my-participant", () => {
|
||||
run2pcParticipantConformance({ create: createMyParticipant });
|
||||
|
||||
it("cancels a stuck prepare on signal abort", { timeout: 30_000 }, async () => {
|
||||
// adapter-specific assertion
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface TwoPcParticipantFactory`
|
||||
- `create(): Promise<{ participant: RegisteredTwoPcParticipant; teardown: () => Promise<void> }>`
|
||||
- `function run2pcParticipantConformance(factory): void`
|
||||
- Registers six `describe`/`it` blocks against the surrounding
|
||||
runner. Returns nothing.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The harness deliberately does NOT use a shared Postgres connection
|
||||
across scenarios. Each scenario calls `factory.create()` to get a
|
||||
fresh participant; the factory is expected to wire its own connection
|
||||
(directly to a Postgres container, against a shared pool, or against
|
||||
an `env-bridge` passthrough URL). This isolation prevents one
|
||||
scenario's leftover prepared transactions from contaminating the
|
||||
next.
|
||||
|
||||
Gids embed `Date.now()` + a random suffix so re-runs within the same
|
||||
millisecond never collide on the underlying `pg_prepared_xacts`
|
||||
table. The cleanup `ROLLBACK PREPARED` at the end of the
|
||||
prepare-only scenario is best-effort: failure to roll back leaves
|
||||
the gid prepared until the next teardown sweep, but the harness's
|
||||
own teardown handle is the primary cleanup path.
|
||||
|
||||
The harness is intentionally NOT exported from the package root (`./`
|
||||
subpath) — only `./testing`. This mirrors the convention used by
|
||||
sibling packages (`@suaveplan/i18n-shared/testing`,
|
||||
`@suaveplan/api-host/testing`) so production code can not accidentally
|
||||
depend on test-only symbols.
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { run2pcParticipantConformance } from "./conformance.js";
|
||||
|
||||
/**
|
||||
* Build a fake Postgres-shaped participant that maintains an internal
|
||||
* set of "prepared" gids. Used purely to exercise the conformance
|
||||
* harness against deterministic semantics — the actual integration
|
||||
* test runs the same harness against a real Postgres container.
|
||||
*/
|
||||
function fakeParticipantFactory(): {
|
||||
create(): Promise<{
|
||||
participant: RegisteredTwoPcParticipant;
|
||||
teardown(): Promise<void>;
|
||||
}>;
|
||||
} {
|
||||
return {
|
||||
async create() {
|
||||
const prepared = new Set<string>();
|
||||
const participant: RegisteredTwoPcParticipant = {
|
||||
id: "fake",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
const trimmed = sql.trim();
|
||||
const match = trimmed.match(
|
||||
/^(PREPARE TRANSACTION|COMMIT PREPARED|ROLLBACK PREPARED) '(.+)'$/
|
||||
);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
const verb = match[1] as string;
|
||||
const gid = (match[2] as string).replace(/''/g, "'");
|
||||
if (verb === "PREPARE TRANSACTION") {
|
||||
prepared.add(gid);
|
||||
return undefined;
|
||||
}
|
||||
if (verb === "COMMIT PREPARED") {
|
||||
if (!prepared.has(gid)) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`prepared transaction with identifier '${gid}' does not exist`
|
||||
),
|
||||
{ code: "42704" }
|
||||
);
|
||||
}
|
||||
prepared.delete(gid);
|
||||
return undefined;
|
||||
}
|
||||
if (verb === "ROLLBACK PREPARED") {
|
||||
if (!prepared.has(gid)) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`prepared transaction with identifier '${gid}' does not exist`
|
||||
),
|
||||
{ code: "42704" }
|
||||
);
|
||||
}
|
||||
prepared.delete(gid);
|
||||
return undefined;
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}),
|
||||
};
|
||||
return {
|
||||
participant,
|
||||
async teardown() {
|
||||
prepared.clear();
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
run2pcParticipantConformance(fakeParticipantFactory());
|
||||
|
||||
describe("run2pcParticipantConformance smoke", () => {
|
||||
it("is callable without throwing at module load", { timeout: 5000 }, () => {
|
||||
expect(typeof run2pcParticipantConformance).toBe("function");
|
||||
});
|
||||
});
|
||||
155
packages/api-stack/api/api-2pc/src/testing/conformance.ts
Normal file
155
packages/api-stack/api/api-2pc/src/testing/conformance.ts
Normal file
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Factory the conformance suite calls before every scenario. Returns
|
||||
* a fresh participant + a teardown handle. The harness drives
|
||||
* prepare/commit/rollback semantics against the participant; the
|
||||
* factory's only job is to wire a real (or sufficiently realistic)
|
||||
* Postgres connection in the returned `participant.connection`.
|
||||
*/
|
||||
export interface TwoPcParticipantFactory {
|
||||
create(): Promise<{
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
teardown(): Promise<void>;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Conformance suite. Re-exported on the `./testing` subpath so every
|
||||
* participant implementation can drop the following one-liner into
|
||||
* its `tests/conformance/run-2pc-participant.test.ts`:
|
||||
*
|
||||
* `run2pcParticipantConformance({ create: createMyParticipant })`
|
||||
*
|
||||
* The suite asserts six invariants:
|
||||
*
|
||||
* 1. A writer's prepare callback returning `commit` must be valid
|
||||
* for the coordinator to subsequently call `connection.run` with
|
||||
* `PREPARE TRANSACTION '<gid>'` and have the statement succeed
|
||||
* against the underlying engine.
|
||||
* 2. `COMMIT PREPARED '<gid>'` against a prepared gid must succeed.
|
||||
* 3. `ROLLBACK PREPARED '<gid>'` against a prepared gid must
|
||||
* succeed.
|
||||
* 4. A second `ROLLBACK PREPARED` against the same already-rolled-back
|
||||
* gid must raise SQLSTATE `42704` (or a "does not exist" message),
|
||||
* proving the idempotency anchor the coordinator depends on.
|
||||
* 5. The participant's `connection.engine` must be `"postgres"`.
|
||||
* 6. The participant's `voterMode` must be `"writer"` (read-only
|
||||
* voters are not subject to this suite — they do not run prepare
|
||||
* SQL).
|
||||
*/
|
||||
export function run2pcParticipantConformance(
|
||||
factory: TwoPcParticipantFactory
|
||||
): void {
|
||||
describe("2PC participant conformance", () => {
|
||||
it("declares engine 'postgres' and voterMode 'writer'", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
try {
|
||||
expect(participant.connection.engine).toBe("postgres");
|
||||
expect(participant.voterMode).toBe("writer");
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("PREPARE TRANSACTION succeeds after a commit-voted prepare callback", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
const vote = await participant.prepare(gid, {});
|
||||
expect(vote.vote).toBe("commit");
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
// Cleanup so the test doesn't leave row locks behind.
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("COMMIT PREPARED resolves a prepared gid", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-commit-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await participant.prepare(gid, {});
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
await participant.connection.run(
|
||||
`COMMIT PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("ROLLBACK PREPARED resolves a prepared gid", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-rollback-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await participant.prepare(gid, {});
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("a second ROLLBACK PREPARED raises the documented 'does not exist' error", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-idem-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await participant.prepare(gid, {});
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
let caught: unknown;
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeDefined();
|
||||
const e = caught as { code?: string; message?: string };
|
||||
const matches =
|
||||
e.code === "42704" ||
|
||||
(typeof e.message === "string" &&
|
||||
e.message.includes("does not exist"));
|
||||
expect(matches).toBe(true);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
4
packages/api-stack/api/api-2pc/src/testing/index.ts
Normal file
4
packages/api-stack/api/api-2pc/src/testing/index.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
run2pcParticipantConformance,
|
||||
type TwoPcParticipantFactory,
|
||||
} from "./conformance.js";
|
||||
8
packages/api-stack/api/api-2pc/src/types/index.ts
Normal file
8
packages/api-stack/api/api-2pc/src/types/index.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
export type {
|
||||
PreparableConnection,
|
||||
RegisteredTwoPcParticipant,
|
||||
Run2PcInput,
|
||||
Run2PcOutcome,
|
||||
TwoPcCoordinatorState,
|
||||
TwoPcPrepareResult,
|
||||
} from "./types.types.js";
|
||||
108
packages/api-stack/api/api-2pc/src/types/types.types.md
Normal file
108
packages/api-stack/api/api-2pc/src/types/types.types.md
Normal file
@@ -0,0 +1,108 @@
|
||||
# `types/`
|
||||
|
||||
## Purpose
|
||||
|
||||
Shared cross-module type definitions for the 2PC coordinator. Holds
|
||||
the `TwoPcCoordinatorState` state-machine alphabet, the
|
||||
`PreparableConnection` and `RegisteredTwoPcParticipant` participant
|
||||
shapes, the `Run2PcInput` / `Run2PcOutcome` public input/output
|
||||
contracts, and the `TwoPcPrepareResult` per-participant prepare
|
||||
return type.
|
||||
|
||||
Centralising these types here prevents the cyclic-import smell that
|
||||
would otherwise emerge from `coordinator/`, `phases/`, `recovery/`,
|
||||
and `registry/` all needing to know the same wire shapes. The
|
||||
sibling modules import from this barrel and stay decoupled from each
|
||||
other at the type level.
|
||||
|
||||
## Features
|
||||
|
||||
- `TwoPcCoordinatorState` — string-literal union covering
|
||||
`voting`, `committing`, `aborting`, `committed`, `aborted`,
|
||||
`in_doubt`.
|
||||
- `PreparableConnection` — structural connection shape exposing
|
||||
`engine` (a `DatabaseEngine` from `@suaveplan/api-types/db`) and
|
||||
`run(sql, params?)` returning a `Promise<R>`.
|
||||
- `RegisteredTwoPcParticipant` — coordinator-side participant view
|
||||
carrying `id`, `voterMode`, `connection`, and an async
|
||||
`prepare(txId, payload) => TwoPcPrepareResult` callback.
|
||||
- `TwoPcPrepareResult` — `{ vote, writeIntent }` returned by a
|
||||
participant's `prepare` callback.
|
||||
- `Run2PcInput` / `Run2PcOutcome` — public input/output contracts
|
||||
accepted and returned by `run2pc`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type {
|
||||
Run2PcInput,
|
||||
RegisteredTwoPcParticipant,
|
||||
} from "@suaveplan/api-2pc";
|
||||
|
||||
const input: Run2PcInput = {
|
||||
participants: [{ id: "billing", input: { invoiceId: "inv-1" } }],
|
||||
timeoutMs: 5000,
|
||||
};
|
||||
|
||||
const billing: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: { engine: "postgres", run: pg.unsafe.bind(pg) },
|
||||
prepare: async (_txId) => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Read-only voter typing — combine `voterMode: "read-only"` with a
|
||||
`writeIntent: false` invariant on the `prepare` return shape:
|
||||
|
||||
```ts
|
||||
import type {
|
||||
RegisteredTwoPcParticipant,
|
||||
TwoPcPrepareResult,
|
||||
} from "@suaveplan/api-2pc";
|
||||
|
||||
async function auditPrepare(): Promise<TwoPcPrepareResult> {
|
||||
const ok = await checkConsistency();
|
||||
return { vote: ok ? "commit" : "abort", writeIntent: false };
|
||||
}
|
||||
|
||||
const audit: RegisteredTwoPcParticipant = {
|
||||
id: "audit",
|
||||
voterMode: "read-only",
|
||||
connection: { engine: "surrealdb", run: surrealUnsafe },
|
||||
prepare: () => auditPrepare(),
|
||||
};
|
||||
```
|
||||
|
||||
The coordinator refuses a read-only voter whose `prepare` returns
|
||||
`writeIntent: true` with `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`
|
||||
— the static type doesn't enforce that pairing, but the runtime
|
||||
boundary does.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type TwoPcCoordinatorState`
|
||||
- `interface PreparableConnection`
|
||||
- `interface RegisteredTwoPcParticipant`
|
||||
- `interface TwoPcPrepareResult`
|
||||
- `interface Run2PcInput`
|
||||
- `interface Run2PcOutcome`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Every interface is `readonly` on every field. This is deliberate —
|
||||
the coordinator caches inputs (participant lookups, ledger entries)
|
||||
and a mutation by accident would corrupt in-flight state. The
|
||||
strictness is enforced at the type level via `readonly` plus
|
||||
`ReadonlyArray<...>` and at the runtime level by the boundary Zod
|
||||
schemas in `options/` (for caller-facing inputs).
|
||||
|
||||
The `Run2PcOutcome.state` field uses `Extract<TwoPcCoordinatorState,
|
||||
"committed" | "aborted" | "in_doubt">` so callers do not have to
|
||||
exhaustively switch over the in-flight states (`voting`,
|
||||
`committing`, `aborting`) that never appear on a returned outcome.
|
||||
The structural typing of `PreparableConnection` keeps the package
|
||||
independent of `pg` and `postgres` (porsager) — both clients can be
|
||||
adapted at the seam with a one-line wrapper.
|
||||
117
packages/api-stack/api/api-2pc/src/types/types.types.ts
Normal file
117
packages/api-stack/api/api-2pc/src/types/types.types.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
import type {
|
||||
DatabaseEngine,
|
||||
PreparedStatement,
|
||||
} from "@suaveplan/api-types/db";
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
|
||||
/**
|
||||
* Coordinator-side lifecycle state machine for a single 2PC run. Mirrors
|
||||
* the design doc §9.2 vocabulary (`voting → committing | aborting →
|
||||
* committed | aborted | in_doubt`) — `voting` is the prepare phase,
|
||||
* the terminal states are persisted to `suaveplan_2pc_transactions` for
|
||||
* recovery.
|
||||
*/
|
||||
export type TwoPcCoordinatorState =
|
||||
| "voting"
|
||||
| "committing"
|
||||
| "aborting"
|
||||
| "committed"
|
||||
| "aborted"
|
||||
| "in_doubt";
|
||||
|
||||
/**
|
||||
* Participant write surface used by the coordinator during phase 1 and
|
||||
* phase 2. Modelled as a single structural method `run(sql, params)`
|
||||
* to match `postgres` (porsager/postgres)'s `sql.unsafe(...)` and
|
||||
* `pg.Client.query(text, values)` interchangeably. Returning a row
|
||||
* shape is irrelevant for 2PC SQL — the coordinator only cares that
|
||||
* the statement either succeeds or throws.
|
||||
*/
|
||||
export interface PreparableConnection {
|
||||
/** Engine identifier — coordinator gates on `db.preparable` capability. */
|
||||
readonly engine: DatabaseEngine;
|
||||
/**
|
||||
* Execute an arbitrary SQL statement. Used for
|
||||
* `PREPARE TRANSACTION '<gid>'`, `COMMIT PREPARED '<gid>'`,
|
||||
* `ROLLBACK PREPARED '<gid>'`, and the recovery's
|
||||
* `SELECT gid FROM pg_prepared_xacts ...` probe.
|
||||
*/
|
||||
run<R = unknown>(sql: string, params?: ReadonlyArray<unknown>): Promise<R>;
|
||||
/**
|
||||
* Optional prepared-statement bridge. The 2PC coordinator does not
|
||||
* use prepared statements itself (every call is one-shot SQL), but
|
||||
* the participant SHOULD expose its production `prepare` hook so
|
||||
* the in-package conformance suite can verify the engine actually
|
||||
* supports prepared transactions.
|
||||
*/
|
||||
prepare?<R = unknown>(sql: string): Promise<PreparedStatement<R>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coordinator-side view of a participant. The `id` is unique per
|
||||
* coordinator and is folded into the global transaction id passed to
|
||||
* `PREPARE TRANSACTION '<gid>'`. `voterMode` distinguishes write
|
||||
* participants (full prepare/commit/rollback SQL) from read-only
|
||||
* voters (vote-only — no SQL crosses the wire in phase 2).
|
||||
*/
|
||||
export interface RegisteredTwoPcParticipant {
|
||||
readonly id: string;
|
||||
readonly voterMode: "writer" | "read-only";
|
||||
readonly connection: PreparableConnection;
|
||||
/**
|
||||
* Prepare callback — runs the participant's business work inside
|
||||
* a transaction, returns `commit` if the work passed local
|
||||
* invariants. The coordinator turns a `commit` vote into a
|
||||
* `PREPARE TRANSACTION` SQL call. Write-intent payloads from a
|
||||
* read-only voter are rejected with
|
||||
* `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`.
|
||||
*/
|
||||
prepare(
|
||||
txId: string,
|
||||
input: Readonly<Record<string, unknown>>
|
||||
): Promise<TwoPcPrepareResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return value from a participant's prepare callback. `writeIntent`
|
||||
* carries whether the work would mutate state — read-only voters MUST
|
||||
* return `writeIntent: false` (the coordinator refuses otherwise).
|
||||
*/
|
||||
export interface TwoPcPrepareResult {
|
||||
readonly vote: TwoPcVote;
|
||||
readonly writeIntent: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caller-supplied input to `run2pc`. `participants` is an ordered list
|
||||
* — phase 1 collects votes left-to-right, phase 2 commits/rolls back
|
||||
* left-to-right. `timeoutMs` is the phase-1 deadline; phase 2 is
|
||||
* unbounded by design (the protocol cannot tolerate giving up on
|
||||
* commit-prepared mid-flight).
|
||||
*/
|
||||
export interface Run2PcInput {
|
||||
readonly participants: ReadonlyArray<{
|
||||
readonly id: string;
|
||||
readonly input: Readonly<Record<string, unknown>>;
|
||||
}>;
|
||||
readonly timeoutMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only outcome handed back from `run2pc`. Includes the final
|
||||
* coordinator state and the participant votes — useful for callers
|
||||
* that want to surface per-participant outcomes in audit logs.
|
||||
*/
|
||||
export interface Run2PcOutcome {
|
||||
readonly txId: string;
|
||||
readonly state: Extract<
|
||||
TwoPcCoordinatorState,
|
||||
"committed" | "aborted" | "in_doubt"
|
||||
>;
|
||||
readonly votes: ReadonlyArray<{
|
||||
readonly participantId: string;
|
||||
readonly vote: TwoPcVote;
|
||||
}>;
|
||||
readonly prepareDurationMs: number;
|
||||
readonly commitDurationMs: number;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Vitest globalSetup — registers the Postgres fixture in the
|
||||
* globalSetup process's module scope.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
import { createIntegrationSetup } from "@suaveplan/testing-integration";
|
||||
import { PostgresFixture } from "@suaveplan/testing-integration-postgres";
|
||||
|
||||
createIntegrationSetup({
|
||||
postgres: new PostgresFixture(),
|
||||
});
|
||||
|
||||
export {
|
||||
setup,
|
||||
teardown,
|
||||
} from "@suaveplan/testing-integration/setup/global-setup";
|
||||
@@ -0,0 +1,33 @@
|
||||
import { inject as vitestInject } from "@suaveplan/testing/runner";
|
||||
import type { PostgresConfig } from "@suaveplan/testing-integration-postgres";
|
||||
|
||||
interface IntegrationConfigs {
|
||||
postgres: PostgresConfig;
|
||||
}
|
||||
|
||||
export function inject<K extends keyof IntegrationConfigs>(
|
||||
key: K
|
||||
): IntegrationConfigs[K] {
|
||||
const all = (vitestInject as (k: string) => IntegrationConfigs)(
|
||||
"__integration_configs__"
|
||||
);
|
||||
return all[key];
|
||||
}
|
||||
|
||||
export const postgresUnavailable: boolean = (() => {
|
||||
try {
|
||||
const failed = (vitestInject as (k: string) => string[] | undefined)(
|
||||
"__integration_failed__"
|
||||
);
|
||||
if (failed === undefined) return true;
|
||||
if (failed.includes("postgres")) return true;
|
||||
const configs = (
|
||||
vitestInject as (
|
||||
k: string
|
||||
) => Record<string, Record<string, unknown>> | undefined
|
||||
)("__integration_configs__");
|
||||
return configs === undefined || configs.postgres === undefined;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* Real-Postgres integration test for the 2PC coordinator. Drives a
|
||||
* single coordinator against TWO Postgres connections (acting as two
|
||||
* participants on the same physical instance) and asserts the full
|
||||
* commit + abort + recovery scenarios end-to-end.
|
||||
*
|
||||
* NOTE: 2PC requires `max_prepared_transactions > 0` in the Postgres
|
||||
* config. The default-image fixture starts with `max_prepared_transactions=0`,
|
||||
* so this test bootstraps a fresh config and reloads it via SQL before
|
||||
* running. We can't ALTER SYSTEM SET inside an in-memory fixture without
|
||||
* restarting, so we run the prepare against an in-test-cluster ALTER
|
||||
* SYSTEM + SELECT pg_reload_conf() flow that postgres supports without
|
||||
* a full restart.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import postgres from "postgres";
|
||||
import {
|
||||
createTwoPcCoordinator,
|
||||
type LedgerDb,
|
||||
type RegisteredTwoPcParticipant,
|
||||
TwoPcParticipantRegistry,
|
||||
} from "../../src/index.js";
|
||||
import { TwoPcLedgerStore } from "../../src/persistence/persistence.js";
|
||||
import {
|
||||
buildProbeFromRegistry,
|
||||
resolveRecovery,
|
||||
} from "../../src/recovery/index.js";
|
||||
import { inject, postgresUnavailable } from "./integration-setup.js";
|
||||
|
||||
interface PgClient {
|
||||
readonly engine: "postgres";
|
||||
run<R = unknown>(sql: string, params?: ReadonlyArray<unknown>): Promise<R>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
function makeClient(url: string): PgClient {
|
||||
const sql = postgres(url, { max: 4, prepare: false });
|
||||
return {
|
||||
engine: "postgres",
|
||||
async run<R>(text: string, params?: ReadonlyArray<unknown>) {
|
||||
const out = await sql.unsafe(text, params as unknown[]);
|
||||
return out as unknown as R;
|
||||
},
|
||||
async close() {
|
||||
await sql.end({ timeout: 1 });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureMaxPrepared(adminUrl: string): Promise<void> {
|
||||
const sql = postgres(adminUrl, { max: 1, prepare: false });
|
||||
try {
|
||||
const cur = (await sql.unsafe(
|
||||
"SHOW max_prepared_transactions"
|
||||
)) as unknown as { max_prepared_transactions: string }[];
|
||||
const current = Number.parseInt(
|
||||
cur[0]?.max_prepared_transactions ?? "0",
|
||||
10
|
||||
);
|
||||
if (Number.isFinite(current) && current > 0) return;
|
||||
// ALTER SYSTEM requires a restart for max_prepared_transactions.
|
||||
// For a fresh fixture this is the first run; we cannot reload
|
||||
// dynamically, so we surface that the test cannot proceed.
|
||||
throw new Error(
|
||||
"max_prepared_transactions=0 on the Postgres fixture; cannot run 2PC integration tests"
|
||||
);
|
||||
} finally {
|
||||
await sql.end({ timeout: 1 });
|
||||
}
|
||||
}
|
||||
|
||||
function makeLedger(): { ledgerDb: LedgerDb; ledger: TwoPcLedgerStore } {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
return { ledgerDb: db, ledger: new TwoPcLedgerStore(db) };
|
||||
}
|
||||
|
||||
describe.skipIf(postgresUnavailable)("real Postgres 2PC coordinator", () => {
|
||||
it("commits successfully across two participants", {
|
||||
timeout: 60000,
|
||||
}, async () => {
|
||||
const { url } = inject("postgres");
|
||||
await ensureMaxPrepared(url);
|
||||
const c1 = makeClient(url);
|
||||
const c2 = makeClient(url);
|
||||
await c1.run("CREATE TABLE IF NOT EXISTS twopc_smoke_a (n int)");
|
||||
await c2.run("CREATE TABLE IF NOT EXISTS twopc_smoke_b (n int)");
|
||||
await c1.run("DELETE FROM twopc_smoke_a");
|
||||
await c2.run("DELETE FROM twopc_smoke_b");
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
const a: RegisteredTwoPcParticipant = {
|
||||
id: "a",
|
||||
voterMode: "writer",
|
||||
connection: c1,
|
||||
prepare: async (txId) => {
|
||||
await c1.run("BEGIN");
|
||||
await c1.run("INSERT INTO twopc_smoke_a (n) VALUES (1)");
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
const b: RegisteredTwoPcParticipant = {
|
||||
id: "b",
|
||||
voterMode: "writer",
|
||||
connection: c2,
|
||||
prepare: async (txId) => {
|
||||
await c2.run("BEGIN");
|
||||
await c2.run("INSERT INTO twopc_smoke_b (n) VALUES (2)");
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
registry.register(a);
|
||||
registry.register(b);
|
||||
|
||||
const { ledgerDb } = makeLedger();
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-it" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () =>
|
||||
`tx-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
expect(outcome.state).toBe("committed");
|
||||
const rowsA = (await c1.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_a"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
const rowsB = (await c2.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_b"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
expect(rowsA[0]?.c).toBe(1);
|
||||
expect(rowsB[0]?.c).toBe(1);
|
||||
|
||||
await c1.close();
|
||||
await c2.close();
|
||||
});
|
||||
|
||||
it("rolls back everywhere when one participant votes abort", {
|
||||
timeout: 60000,
|
||||
}, async () => {
|
||||
const { url } = inject("postgres");
|
||||
const c1 = makeClient(url);
|
||||
const c2 = makeClient(url);
|
||||
await c1.run("CREATE TABLE IF NOT EXISTS twopc_smoke_c (n int)");
|
||||
await c2.run("CREATE TABLE IF NOT EXISTS twopc_smoke_d (n int)");
|
||||
await c1.run("DELETE FROM twopc_smoke_c");
|
||||
await c2.run("DELETE FROM twopc_smoke_d");
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "c",
|
||||
voterMode: "writer",
|
||||
connection: c1,
|
||||
prepare: async () => {
|
||||
await c1.run("BEGIN");
|
||||
await c1.run("INSERT INTO twopc_smoke_c (n) VALUES (3)");
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
});
|
||||
registry.register({
|
||||
id: "d",
|
||||
voterMode: "writer",
|
||||
connection: c2,
|
||||
prepare: async () => {
|
||||
return { vote: "abort", writeIntent: false };
|
||||
},
|
||||
});
|
||||
|
||||
const { ledgerDb } = makeLedger();
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-it" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () =>
|
||||
`tx-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
});
|
||||
|
||||
await expect(
|
||||
coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "c", input: {} },
|
||||
{ id: "d", input: {} },
|
||||
],
|
||||
})
|
||||
).rejects.toMatchObject({ twoPcCode: "TWOPC.VOTE_ABORT" });
|
||||
|
||||
const rowsC = (await c1.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_c"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
expect(rowsC[0]?.c).toBe(0);
|
||||
|
||||
await c1.close();
|
||||
await c2.close();
|
||||
});
|
||||
|
||||
it("recovery commits an orphan whose ledger row says committing", {
|
||||
timeout: 60000,
|
||||
}, async () => {
|
||||
const { url } = inject("postgres");
|
||||
const c1 = makeClient(url);
|
||||
await c1.run("CREATE TABLE IF NOT EXISTS twopc_smoke_e (n int)");
|
||||
await c1.run("DELETE FROM twopc_smoke_e");
|
||||
|
||||
const { ledger } = makeLedger();
|
||||
// Simulate a crash: PREPARE the transaction, write the ledger row
|
||||
// as "committing", but never run COMMIT PREPARED.
|
||||
const txId = `recover-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2, 8)}`;
|
||||
const gid = `2pc-host-it-${txId}-e`;
|
||||
await c1.run("BEGIN");
|
||||
await c1.run("INSERT INTO twopc_smoke_e (n) VALUES (42)");
|
||||
await c1.run(`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`);
|
||||
ledger.insert({
|
||||
id: txId,
|
||||
coordinator: "host-it",
|
||||
participants: [
|
||||
{ participantId: "e", voterMode: "writer", vote: "commit" },
|
||||
],
|
||||
});
|
||||
ledger.markAllVotedCommit({
|
||||
id: txId,
|
||||
participants: [
|
||||
{ participantId: "e", voterMode: "writer", vote: "commit" },
|
||||
],
|
||||
preparedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "e",
|
||||
voterMode: "writer",
|
||||
connection: c1,
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "host-it",
|
||||
ledger,
|
||||
probes: buildProbeFromRegistry(registry),
|
||||
});
|
||||
expect(report.committed).toContain(gid);
|
||||
const rows = (await c1.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_e"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
expect(rows[0]?.c).toBe(1);
|
||||
|
||||
await c1.close();
|
||||
});
|
||||
});
|
||||
12
packages/api-stack/api/api-2pc/tsconfig.lib.json
Normal file
12
packages/api-stack/api/api-2pc/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
37
packages/api-stack/api/api-2pc/vite.config.ts
Normal file
37
packages/api-stack/api/api-2pc/vite.config.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/api-2pc",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
/^@suaveplan\//,
|
||||
/^node:/,
|
||||
/^@opentelemetry\//,
|
||||
"tslib",
|
||||
"zod",
|
||||
"postgres",
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
31
packages/api-stack/api/api-2pc/vitest.config.ts
Normal file
31
packages/api-stack/api/api-2pc/vitest.config.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
include: ["src/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**", "tests/integration/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/index.ts",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
19
packages/api-stack/api/api-2pc/vitest.integration.config.ts
Normal file
19
packages/api-stack/api/api-2pc/vitest.integration.config.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
const globalSetup = fileURLToPath(
|
||||
new URL("./tests/integration/global-setup.ts", import.meta.url)
|
||||
);
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
globalSetup: [globalSetup],
|
||||
include: ["tests/integration/**/*.integration.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**", "src/**"],
|
||||
testTimeout: 90000,
|
||||
hookTimeout: 180000,
|
||||
},
|
||||
});
|
||||
85
packages/api-stack/api/api-audit-file/README.md
Normal file
85
packages/api-stack/api/api-audit-file/README.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# @suaveplan/api-audit-file — audit sink writing JSON-per-line, HMAC-SHA-256 signed records to a rotated file. Size + time rotation, maxFiles retention, bounded-queue backpressure with observable drops, append + strict-sync write modes. Implements `RequestAuditSink` from `@suaveplan/api-host/request-pipeline`.
|
||||
|
||||
File-rotating, HMAC-signed audit sink for `@suaveplan/api-host`.
|
||||
Implements `RequestAuditSink` from
|
||||
`@suaveplan/api-host/request-pipeline`: writes one JSON line per
|
||||
record with an HMAC-SHA-256 signature, rolls the file on a size OR age
|
||||
trigger, and prunes the oldest rotated siblings to a configurable
|
||||
`maxFiles` retention cap.
|
||||
|
||||
Suitable for single-host deployments, on-prem environments without a
|
||||
log shipper, and any host that prefers durable local storage of the
|
||||
audit trail. The sibling `@suaveplan/api-audit-stdout` covers the
|
||||
log-shipped use case; both sinks share the wire shape so dashboards
|
||||
union on metric name and partition by the `sink` attribute.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [error-codes](./src/error-codes/error-codes.md) — Frozen registry of the six stable `AUDIT_FILE.*` error codes raised by the file-rotating audit sink, plus the concrete…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — Frozen registry of the six stable `AUDIT_FILE.*` error codes raised by the file-rotating audit sink, plus the concrete…
|
||||
- [fs-helpers](./src/fs-helpers/fs-helpers.md) — Isolates the `node:fs` surface the sink touches behind an injectable `FsAdapter` interface and concrete `nodeFsAdapter`…
|
||||
- [fs-helpers](./src/fs-helpers/fs-helpers.md) — Isolates the `node:fs` surface the sink touches behind an injectable `FsAdapter` interface and concrete `nodeFsAdapter`…
|
||||
- [hmac](./src/hmac/hmac.md) — Wraps `@suaveplan/crypto-hash` HMAC-SHA-256 into the two operations the file sink needs: signing a canonical record on…
|
||||
- [hmac](./src/hmac/hmac.md) — Wraps `@suaveplan/crypto-hash` HMAC-SHA-256 into the two operations the file sink needs: signing a canonical record on…
|
||||
- [instruments](./src/instruments/instruments.md) — Constructs the OTel counter bundle the file sink emits and exposes the canonical `sink="file"` attribute set. Every obs…
|
||||
- [instruments](./src/instruments/instruments.md) — Constructs the OTel counter bundle the file sink emits and exposes the canonical `sink="file"` attribute set. Every obs…
|
||||
- [retention](./src/retention/retention.md) — Enforces the `maxFiles` retention dimension of the rotation policy by deleting the oldest rotated files until at most `…
|
||||
- [retention](./src/retention/retention.md) — Enforces the `maxFiles` retention dimension of the rotation policy by deleting the oldest rotated files until at most `…
|
||||
- [rotation](./src/rotation/rotation.md) — Resolves the rotation policy carried by every `createFileAuditSink` instance and implements the rotation primitive itse…
|
||||
- [rotation](./src/rotation/rotation.md) — Resolves the rotation policy carried by every `createFileAuditSink` instance and implements the rotation primitive itse…
|
||||
- [schema](./src/schema/schema.md) — Defines the Zod schema and canonical-bytes function that bracket every audit record before it is signed and written. Th…
|
||||
- [schema](./src/schema/schema.md) — Defines the Zod schema and canonical-bytes function that bracket every audit record before it is signed and written. Th…
|
||||
- [sink](./src/sink/sink.md) — The public entry point of `@suaveplan/api-audit-file`. The `createFileAuditSink` factory composes the schema, hmac, rot…
|
||||
- [sink](./src/sink/sink.md) — The public entry point of `@suaveplan/api-audit-file`. The `createFileAuditSink` factory composes the schema, hmac, rot…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/api-audit-file
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
```ts
|
||||
import { createFileAuditSink } from "@suaveplan/api-audit-file/sink";
|
||||
|
||||
const sink = await createFileAuditSink({
|
||||
path: "/var/log/suaveplan/audit.log",
|
||||
hmacKey: new TextEncoder().encode(process.env.AUDIT_HMAC_KEY ?? ""),
|
||||
rotation: { maxSizeBytes: 100 * 1024 * 1024, maxAgeMs: 86_400_000, maxFiles: 30 },
|
||||
appendMode: "strict-sync",
|
||||
});
|
||||
|
||||
await sink.emit(envelopeFromHostPipeline);
|
||||
await sink.dispose();
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- `./sink` — `createFileAuditSink`, `FileAuditSink`, `CreateFileAuditSinkOptions`, `AppendMode`.
|
||||
- `./rotation` — `RotationPolicy`, `RotationState`, `resolveRotationPolicy`, `shouldRotate`, `rotate`.
|
||||
- `./retention` — `enforceRetention`.
|
||||
- `./schema` — `AuditRecord`, `AuditRecordSchema`, `canonicalise`, `buildSignedLine`.
|
||||
- `./hmac` — `signRecord`, `verifyRecord`.
|
||||
- `./instruments` — `AuditFileInstruments`, `createAuditFileInstruments`, `SINK_ATTRIBUTE`.
|
||||
- `./fs-helpers` — `FsAdapter`, `nodeFsAdapter`, `listRotatedFiles`, `rotatedFilePathAt`.
|
||||
- `./error-codes` — `AUDIT_FILE_CODES`, `AuditFileError`, `AuditFileCode`.
|
||||
|
||||
## Error codes
|
||||
|
||||
`AUDIT_FILE.INVALID_RECORD`, `AUDIT_FILE.WRITER_FAILED`,
|
||||
`AUDIT_FILE.QUEUE_FULL`, `AUDIT_FILE.ROTATION_FAILED`,
|
||||
`AUDIT_FILE.RETENTION_FAILED`, `AUDIT_FILE.PATH_NOT_WRITEABLE`.
|
||||
|
||||
## OTel instruments
|
||||
|
||||
- `api.audit.records_total{sink="file"}`
|
||||
- `api.audit.dropped_total{sink="file"}`
|
||||
- `api.audit.rotation_evicted_total{sink="file"}`
|
||||
|
||||
## Tier
|
||||
|
||||
Server-tier — depends on `node:fs/promises`.
|
||||
|
||||
108
packages/api-stack/api/api-audit-file/package.json
Normal file
108
packages/api-stack/api/api-audit-file/package.json
Normal file
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"name": "@suaveplan/api-audit-file",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Stage-8 audit sink writing JSON-per-line, HMAC-SHA-256 signed records to a rotated file. Size + time rotation, maxFiles retention, bounded-queue backpressure with observable drops, append + strict-sync write modes. Implements `RequestAuditSink` from `@suaveplan/api-host/request-pipeline`.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./sink": {
|
||||
"import": {
|
||||
"types": "./dist/sink/sink.d.ts",
|
||||
"default": "./dist/sink/sink.js"
|
||||
}
|
||||
},
|
||||
"./rotation": {
|
||||
"import": {
|
||||
"types": "./dist/rotation/rotation.d.ts",
|
||||
"default": "./dist/rotation/rotation.js"
|
||||
}
|
||||
},
|
||||
"./retention": {
|
||||
"import": {
|
||||
"types": "./dist/retention/retention.d.ts",
|
||||
"default": "./dist/retention/retention.js"
|
||||
}
|
||||
},
|
||||
"./schema": {
|
||||
"import": {
|
||||
"types": "./dist/schema/schema.d.ts",
|
||||
"default": "./dist/schema/schema.js"
|
||||
}
|
||||
},
|
||||
"./hmac": {
|
||||
"import": {
|
||||
"types": "./dist/hmac/hmac.d.ts",
|
||||
"default": "./dist/hmac/hmac.js"
|
||||
}
|
||||
},
|
||||
"./error-codes": {
|
||||
"import": {
|
||||
"types": "./dist/error-codes/error-codes.d.ts",
|
||||
"default": "./dist/error-codes/error-codes.js"
|
||||
}
|
||||
},
|
||||
"./instruments": {
|
||||
"import": {
|
||||
"types": "./dist/instruments/instruments.d.ts",
|
||||
"default": "./dist/instruments/instruments.js"
|
||||
}
|
||||
},
|
||||
"./fs-helpers": {
|
||||
"import": {
|
||||
"types": "./dist/fs-helpers/fs-helpers.d.ts",
|
||||
"default": "./dist/fs-helpers/fs-helpers.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/crypto-hash": "workspace:^",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@opentelemetry/sdk-metrics": "1.30.0",
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
# `error-codes`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Frozen registry of the six stable `AUDIT_FILE.*` error codes raised by
|
||||
the file-rotating audit sink, plus the concrete
|
||||
`AuditFileError extends SuaveplanError` subclass that carries them.
|
||||
Operators rely on the codes for dashboard slicing; package consumers
|
||||
rely on the typed subclass to differentiate sink failures from generic
|
||||
host errors without parsing message strings.
|
||||
|
||||
The registry is built with `defineCodes("AUDIT_FILE", [...])` from
|
||||
`@suaveplan/error`, so every code is recorded in the global registry
|
||||
exactly once at module load. The OpenSpec conformance gate verifies
|
||||
every entry here is also raised by a production code path elsewhere in
|
||||
`src/` — no dead codes are permitted.
|
||||
|
||||
## Features
|
||||
|
||||
- Six stable codes: `INVALID_RECORD`, `WRITER_FAILED`, `QUEUE_FULL`,
|
||||
`ROTATION_FAILED`, `RETENTION_FAILED`, `PATH_NOT_WRITEABLE`.
|
||||
- Typed union `AuditFileCode` for narrowing on `error.code`.
|
||||
- `AuditFileError` subclass with stable `error.name === "AuditFileError"`.
|
||||
- Forwards `cause`, `context`, `redacted`, and other
|
||||
`SuaveplanErrorOptions` to the base constructor unchanged.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "@suaveplan/api-audit-file/error-codes";
|
||||
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.PATH_NOT_WRITEABLE,
|
||||
"error.audit.file.path_not_writeable",
|
||||
{ context: { path: "/var/log/suaveplan/audit.log" } }
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Narrow on the union after catching to drive code-specific recovery:
|
||||
|
||||
```ts
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
type AuditFileCode,
|
||||
AuditFileError,
|
||||
} from "@suaveplan/api-audit-file/error-codes";
|
||||
|
||||
function recover(err: unknown): "retry" | "drop" {
|
||||
if (!(err instanceof AuditFileError)) throw err;
|
||||
const code: AuditFileCode = err.code;
|
||||
if (code === AUDIT_FILE_CODES.QUEUE_FULL) return "drop";
|
||||
if (code === AUDIT_FILE_CODES.WRITER_FAILED) return "retry";
|
||||
if (code === AUDIT_FILE_CODES.ROTATION_FAILED) return "retry";
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `AUDIT_FILE_CODES: Readonly<Record<Name, "AUDIT_FILE.<Name>">>`
|
||||
- `type AuditFileCode = (typeof AUDIT_FILE_CODES)[keyof typeof AUDIT_FILE_CODES]`
|
||||
- `AUDIT_FILE_ERROR_NAME = "AuditFileError"`
|
||||
- `class AuditFileError extends SuaveplanError<AuditFileCode>`
|
||||
- `new AuditFileError(code, messageKey, options?)`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The codes are declared `as const` so TypeScript freezes both the keys
|
||||
and the fully-qualified string values. The `name` literal is exported
|
||||
separately because some structured loggers group errors on `name`
|
||||
before they inspect `code` — pinning a constant keeps that grouping
|
||||
stable across releases. The module is side-effect-imported from
|
||||
`src/index.ts` so a consumer who only loads a subpath
|
||||
(`@suaveplan/api-audit-file/sink`) still triggers code registration.
|
||||
@@ -0,0 +1,65 @@
|
||||
import {
|
||||
defineCodes,
|
||||
SuaveplanError,
|
||||
type SuaveplanErrorOptions,
|
||||
} from "@suaveplan/error";
|
||||
|
||||
/**
|
||||
* Frozen registry of stable error codes raised by the
|
||||
* `@suaveplan/api-audit-file` sink. Codes are documented in the design
|
||||
* doc (§12.5) and surface verbatim on `RequestAuditSink` callers and
|
||||
* operator dashboards so post-hoc triage of audit-write failures is
|
||||
* trivial.
|
||||
*
|
||||
* - `INVALID_RECORD` — the record handed to `emit()` failed Zod
|
||||
* validation against the canonical `RequestAuditEnvelope` shape.
|
||||
* - `WRITER_FAILED` — a low-level `node:fs` write surfaced an `EIO`,
|
||||
* `ENOSPC`, or other error after the file was opened successfully.
|
||||
* - `QUEUE_FULL` — the bounded in-memory queue rejected the record
|
||||
* because the writer is stalled and `maxQueueSize` is exceeded.
|
||||
* Always paired with a `dropped_total` increment.
|
||||
* - `ROTATION_FAILED` — size or time rotation failed mid-rename. The
|
||||
* sink stays operational against the current file; the next write
|
||||
* re-attempts rotation.
|
||||
* - `RETENTION_FAILED` — `maxFiles` eviction failed to delete the
|
||||
* oldest rotated file. The sink keeps writing the current file; the
|
||||
* surplus file is reported via the OTel attribute on the next eviction.
|
||||
* - `PATH_NOT_WRITEABLE` — sink construction discovered the path's
|
||||
* parent directory is missing, not a directory, or not writeable.
|
||||
* Raised synchronously from `createFileAuditSink` so misconfiguration
|
||||
* fails fast at host boot.
|
||||
*/
|
||||
export const AUDIT_FILE_CODES = defineCodes("AUDIT_FILE", [
|
||||
"INVALID_RECORD",
|
||||
"WRITER_FAILED",
|
||||
"QUEUE_FULL",
|
||||
"ROTATION_FAILED",
|
||||
"RETENTION_FAILED",
|
||||
"PATH_NOT_WRITEABLE",
|
||||
] as const);
|
||||
|
||||
/**
|
||||
* Union of fully-qualified `AUDIT_FILE.*` codes. Use as the `code`
|
||||
* field type on any `AuditFileError` throwable.
|
||||
*/
|
||||
export type AuditFileCode =
|
||||
(typeof AUDIT_FILE_CODES)[keyof typeof AUDIT_FILE_CODES];
|
||||
|
||||
/** Stable `error.name` value for every throwable this package emits. */
|
||||
export const AUDIT_FILE_ERROR_NAME = "AuditFileError" as const;
|
||||
|
||||
/**
|
||||
* Concrete `SuaveplanError` subclass for file-audit failures. Mirrors
|
||||
* the shape of every other api-stack error class so callers may
|
||||
* `instanceof`-check or code-match without parsing message strings.
|
||||
*/
|
||||
export class AuditFileError extends SuaveplanError<AuditFileCode> {
|
||||
public constructor(
|
||||
code: AuditFileCode,
|
||||
messageKey: string,
|
||||
options?: SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, messageKey, options);
|
||||
this.name = AUDIT_FILE_ERROR_NAME;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
AUDIT_FILE_CODES,
|
||||
AUDIT_FILE_ERROR_NAME,
|
||||
type AuditFileCode,
|
||||
AuditFileError,
|
||||
} from "./error-codes.js";
|
||||
@@ -0,0 +1,86 @@
|
||||
# `fs-helpers`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Isolates the `node:fs` surface the sink touches behind an injectable
|
||||
`FsAdapter` interface and concrete `nodeFsAdapter` implementation. The
|
||||
indirection lets tests run the sink against a real filesystem under a
|
||||
temp directory (the default for this package's test suite) while
|
||||
keeping a clean seam for in-memory adapters used by external consumers.
|
||||
|
||||
The module also exports the rotated-file enumeration helper used by
|
||||
the retention module — keeping the file-name conventions next to the
|
||||
filesystem primitives means rotation, retention, and sink all share
|
||||
the same view of "what counts as a rotated sibling of this base path".
|
||||
|
||||
## Features
|
||||
|
||||
- `FsAdapter` — six-method surface: `appendLine`, `fileSize`, `rename`,
|
||||
`unlink`, `readdir`, `ensureWriteableDir`.
|
||||
- `nodeFsAdapter` — production adapter against `node:fs/promises`.
|
||||
- `listRotatedFiles(fs, basePath)` — returns full paths of every
|
||||
rotated sibling, sorted oldest-first by embedded timestamp.
|
||||
- `rotatedFilePathAt(basePath, iso)` — builds the rotated path for the
|
||||
supplied ISO timestamp; `:` is replaced with `-` so the name is
|
||||
Windows-safe and grep-friendly.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { nodeFsAdapter } from "@suaveplan/api-audit-file/fs-helpers";
|
||||
|
||||
await nodeFsAdapter.ensureWriteableDir("/var/log/suaveplan");
|
||||
await nodeFsAdapter.appendLine("/var/log/suaveplan/audit.log", "...\n", false);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Provide an injectable adapter to drive the sink against an in-memory
|
||||
filesystem:
|
||||
|
||||
```ts
|
||||
import type { FsAdapter } from "@suaveplan/api-audit-file/fs-helpers";
|
||||
|
||||
const inMemory = new Map<string, string>();
|
||||
const adapter: FsAdapter = {
|
||||
async appendLine(p, line) {
|
||||
inMemory.set(p, (inMemory.get(p) ?? "") + line);
|
||||
},
|
||||
async fileSize(p) {
|
||||
return Buffer.byteLength(inMemory.get(p) ?? "", "utf8");
|
||||
},
|
||||
async rename(from, to) {
|
||||
inMemory.set(to, inMemory.get(from) ?? "");
|
||||
inMemory.delete(from);
|
||||
},
|
||||
async unlink(p) {
|
||||
inMemory.delete(p);
|
||||
},
|
||||
async readdir() {
|
||||
return [...inMemory.keys()];
|
||||
},
|
||||
async ensureWriteableDir() {
|
||||
/* always writeable */
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface FsAdapter` — see Features above.
|
||||
- `const nodeFsAdapter: FsAdapter`
|
||||
- `listRotatedFiles(fs: FsAdapter, basePath: string): Promise<string[]>`
|
||||
- `rotatedFilePathAt(basePath: string, isoTimestamp: string): string`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`ensureWriteableDir` calls `mkdir(..., { recursive: true })` then a
|
||||
`W_OK` `access` probe so a fresh boot creates the directory and a
|
||||
borked permission setup throws `PATH_NOT_WRITEABLE` synchronously
|
||||
during sink construction. `fileSize` swallows `ENOENT` and returns `0`
|
||||
so the rotation logic naturally treats a not-yet-created file as
|
||||
empty — that simplifies the open-on-first-write code path.
|
||||
@@ -0,0 +1,133 @@
|
||||
import { chmod, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import {
|
||||
listRotatedFiles,
|
||||
nodeFsAdapter,
|
||||
rotatedFilePathAt,
|
||||
} from "./fs-helpers.js";
|
||||
|
||||
async function freshDir(): Promise<string> {
|
||||
return mkdtemp(join(tmpdir(), "audit-fs-"));
|
||||
}
|
||||
|
||||
describe("api-audit-file fs-helpers", () => {
|
||||
it("ensureWriteableDir creates the dir then probes W_OK", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const sub = join(dir, "deep", "nested");
|
||||
await nodeFsAdapter.ensureWriteableDir(sub);
|
||||
await writeFile(join(sub, "ok.txt"), "yes", "utf8");
|
||||
});
|
||||
|
||||
it("ensureWriteableDir raises PATH_NOT_WRITEABLE on permission denied", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
await chmod(dir, 0o400);
|
||||
let caught: unknown;
|
||||
try {
|
||||
await nodeFsAdapter.ensureWriteableDir(join(dir, "child"));
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
} finally {
|
||||
await chmod(dir, 0o700);
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.PATH_NOT_WRITEABLE
|
||||
);
|
||||
});
|
||||
|
||||
it("fileSize returns the byte length of an existing file", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const path = join(dir, "x.log");
|
||||
await writeFile(path, "0123456789", "utf8");
|
||||
const size = await nodeFsAdapter.fileSize(path);
|
||||
expect(size).toBe(10);
|
||||
});
|
||||
|
||||
it("fileSize returns 0 for a non-existent path", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const size = await nodeFsAdapter.fileSize(join(dir, "absent.log"));
|
||||
expect(size).toBe(0);
|
||||
});
|
||||
|
||||
it("rename moves a file between paths", { timeout: 30_000 }, async () => {
|
||||
const dir = await freshDir();
|
||||
const a = join(dir, "a.log");
|
||||
const b = join(dir, "b.log");
|
||||
await writeFile(a, "hello", "utf8");
|
||||
await nodeFsAdapter.rename(a, b);
|
||||
expect(await nodeFsAdapter.fileSize(b)).toBe(5);
|
||||
expect(await nodeFsAdapter.fileSize(a)).toBe(0);
|
||||
});
|
||||
|
||||
it("unlink removes a file", { timeout: 30_000 }, async () => {
|
||||
const dir = await freshDir();
|
||||
const path = join(dir, "x.log");
|
||||
await writeFile(path, "x", "utf8");
|
||||
await nodeFsAdapter.unlink(path);
|
||||
expect(await nodeFsAdapter.fileSize(path)).toBe(0);
|
||||
});
|
||||
|
||||
it("readdir enumerates directory entries", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
await writeFile(join(dir, "a"), "a", "utf8");
|
||||
await writeFile(join(dir, "b"), "b", "utf8");
|
||||
const entries = await nodeFsAdapter.readdir(dir);
|
||||
expect(entries.sort()).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("listRotatedFiles returns [] when readdir throws", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const failingAdapter = {
|
||||
...nodeFsAdapter,
|
||||
async readdir() {
|
||||
throw new Error("EACCES");
|
||||
},
|
||||
};
|
||||
const out = await listRotatedFiles(failingAdapter, "/tmp/x/audit.log");
|
||||
expect(out).toEqual([]);
|
||||
});
|
||||
|
||||
it("listRotatedFiles ignores the active file and non-rotated siblings", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const basePath = join(dir, "audit.log");
|
||||
await writeFile(basePath, "active", "utf8");
|
||||
await writeFile(
|
||||
`${basePath}.2026-05-26T10-00-00.000Z.log`,
|
||||
"x",
|
||||
"utf8"
|
||||
);
|
||||
await writeFile(join(dir, "README.md"), "doc", "utf8");
|
||||
const out = await listRotatedFiles(nodeFsAdapter, basePath);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0]).toContain("audit.log.2026-05-26T10-00-00.000Z.log");
|
||||
});
|
||||
|
||||
it("rotatedFilePathAt replaces colons with hyphens", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const path = rotatedFilePathAt(
|
||||
"/var/log/audit.log",
|
||||
"2026-05-26T12:00:00.000Z"
|
||||
);
|
||||
expect(path).toBe("/var/log/audit.log.2026-05-26T12-00-00.000Z.log");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import { constants } from "node:fs";
|
||||
import {
|
||||
access,
|
||||
mkdir,
|
||||
open,
|
||||
readdir,
|
||||
rename,
|
||||
stat,
|
||||
unlink,
|
||||
} from "node:fs/promises";
|
||||
import { basename, dirname, join } from "node:path";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
|
||||
/**
|
||||
* Injectable adapter so the sink can run against an in-memory filesystem
|
||||
* in tests when desired. All production paths default to `node:fs`
|
||||
* primitives wired in {@link nodeFsAdapter}.
|
||||
*/
|
||||
export interface FsAdapter {
|
||||
appendLine(path: string, line: string, syncOnWrite: boolean): Promise<void>;
|
||||
fileSize(path: string): Promise<number>;
|
||||
rename(from: string, to: string): Promise<void>;
|
||||
unlink(path: string): Promise<void>;
|
||||
readdir(dir: string): Promise<string[]>;
|
||||
ensureWriteableDir(dir: string): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concrete `node:fs.promises` adapter. Production sinks construct this
|
||||
* once during boot; tests may swap in an in-memory shim that implements
|
||||
* the same surface.
|
||||
*/
|
||||
export const nodeFsAdapter: FsAdapter = {
|
||||
async appendLine(path, line, syncOnWrite) {
|
||||
const handle = await open(path, "a");
|
||||
try {
|
||||
await handle.appendFile(line, "utf8");
|
||||
if (syncOnWrite) await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
},
|
||||
async fileSize(path) {
|
||||
try {
|
||||
const s = await stat(path);
|
||||
return s.size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
},
|
||||
async rename(from, to) {
|
||||
await rename(from, to);
|
||||
},
|
||||
async unlink(path) {
|
||||
await unlink(path);
|
||||
},
|
||||
async readdir(dir) {
|
||||
return readdir(dir);
|
||||
},
|
||||
async ensureWriteableDir(dir) {
|
||||
try {
|
||||
await mkdir(dir, { recursive: true });
|
||||
await access(dir, constants.W_OK);
|
||||
} catch (cause) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.PATH_NOT_WRITEABLE,
|
||||
"error.audit.file.path_not_writeable",
|
||||
{ context: { dir }, cause }
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* List the rotated-file siblings of `basePath`, returning their full
|
||||
* paths sorted oldest-first (ascending by the embedded ISO timestamp).
|
||||
* Files NOT matching the rotation suffix `<base>.<utc-iso>.log` are
|
||||
* ignored so an operator's `.gitkeep` or unrelated content in the
|
||||
* audit directory does not throw the retention logic off.
|
||||
*/
|
||||
export async function listRotatedFiles(
|
||||
fs: FsAdapter,
|
||||
basePath: string
|
||||
): Promise<string[]> {
|
||||
const dir = dirname(basePath);
|
||||
const base = basename(basePath);
|
||||
const prefix = `${base}.`;
|
||||
const suffix = ".log";
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await fs.readdir(dir);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
const matches = entries.filter(
|
||||
(e) => e.startsWith(prefix) && e.endsWith(suffix) && e !== base
|
||||
);
|
||||
matches.sort();
|
||||
return matches.map((m) => join(dir, m));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the rotated-file name for `basePath` at the supplied
|
||||
* `Date`-rendered timestamp. The timestamp is the file's ROTATION
|
||||
* moment (i.e. the close time of the segment), not the open time.
|
||||
*/
|
||||
export function rotatedFilePathAt(
|
||||
basePath: string,
|
||||
isoTimestamp: string
|
||||
): string {
|
||||
return `${basePath}.${isoTimestamp.replace(/:/g, "-")}.log`;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
type FsAdapter,
|
||||
listRotatedFiles,
|
||||
nodeFsAdapter,
|
||||
rotatedFilePathAt,
|
||||
} from "./fs-helpers.js";
|
||||
76
packages/api-stack/api/api-audit-file/src/hmac/hmac.md
Normal file
76
packages/api-stack/api/api-audit-file/src/hmac/hmac.md
Normal file
@@ -0,0 +1,76 @@
|
||||
# `hmac`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Wraps `@suaveplan/crypto-hash` HMAC-SHA-256 into the two operations the
|
||||
file sink needs: signing a canonical record on write and verifying a
|
||||
record + signature on read. The wrapper hex-encodes the digest so the
|
||||
on-disk line stays human-greppable and ASCII-clean, and exposes a
|
||||
constant-time verifier so tampering checks do not leak position
|
||||
information through timing.
|
||||
|
||||
## Features
|
||||
|
||||
- `signRecord(key, bytes): string` — 64-char lowercase hex of
|
||||
HMAC-SHA-256 over `bytes` using `key`.
|
||||
- `verifyRecord(key, bytes, hmacHex): boolean` — constant-time
|
||||
comparison of recomputed HMAC against the provided hex.
|
||||
- Empty-key guard raises a typed `Error` so misconfiguration cannot
|
||||
silently produce a deterministic null-MAC.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { signRecord, verifyRecord } from "@suaveplan/api-audit-file/hmac";
|
||||
|
||||
const key = new TextEncoder().encode("super-secret-rotating-key");
|
||||
const bytes = new TextEncoder().encode('{"requestId":"r-1"}');
|
||||
const hmac = signRecord(key, bytes);
|
||||
verifyRecord(key, bytes, hmac); // → true
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Use with the canonical-bytes helper to verify an on-disk line:
|
||||
|
||||
```ts
|
||||
import {
|
||||
AuditRecordSchema,
|
||||
canonicalise,
|
||||
} from "@suaveplan/api-audit-file/schema";
|
||||
import { verifyRecord } from "@suaveplan/api-audit-file/hmac";
|
||||
|
||||
for await (const line of lines) {
|
||||
const { record, hmac } = JSON.parse(line) as {
|
||||
record: unknown;
|
||||
hmac: string;
|
||||
};
|
||||
const parsed = AuditRecordSchema.parse(record);
|
||||
const ok = verifyRecord(key, canonicalise(parsed), hmac);
|
||||
if (!ok) throw new Error("hmac mismatch");
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `signRecord(key: Uint8Array, bytes: Uint8Array): string`
|
||||
- `verifyRecord(key: Uint8Array, bytes: Uint8Array, hmacHex: string): boolean`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The hex encoder is hand-rolled rather than borrowed from
|
||||
`@suaveplan/crypto-utils` so this module stays a leaf of the dependency
|
||||
graph — the sink runs on every audit-emitting request, and keeping the
|
||||
hot path's transitive deps minimal helps cold-start latency on
|
||||
serverless deploys. Constant-time comparison XORs each character code
|
||||
and accumulates into a bitmask; the boolean conversion at the end keeps
|
||||
the early-return branch (length mismatch) bounded to attacker-visible
|
||||
metadata only. The 64-character length check up front is intentionally
|
||||
outside the constant-time region because the length of the supplied
|
||||
HMAC is not secret — an attacker who supplies a malformed hex string
|
||||
sees an immediate rejection but learns nothing about the recomputed
|
||||
digest.
|
||||
69
packages/api-stack/api/api-audit-file/src/hmac/hmac.test.ts
Normal file
69
packages/api-stack/api/api-audit-file/src/hmac/hmac.test.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { signRecord, verifyRecord } from "./hmac.js";
|
||||
|
||||
const KEY = new TextEncoder().encode("super-secret-rotating-key");
|
||||
const ALT_KEY = new TextEncoder().encode("another-key");
|
||||
|
||||
describe("api-audit-file hmac", () => {
|
||||
it("produces a 64-char lowercase hex digest", { timeout: 5000 }, () => {
|
||||
const bytes = new TextEncoder().encode('{"requestId":"r-1"}');
|
||||
const out = signRecord(KEY, bytes);
|
||||
expect(out).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
it("is deterministic for the same key + bytes", { timeout: 5000 }, () => {
|
||||
const bytes = new TextEncoder().encode("payload");
|
||||
expect(signRecord(KEY, bytes)).toBe(signRecord(KEY, bytes));
|
||||
});
|
||||
|
||||
it("differs when the key changes", { timeout: 5000 }, () => {
|
||||
const bytes = new TextEncoder().encode("payload");
|
||||
expect(signRecord(KEY, bytes)).not.toBe(signRecord(ALT_KEY, bytes));
|
||||
});
|
||||
|
||||
it("rejects an empty key with a descriptive throw", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() => signRecord(new Uint8Array(0), new Uint8Array(4))).toThrow(
|
||||
/key/
|
||||
);
|
||||
});
|
||||
|
||||
it("verifies a freshly-signed record", { timeout: 5000 }, () => {
|
||||
const bytes = new TextEncoder().encode("payload");
|
||||
const hex = signRecord(KEY, bytes);
|
||||
expect(verifyRecord(KEY, bytes, hex)).toBe(true);
|
||||
});
|
||||
|
||||
it("tampering invalidates HMAC", { timeout: 5000 }, () => {
|
||||
const bytes = new TextEncoder().encode('{"requestId":"r-1"}');
|
||||
const hex = signRecord(KEY, bytes);
|
||||
const tampered = new Uint8Array(bytes);
|
||||
tampered[1] = (tampered[1] as number) ^ 0x01;
|
||||
expect(verifyRecord(KEY, tampered, hex)).toBe(false);
|
||||
});
|
||||
|
||||
it("fails verification on a wrong-length hex", { timeout: 5000 }, () => {
|
||||
const bytes = new TextEncoder().encode("payload");
|
||||
expect(verifyRecord(KEY, bytes, "deadbeef")).toBe(false);
|
||||
});
|
||||
|
||||
it("fails verification on a same-length but mismatched hex", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const bytes = new TextEncoder().encode("payload");
|
||||
const wrong = "0".repeat(64);
|
||||
expect(verifyRecord(KEY, bytes, wrong)).toBe(false);
|
||||
});
|
||||
|
||||
it("zero-pads single-hex-digit bytes", { timeout: 5000 }, () => {
|
||||
// Trigger the `b < 16` branch in bytesToHex deterministically by
|
||||
// signing inputs that lead to at least one < 0x10 digest byte;
|
||||
// statistically inevitable but we verify the regex on output.
|
||||
for (let i = 0; i < 32; i += 1) {
|
||||
const bytes = new Uint8Array([i]);
|
||||
const hex = signRecord(KEY, bytes);
|
||||
expect(hex).toMatch(/^[0-9a-f]{64}$/);
|
||||
}
|
||||
});
|
||||
});
|
||||
45
packages/api-stack/api/api-audit-file/src/hmac/hmac.ts
Normal file
45
packages/api-stack/api/api-audit-file/src/hmac/hmac.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { hmac as cryptoHmac } from "@suaveplan/crypto-hash";
|
||||
|
||||
/**
|
||||
* HMAC-SHA-256 over the canonical record bytes. Returns lowercase hex
|
||||
* so the disk format is grep-able and platform-portable.
|
||||
*
|
||||
* Keys are typed `Uint8Array` so the caller has explicit control over
|
||||
* binary key material; pass `new TextEncoder().encode(secret)` if a
|
||||
* UTF-8 string is the source. The returned hex is 64 chars
|
||||
* (32 bytes × 2 hex chars).
|
||||
*/
|
||||
export function signRecord(key: Uint8Array, bytes: Uint8Array): string {
|
||||
if (key.byteLength === 0) {
|
||||
throw new Error("hmac key must not be empty");
|
||||
}
|
||||
const digest = cryptoHmac("SHA-256", key, bytes);
|
||||
return bytesToHex(digest);
|
||||
}
|
||||
|
||||
/**
|
||||
* Constant-time verification of an HMAC-hex against the canonical
|
||||
* bytes. Returns `true` on match, `false` on mismatch.
|
||||
*/
|
||||
export function verifyRecord(
|
||||
key: Uint8Array,
|
||||
bytes: Uint8Array,
|
||||
hmacHex: string
|
||||
): boolean {
|
||||
if (hmacHex.length !== 64) return false;
|
||||
const expected = signRecord(key, bytes);
|
||||
let diff = 0;
|
||||
for (let i = 0; i < expected.length; i += 1) {
|
||||
diff |= expected.charCodeAt(i) ^ hmacHex.charCodeAt(i);
|
||||
}
|
||||
return diff === 0;
|
||||
}
|
||||
|
||||
function bytesToHex(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (let i = 0; i < bytes.byteLength; i += 1) {
|
||||
const b = bytes[i] as number;
|
||||
out += (b < 16 ? "0" : "") + b.toString(16);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
1
packages/api-stack/api/api-audit-file/src/hmac/index.ts
Normal file
1
packages/api-stack/api/api-audit-file/src/hmac/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { signRecord, verifyRecord } from "./hmac.js";
|
||||
51
packages/api-stack/api/api-audit-file/src/index.ts
Normal file
51
packages/api-stack/api/api-audit-file/src/index.ts
Normal file
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* `@suaveplan/api-audit-file` — file-rotating, HMAC-signed audit sink.
|
||||
*
|
||||
* Implements the `RequestAuditSink` contract from
|
||||
* `@suaveplan/api-host/request-pipeline`. Writes JSON-per-line with an
|
||||
* HMAC-SHA-256 signature over the canonical record bytes; rolls the
|
||||
* file on size or age; prunes the oldest rotated siblings to keep at
|
||||
* most `maxFiles` retained.
|
||||
*/
|
||||
|
||||
import "./error-codes/error-codes.js";
|
||||
|
||||
export {
|
||||
AUDIT_FILE_CODES,
|
||||
AUDIT_FILE_ERROR_NAME,
|
||||
type AuditFileCode,
|
||||
AuditFileError,
|
||||
} from "./error-codes/index.js";
|
||||
export {
|
||||
type FsAdapter,
|
||||
listRotatedFiles,
|
||||
nodeFsAdapter,
|
||||
rotatedFilePathAt,
|
||||
} from "./fs-helpers/index.js";
|
||||
export { signRecord, verifyRecord } from "./hmac/index.js";
|
||||
export {
|
||||
type AuditFileInstruments,
|
||||
type AuditFileInstrumentsConfig,
|
||||
createAuditFileInstruments,
|
||||
SINK_ATTRIBUTE,
|
||||
} from "./instruments/index.js";
|
||||
export { enforceRetention } from "./retention/index.js";
|
||||
export {
|
||||
type RotationPolicy,
|
||||
type RotationState,
|
||||
resolveRotationPolicy,
|
||||
rotate,
|
||||
shouldRotate,
|
||||
} from "./rotation/index.js";
|
||||
export {
|
||||
type AuditRecord,
|
||||
AuditRecordSchema,
|
||||
buildSignedLine,
|
||||
canonicalise,
|
||||
} from "./schema/index.js";
|
||||
export {
|
||||
type AppendMode,
|
||||
type CreateFileAuditSinkOptions,
|
||||
createFileAuditSink,
|
||||
type FileAuditSink,
|
||||
} from "./sink/index.js";
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
type AuditFileInstruments,
|
||||
type AuditFileInstrumentsConfig,
|
||||
createAuditFileInstruments,
|
||||
SINK_ATTRIBUTE,
|
||||
} from "./instruments.js";
|
||||
@@ -0,0 +1,78 @@
|
||||
# `instruments`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Constructs the OTel counter bundle the file sink emits and exposes the
|
||||
canonical `sink="file"` attribute set. Every observation carries the
|
||||
attribute so a dashboard tracking `api.audit.records_total` across the
|
||||
sink family — `stdout`, `file`, and future siblings — can union the
|
||||
metric name and partition by `sink`. The three counters answer the
|
||||
operator questions "are records flowing?", "are records dropping?",
|
||||
and "is retention pruning old files?".
|
||||
|
||||
## Features
|
||||
|
||||
- `createAuditFileInstruments({ meterProvider? })` — returns a frozen
|
||||
bundle of three counters bound to the supplied or global
|
||||
`MeterProvider`.
|
||||
- `SINK_ATTRIBUTE` — frozen `{ sink: "file" }` literal exported for
|
||||
reuse by the sink, retention, and test harnesses.
|
||||
- Counters: `api.audit.records_total`, `api.audit.dropped_total`,
|
||||
`api.audit.rotation_evicted_total`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { createAuditFileInstruments } from "@suaveplan/api-audit-file/instruments";
|
||||
|
||||
const instruments = createAuditFileInstruments();
|
||||
instruments.recordsTotal.add(1, { sink: "file" });
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Install a real `MeterProvider` and assert deltas in tests:
|
||||
|
||||
```ts
|
||||
import {
|
||||
MeterProvider,
|
||||
InMemoryMetricExporter,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import { createAuditFileInstruments } from "@suaveplan/api-audit-file/instruments";
|
||||
|
||||
const exporter = new InMemoryMetricExporter(0);
|
||||
const reader = new PeriodicExportingMetricReader({ exporter });
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
|
||||
const instruments = createAuditFileInstruments({ meterProvider: provider });
|
||||
instruments.recordsTotal.add(1, { sink: "file" });
|
||||
|
||||
await reader.collect();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface AuditFileInstruments`
|
||||
- `meter: Meter`
|
||||
- `recordsTotal: Counter`
|
||||
- `droppedTotal: Counter`
|
||||
- `rotationEvictedTotal: Counter`
|
||||
- `interface AuditFileInstrumentsConfig`
|
||||
- `meterProvider?: MeterProvider`
|
||||
- `createAuditFileInstruments(config?): AuditFileInstruments`
|
||||
- `SINK_ATTRIBUTE: Readonly<{ sink: "file" }>`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The bundle is intentionally NOT module-level cached — every sink
|
||||
instance gets its own bundle so an operator who reconstructs the host
|
||||
under a fresh `MeterProvider` (a test-host pattern in
|
||||
`@suaveplan/api-host/testing`) sees deltas against the right provider.
|
||||
Counters from the sibling `api-audit-stdout` sink share the metric
|
||||
NAMES with this package so a single dashboard panel covers the family;
|
||||
the `sink="file"` attribute distinguishes the source.
|
||||
@@ -0,0 +1,75 @@
|
||||
import {
|
||||
type Counter,
|
||||
type Meter,
|
||||
type MeterProvider,
|
||||
metrics,
|
||||
} from "@opentelemetry/api";
|
||||
|
||||
const METER_NAME = "@suaveplan/api-audit-file";
|
||||
|
||||
/**
|
||||
* Bundle of OTel counters the sink emits. Each carries a stable
|
||||
* `sink="file"` attribute so dashboards can union the family of audit
|
||||
* sinks (`stdout`, `file`, future siblings) on the metric name and
|
||||
* partition by the attribute.
|
||||
*
|
||||
* - `recordsTotal` — every successful write increments by 1.
|
||||
* - `droppedTotal` — every queue-overflow or boot-time rejection
|
||||
* increments by 1. Operators MUST alert on a non-zero rate.
|
||||
* - `rotationEvictedTotal` — every retention-eviction increments by 1.
|
||||
*/
|
||||
export interface AuditFileInstruments {
|
||||
readonly meter: Meter;
|
||||
readonly recordsTotal: Counter;
|
||||
readonly droppedTotal: Counter;
|
||||
readonly rotationEvictedTotal: Counter;
|
||||
}
|
||||
|
||||
/** Construction options for {@link createAuditFileInstruments}. */
|
||||
export interface AuditFileInstrumentsConfig {
|
||||
readonly meterProvider?: MeterProvider;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the counter bundle against the supplied (or global) meter
|
||||
* provider. The default reads from `@opentelemetry/api`'s global
|
||||
* provider, which is the no-op tracer until the operator installs an
|
||||
* SDK. Tests supply a real `MeterProvider` so the per-instrument
|
||||
* deltas are observable.
|
||||
*/
|
||||
export function createAuditFileInstruments(
|
||||
config: AuditFileInstrumentsConfig = {}
|
||||
): AuditFileInstruments {
|
||||
const provider = config.meterProvider ?? metrics.getMeterProvider();
|
||||
const meter = provider.getMeter(METER_NAME);
|
||||
const recordsTotal = meter.createCounter("api.audit.records_total", {
|
||||
description:
|
||||
"Total audit records successfully written to the file sink",
|
||||
});
|
||||
const droppedTotal = meter.createCounter("api.audit.dropped_total", {
|
||||
description:
|
||||
"Total audit records dropped before reaching disk (queue overflow or validation rejection)",
|
||||
});
|
||||
const rotationEvictedTotal = meter.createCounter(
|
||||
"api.audit.rotation_evicted_total",
|
||||
{
|
||||
description:
|
||||
"Total rotated audit files deleted by retention policy (maxFiles enforcement)",
|
||||
}
|
||||
);
|
||||
return Object.freeze({
|
||||
meter,
|
||||
recordsTotal,
|
||||
droppedTotal,
|
||||
rotationEvictedTotal,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable `{ sink: "file" }` attribute set every instrument observation
|
||||
* carries. Exported so verification harnesses and tests can compare
|
||||
* the canonical literal instead of duplicating the string.
|
||||
*/
|
||||
export const SINK_ATTRIBUTE: Readonly<{ sink: "file" }> = Object.freeze({
|
||||
sink: "file" as const,
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
export { enforceRetention } from "./retention.js";
|
||||
@@ -0,0 +1,67 @@
|
||||
# `retention`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Enforces the `maxFiles` retention dimension of the rotation policy by
|
||||
deleting the oldest rotated files until at most `maxFiles` remain.
|
||||
Retention runs immediately after each rotation succeeds so the disk
|
||||
footprint is bounded across long-running deployments — an audit
|
||||
directory left to grow unchecked is the most common operator pain
|
||||
point with file-based sinks, and this module's existence is exactly
|
||||
the answer to it.
|
||||
|
||||
The increment on `api.audit.rotation_evicted_total{sink="file"}` makes
|
||||
eviction observable. Operators alert on an unexpectedly high rate so a
|
||||
runaway rotation cycle (e.g. clock skew, misconfigured `maxAgeMs`) is
|
||||
visible before the retention sweep drops historical data.
|
||||
|
||||
## Features
|
||||
|
||||
- `enforceRetention(fs, basePath, maxFiles, instruments)` — deletes
|
||||
the oldest rotated files until at most `maxFiles` remain. Idempotent:
|
||||
re-running with no surplus is a no-op.
|
||||
- Counter increment per deletion with the canonical `sink="file"`
|
||||
attribute set.
|
||||
- Surfaces `AUDIT_FILE.RETENTION_FAILED` on a delete failure with the
|
||||
offending path in `context.path` and the underlying `cause`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { enforceRetention } from "@suaveplan/api-audit-file/retention";
|
||||
|
||||
await enforceRetention(fs, "/var/log/audit.log", 30, instruments);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Call from a custom rotation cadence — e.g. an external cron that wants
|
||||
to prune without rotating:
|
||||
|
||||
```ts
|
||||
import { enforceRetention } from "@suaveplan/api-audit-file/retention";
|
||||
|
||||
// Cron job: every hour, prune to a smaller retention than the sink's
|
||||
// default to claw back disk during high-traffic windows.
|
||||
setInterval(async () => {
|
||||
await enforceRetention(fs, basePath, 10, instruments);
|
||||
}, 60 * 60 * 1000);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `enforceRetention(fs: FsAdapter, basePath: string, maxFiles: number, instruments: AuditFileInstruments): Promise<void>`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`listRotatedFiles` returns paths sorted oldest-first by the embedded
|
||||
ISO timestamp; we slice off the first `(count - maxFiles)` entries and
|
||||
unlink them. Sorting is lexicographic on the ISO string, which is also
|
||||
chronological for the `YYYY-MM-DDTHH-MM-SS.sssZ` shape — no extra
|
||||
parsing required. If `unlink` fails midway, the previously-evicted
|
||||
deletions are not undone; the throw surfaces the failure and the next
|
||||
rotation re-attempts retention against whatever survived.
|
||||
@@ -0,0 +1,143 @@
|
||||
import { mkdtemp, readdir, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
InMemoryMetricExporter,
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { nodeFsAdapter } from "../fs-helpers/fs-helpers.js";
|
||||
import {
|
||||
createAuditFileInstruments,
|
||||
SINK_ATTRIBUTE,
|
||||
} from "../instruments/instruments.js";
|
||||
import { enforceRetention } from "./retention.js";
|
||||
|
||||
interface Harness {
|
||||
dir: string;
|
||||
basePath: string;
|
||||
reader: PeriodicExportingMetricReader;
|
||||
instruments: ReturnType<typeof createAuditFileInstruments>;
|
||||
}
|
||||
|
||||
async function setup(): Promise<Harness> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "audit-retention-"));
|
||||
const basePath = join(dir, "audit.log");
|
||||
const exporter = new InMemoryMetricExporter(0);
|
||||
const reader = new PeriodicExportingMetricReader({ exporter });
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
const instruments = createAuditFileInstruments({ meterProvider: provider });
|
||||
return { dir, basePath, reader, instruments };
|
||||
}
|
||||
|
||||
async function makeRotated(basePath: string, isoTimestamps: string[]) {
|
||||
for (const ts of isoTimestamps) {
|
||||
await writeFile(`${basePath}.${ts}.log`, "stub", "utf8");
|
||||
}
|
||||
}
|
||||
|
||||
async function collectMetrics(reader: PeriodicExportingMetricReader) {
|
||||
const { resourceMetrics } = await reader.collect();
|
||||
return resourceMetrics.scopeMetrics.flatMap((sm) => sm.metrics);
|
||||
}
|
||||
|
||||
describe("api-audit-file retention", () => {
|
||||
let h: Harness;
|
||||
|
||||
beforeEach(async () => {
|
||||
h = await setup();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await h.reader.shutdown();
|
||||
});
|
||||
|
||||
it("evicts oldest beyond maxFiles and increments counter", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const timestamps = [
|
||||
"2026-05-26T10-00-00.000Z",
|
||||
"2026-05-26T11-00-00.000Z",
|
||||
"2026-05-26T12-00-00.000Z",
|
||||
"2026-05-26T13-00-00.000Z",
|
||||
];
|
||||
await makeRotated(h.basePath, timestamps);
|
||||
await enforceRetention(nodeFsAdapter, h.basePath, 3, h.instruments);
|
||||
const survivors = (await readdir(h.dir)).filter((f) =>
|
||||
f.endsWith(".log")
|
||||
);
|
||||
expect(survivors).toHaveLength(3);
|
||||
expect(survivors).not.toContain(`audit.log.${timestamps[0]}.log`);
|
||||
const metrics = await collectMetrics(h.reader);
|
||||
const counter = metrics.find(
|
||||
(m) => m.descriptor.name === "api.audit.rotation_evicted_total"
|
||||
);
|
||||
expect(counter).toBeDefined();
|
||||
expect(counter?.dataPoints[0]?.value).toBe(1);
|
||||
expect(counter?.dataPoints[0]?.attributes).toEqual(SINK_ATTRIBUTE);
|
||||
});
|
||||
|
||||
it("is a no-op when below maxFiles", { timeout: 30_000 }, async () => {
|
||||
const timestamps = [
|
||||
"2026-05-26T10-00-00.000Z",
|
||||
"2026-05-26T11-00-00.000Z",
|
||||
];
|
||||
await makeRotated(h.basePath, timestamps);
|
||||
await enforceRetention(nodeFsAdapter, h.basePath, 5, h.instruments);
|
||||
const survivors = (await readdir(h.dir)).filter((f) =>
|
||||
f.endsWith(".log")
|
||||
);
|
||||
expect(survivors).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("surfaces RETENTION_FAILED on unlink failure", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const adapter = {
|
||||
...nodeFsAdapter,
|
||||
async unlink(_p: string) {
|
||||
throw new Error("EACCES");
|
||||
},
|
||||
};
|
||||
await makeRotated(h.basePath, [
|
||||
"2026-05-26T10-00-00.000Z",
|
||||
"2026-05-26T11-00-00.000Z",
|
||||
]);
|
||||
let caught: unknown;
|
||||
try {
|
||||
await enforceRetention(adapter, h.basePath, 1, h.instruments);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.RETENTION_FAILED
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores non-rotated entries in the directory", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
await writeFile(h.basePath, "active", "utf8");
|
||||
await writeFile(join(h.dir, "README.md"), "doc", "utf8");
|
||||
await makeRotated(h.basePath, [
|
||||
"2026-05-26T10-00-00.000Z",
|
||||
"2026-05-26T11-00-00.000Z",
|
||||
]);
|
||||
await enforceRetention(nodeFsAdapter, h.basePath, 1, h.instruments);
|
||||
const entries = await readdir(h.dir);
|
||||
expect(entries).toContain("audit.log");
|
||||
expect(entries).toContain("README.md");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { type FsAdapter, listRotatedFiles } from "../fs-helpers/fs-helpers.js";
|
||||
import type { AuditFileInstruments } from "../instruments/instruments.js";
|
||||
import { SINK_ATTRIBUTE } from "../instruments/instruments.js";
|
||||
|
||||
/**
|
||||
* Inspect the rotated-file siblings of `basePath` and delete the
|
||||
* oldest ones until at most `maxFiles` remain. Increments
|
||||
* `api.audit.rotation_evicted_total{sink="file"}` once per deletion.
|
||||
*
|
||||
* Failures during a delete surface as `AUDIT_FILE.RETENTION_FAILED`
|
||||
* with the offending path in the cause context; the caller's logger
|
||||
* records the failure and the next rotation re-attempts retention so
|
||||
* one transient EBUSY does not permanently disable enforcement.
|
||||
*/
|
||||
export async function enforceRetention(
|
||||
fs: FsAdapter,
|
||||
basePath: string,
|
||||
maxFiles: number,
|
||||
instruments: AuditFileInstruments
|
||||
): Promise<void> {
|
||||
const rotated = await listRotatedFiles(fs, basePath);
|
||||
if (rotated.length <= maxFiles) return;
|
||||
const evictCount = rotated.length - maxFiles;
|
||||
const victims = rotated.slice(0, evictCount);
|
||||
for (const path of victims) {
|
||||
try {
|
||||
await fs.unlink(path);
|
||||
instruments.rotationEvictedTotal.add(1, SINK_ATTRIBUTE);
|
||||
} catch (cause) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.RETENTION_FAILED,
|
||||
"error.audit.file.retention_failed",
|
||||
{ context: { path }, cause }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
type RotationPolicy,
|
||||
type RotationState,
|
||||
resolveRotationPolicy,
|
||||
rotate,
|
||||
shouldRotate,
|
||||
} from "./rotation.js";
|
||||
@@ -0,0 +1,95 @@
|
||||
# `rotation`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Resolves the rotation policy carried by every `createFileAuditSink`
|
||||
instance and implements the rotation primitive itself — the
|
||||
`rename(currentFile → currentFile.<utc-iso>.log)` step plus the
|
||||
post-rotation state reset. The module owns both dimensions documented
|
||||
in the spec: size-based (`maxSizeBytes`, default 100 MiB) and
|
||||
time-based (`maxAgeMs`, default 24h). Either condition reaching its
|
||||
limit triggers a rotation on the next write.
|
||||
|
||||
Rotation lives in a separate module so the sink stays focused on
|
||||
queueing + writing. The retention module reads the policy through the
|
||||
same `RotationState` so the operator-facing knobs are documented in
|
||||
exactly one place.
|
||||
|
||||
## Features
|
||||
|
||||
- `resolveRotationPolicy(input?)` — Zod-equivalent validator returning
|
||||
a frozen `RotationPolicy`. Defaults filled from the design doc.
|
||||
- `shouldRotate(state, nowMs)` — pure predicate; size OR age satisfies.
|
||||
- `rotate(fs, basePath, state, clock)` — performs the rename, updates
|
||||
state, returns the rotated path for retention to inspect.
|
||||
- Raises `AUDIT_FILE.ROTATION_FAILED` on rename failure with full
|
||||
`cause` for forensic logging.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
resolveRotationPolicy,
|
||||
rotate,
|
||||
shouldRotate,
|
||||
} from "@suaveplan/api-audit-file/rotation";
|
||||
|
||||
const policy = resolveRotationPolicy({ maxSizeBytes: 1024 * 1024 });
|
||||
const state = { openedAtMs: Date.now(), bytesWritten: 0, policy };
|
||||
if (shouldRotate(state, Date.now())) {
|
||||
await rotate(fs, "/var/log/audit.log", state, () => Date.now());
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Disable a dimension by setting it to `Number.POSITIVE_INFINITY` —
|
||||
useful when an operator wants pure time rotation regardless of size:
|
||||
|
||||
```ts
|
||||
const policy = resolveRotationPolicy({
|
||||
maxSizeBytes: Number.POSITIVE_INFINITY,
|
||||
maxAgeMs: 60 * 60 * 1000, // hourly
|
||||
});
|
||||
```
|
||||
|
||||
In tests, drive rotation deterministically with a `FakeClock` from
|
||||
`@suaveplan/testing`:
|
||||
|
||||
```ts
|
||||
import { FakeClock } from "@suaveplan/testing";
|
||||
|
||||
const clock = new FakeClock(0);
|
||||
const state = { openedAtMs: 0, bytesWritten: 0, policy };
|
||||
clock.advance(61_000);
|
||||
await rotate(fs, basePath, state, clock.read);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface RotationPolicy`
|
||||
- `maxSizeBytes: number` — bytes before size rotation; default 100 MiB.
|
||||
- `maxAgeMs: number` — ms before time rotation; default 24h.
|
||||
- `maxFiles: number` — kept rotated files; default 30.
|
||||
- `interface RotationState`
|
||||
- `openedAtMs: number` — wall-clock open time of current segment.
|
||||
- `bytesWritten: number` — running byte count.
|
||||
- `policy: RotationPolicy` — frozen at construction.
|
||||
- `resolveRotationPolicy(input?): RotationPolicy`
|
||||
- `shouldRotate(state, nowMs): boolean`
|
||||
- `rotate(fs, basePath, state, clock): Promise<string>` — returns the
|
||||
rotated file path.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The rotated suffix replaces `:` with `-` so the resulting filename is
|
||||
Windows-portable and grep-friendly across shells. State mutation
|
||||
happens AFTER the rename succeeds — a failed rename leaves the original
|
||||
state intact, so the next call retries against the same segment rather
|
||||
than corrupting accounting. `bytesWritten` resets to 0 instead of
|
||||
re-stat'ing the new empty file; the next successful append updates it
|
||||
in lock-step with the on-disk length.
|
||||
@@ -0,0 +1,170 @@
|
||||
import { mkdtemp, readdir, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { nodeFsAdapter } from "../fs-helpers/fs-helpers.js";
|
||||
import {
|
||||
type RotationState,
|
||||
resolveRotationPolicy,
|
||||
rotate,
|
||||
shouldRotate,
|
||||
} from "./rotation.js";
|
||||
|
||||
let dir: string;
|
||||
let basePath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), "audit-rot-"));
|
||||
basePath = join(dir, "audit.log");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// tmp dirs cleaned by os.tmpdir lifecycle
|
||||
});
|
||||
|
||||
describe("api-audit-file rotation — policy resolver", () => {
|
||||
it("applies the documented defaults", { timeout: 5000 }, () => {
|
||||
const p = resolveRotationPolicy();
|
||||
expect(p.maxSizeBytes).toBe(100 * 1024 * 1024);
|
||||
expect(p.maxAgeMs).toBe(24 * 60 * 60 * 1000);
|
||||
expect(p.maxFiles).toBe(30);
|
||||
});
|
||||
|
||||
it("merges partial input over defaults", { timeout: 5000 }, () => {
|
||||
const p = resolveRotationPolicy({ maxSizeBytes: 1024 });
|
||||
expect(p.maxSizeBytes).toBe(1024);
|
||||
expect(p.maxAgeMs).toBe(24 * 60 * 60 * 1000);
|
||||
});
|
||||
|
||||
it("rejects maxSizeBytes < 1", { timeout: 5000 }, () => {
|
||||
expect(() => resolveRotationPolicy({ maxSizeBytes: 0 })).toThrow(
|
||||
AuditFileError
|
||||
);
|
||||
try {
|
||||
resolveRotationPolicy({ maxSizeBytes: 0 });
|
||||
} catch (err) {
|
||||
expect((err as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects maxAgeMs < 1", { timeout: 5000 }, () => {
|
||||
expect(() => resolveRotationPolicy({ maxAgeMs: 0 })).toThrow();
|
||||
});
|
||||
|
||||
it("rejects maxFiles < 1", { timeout: 5000 }, () => {
|
||||
expect(() => resolveRotationPolicy({ maxFiles: 0 })).toThrow();
|
||||
});
|
||||
|
||||
it("rejects non-integer maxFiles", { timeout: 5000 }, () => {
|
||||
expect(() => resolveRotationPolicy({ maxFiles: 2.5 })).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("api-audit-file rotation — shouldRotate", () => {
|
||||
const policy = resolveRotationPolicy({
|
||||
maxSizeBytes: 1024,
|
||||
maxAgeMs: 60_000,
|
||||
});
|
||||
|
||||
it("triggers on size overflow", { timeout: 5000 }, () => {
|
||||
const state: RotationState = {
|
||||
openedAtMs: 0,
|
||||
bytesWritten: 1024,
|
||||
policy,
|
||||
};
|
||||
expect(shouldRotate(state, 0)).toBe(true);
|
||||
});
|
||||
|
||||
it("triggers on age overflow", { timeout: 5000 }, () => {
|
||||
const state: RotationState = {
|
||||
openedAtMs: 0,
|
||||
bytesWritten: 0,
|
||||
policy,
|
||||
};
|
||||
expect(shouldRotate(state, 60_000)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not trigger when under both limits", { timeout: 5000 }, () => {
|
||||
const state: RotationState = {
|
||||
openedAtMs: 0,
|
||||
bytesWritten: 100,
|
||||
policy,
|
||||
};
|
||||
expect(shouldRotate(state, 10_000)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("api-audit-file rotation — rotate()", () => {
|
||||
it("size rotation produces timestamp-suffixed file", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
await writeFile(basePath, "x".repeat(2048), "utf8");
|
||||
const policy = resolveRotationPolicy({ maxSizeBytes: 1024 });
|
||||
const state: RotationState = {
|
||||
openedAtMs: 0,
|
||||
bytesWritten: 2048,
|
||||
policy,
|
||||
};
|
||||
const rotatedPath = await rotate(nodeFsAdapter, basePath, state, () =>
|
||||
Date.parse("2026-05-26T12:00:00.000Z")
|
||||
);
|
||||
expect(rotatedPath).toContain("audit.log.2026-05-26T12-00-00.000Z.log");
|
||||
const entries = await readdir(dir);
|
||||
expect(entries).toContain("audit.log.2026-05-26T12-00-00.000Z.log");
|
||||
expect(state.bytesWritten).toBe(0);
|
||||
});
|
||||
|
||||
it("time rotation triggers on clock advance", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
await writeFile(basePath, "data", "utf8");
|
||||
const policy = resolveRotationPolicy({ maxAgeMs: 60_000 });
|
||||
const state: RotationState = {
|
||||
openedAtMs: 0,
|
||||
bytesWritten: 4,
|
||||
policy,
|
||||
};
|
||||
expect(shouldRotate(state, 61_000)).toBe(true);
|
||||
await rotate(nodeFsAdapter, basePath, state, () => 61_000);
|
||||
expect(state.openedAtMs).toBe(61_000);
|
||||
});
|
||||
|
||||
it("surfaces ROTATION_FAILED on rename failure", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const policy = resolveRotationPolicy();
|
||||
const state: RotationState = {
|
||||
openedAtMs: 0,
|
||||
bytesWritten: 0,
|
||||
policy,
|
||||
};
|
||||
const failingAdapter = {
|
||||
...nodeFsAdapter,
|
||||
async rename() {
|
||||
throw new Error("EBUSY");
|
||||
},
|
||||
};
|
||||
let caught: unknown;
|
||||
try {
|
||||
await rotate(failingAdapter, basePath, state, () => 0);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED
|
||||
);
|
||||
});
|
||||
});
|
||||
106
packages/api-stack/api/api-audit-file/src/rotation/rotation.ts
Normal file
106
packages/api-stack/api/api-audit-file/src/rotation/rotation.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import type { Clock } from "@suaveplan/types/clock";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { type FsAdapter, rotatedFilePathAt } from "../fs-helpers/fs-helpers.js";
|
||||
|
||||
/**
|
||||
* Rotation policy resolved + validated at sink construction. Carries
|
||||
* the defaults documented in the design doc: 100 MiB size cap, 24-hour
|
||||
* age cap. The retention `maxFiles` lives here for ergonomics — the
|
||||
* retention module reads it through the same {@link RotationState}.
|
||||
*/
|
||||
export interface RotationPolicy {
|
||||
readonly maxSizeBytes: number;
|
||||
readonly maxAgeMs: number;
|
||||
readonly maxFiles: number;
|
||||
}
|
||||
|
||||
/** Mutable rotation state carried per sink. */
|
||||
export interface RotationState {
|
||||
/** Wall-clock ms when the current segment was opened. */
|
||||
openedAtMs: number;
|
||||
/** Last observed byte size of the current segment. */
|
||||
bytesWritten: number;
|
||||
/** Resolved policy, frozen. */
|
||||
readonly policy: RotationPolicy;
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_SIZE_BYTES = 100 * 1024 * 1024;
|
||||
const DEFAULT_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
const DEFAULT_MAX_FILES = 30;
|
||||
|
||||
/**
|
||||
* Resolve a partial policy against defaults, raising on invalid input.
|
||||
* `maxSizeBytes >= 1`, `maxAgeMs >= 1`, `maxFiles >= 1`. Callers may
|
||||
* disable a dimension by setting it to `Number.POSITIVE_INFINITY`.
|
||||
*/
|
||||
export function resolveRotationPolicy(
|
||||
input?: Partial<RotationPolicy>
|
||||
): RotationPolicy {
|
||||
const maxSizeBytes = input?.maxSizeBytes ?? DEFAULT_MAX_SIZE_BYTES;
|
||||
const maxAgeMs = input?.maxAgeMs ?? DEFAULT_MAX_AGE_MS;
|
||||
const maxFiles = input?.maxFiles ?? DEFAULT_MAX_FILES;
|
||||
if (!(maxSizeBytes >= 1)) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED,
|
||||
"error.audit.file.rotation.invalid_policy",
|
||||
{ context: { field: "maxSizeBytes", value: maxSizeBytes } }
|
||||
);
|
||||
}
|
||||
if (!(maxAgeMs >= 1)) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED,
|
||||
"error.audit.file.rotation.invalid_policy",
|
||||
{ context: { field: "maxAgeMs", value: maxAgeMs } }
|
||||
);
|
||||
}
|
||||
if (!Number.isInteger(maxFiles) || maxFiles < 1) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED,
|
||||
"error.audit.file.rotation.invalid_policy",
|
||||
{ context: { field: "maxFiles", value: maxFiles } }
|
||||
);
|
||||
}
|
||||
return Object.freeze({ maxSizeBytes, maxAgeMs, maxFiles });
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether the current segment needs to rotate BEFORE the next
|
||||
* line is appended. Returns `true` when size OR age has been reached.
|
||||
*/
|
||||
export function shouldRotate(state: RotationState, nowMs: number): boolean {
|
||||
if (state.bytesWritten >= state.policy.maxSizeBytes) return true;
|
||||
if (nowMs - state.openedAtMs >= state.policy.maxAgeMs) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rotate the current segment by renaming it to a timestamp-suffixed
|
||||
* sibling. Returns the rotated path so the caller can hand it to the
|
||||
* retention module. On rename failure, raises
|
||||
* `AUDIT_FILE.ROTATION_FAILED`; the caller's outer loop catches and
|
||||
* surfaces via the configured logger so the next write retries cleanly.
|
||||
*/
|
||||
export async function rotate(
|
||||
fs: FsAdapter,
|
||||
basePath: string,
|
||||
state: RotationState,
|
||||
clock: Clock
|
||||
): Promise<string> {
|
||||
const rotatedAt = new Date(clock()).toISOString();
|
||||
const rotatedPath = rotatedFilePathAt(basePath, rotatedAt);
|
||||
try {
|
||||
await fs.rename(basePath, rotatedPath);
|
||||
} catch (cause) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED,
|
||||
"error.audit.file.rotation_failed",
|
||||
{ context: { basePath, rotatedPath }, cause }
|
||||
);
|
||||
}
|
||||
state.openedAtMs = clock();
|
||||
state.bytesWritten = 0;
|
||||
return rotatedPath;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
type AuditRecord,
|
||||
AuditRecordSchema,
|
||||
buildSignedLine,
|
||||
canonicalise,
|
||||
} from "./schema.js";
|
||||
79
packages/api-stack/api/api-audit-file/src/schema/schema.md
Normal file
79
packages/api-stack/api/api-audit-file/src/schema/schema.md
Normal file
@@ -0,0 +1,79 @@
|
||||
# `schema`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the Zod schema and canonical-bytes function that bracket every
|
||||
audit record before it is signed and written. The schema mirrors the
|
||||
`RequestAuditEnvelope` interface exported by
|
||||
`@suaveplan/api-host/request-pipeline`; the canonical-bytes function
|
||||
gives every other module a deterministic input for HMAC signing,
|
||||
file-line construction, and signature verification.
|
||||
|
||||
The package re-validates at the sink boundary even though the host
|
||||
constructs envelopes from typed data — operators frequently hand-craft
|
||||
audit records in tests, replay tooling, or migration scripts. The
|
||||
boundary check makes corrupt records fail loudly with a typed
|
||||
`AUDIT_FILE.INVALID_RECORD` instead of silently bypassing the HMAC.
|
||||
|
||||
## Features
|
||||
|
||||
- `AuditRecordSchema` — strict Zod object accepting only the
|
||||
envelope's documented fields. Unknown keys reject.
|
||||
- `AuditRecord` — `z.infer<typeof AuditRecordSchema>`; structurally
|
||||
assignable to `RequestAuditEnvelope`.
|
||||
- `canonicalise(record)` — UTF-8 bytes of the record JSON-stringified
|
||||
with sorted top-level keys.
|
||||
- `buildSignedLine(bytes, hmacHex)` — emits the on-disk JSON line
|
||||
carrying the record + HMAC, with a trailing `\n`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
AuditRecordSchema,
|
||||
buildSignedLine,
|
||||
canonicalise,
|
||||
} from "@suaveplan/api-audit-file/schema";
|
||||
|
||||
const record = AuditRecordSchema.parse(rawEnvelope);
|
||||
const bytes = canonicalise(record);
|
||||
const line = buildSignedLine(bytes, "deadbeef...");
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Verify an on-disk line:
|
||||
|
||||
```ts
|
||||
import { hmac } from "@suaveplan/crypto-hash";
|
||||
import { AuditRecordSchema, canonicalise } from "@suaveplan/api-audit-file/schema";
|
||||
|
||||
const { record, hmac: hmacHex } = JSON.parse(lineFromDisk);
|
||||
const parsed = AuditRecordSchema.parse(record);
|
||||
const bytes = canonicalise(parsed);
|
||||
const expected = Buffer.from(hmac("SHA-256", key, bytes)).toString("hex");
|
||||
if (expected !== hmacHex) {
|
||||
throw new Error("hmac mismatch — record was tampered");
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `AuditRecordSchema: z.ZodObject` (strict)
|
||||
- `type AuditRecord = z.infer<typeof AuditRecordSchema>`
|
||||
- `canonicalise(record: AuditRecord): Uint8Array`
|
||||
- `buildSignedLine(canonicalBytes: Uint8Array, hmacHex: string): string`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The canonical bytes drop `undefined` values for the optional `cause`
|
||||
slot so the absence of an error never affects the signature. Top-level
|
||||
keys are sorted alphabetically to give every implementation (TypeScript,
|
||||
Go log-reader, Python forensic tooling) the same input bytes for
|
||||
verification regardless of construction order. `principal.scopes` is
|
||||
preserved in array order — the order is semantically meaningful in the
|
||||
pipeline so we do not collapse it.
|
||||
70
packages/api-stack/api/api-audit-file/src/schema/schema.ts
Normal file
70
packages/api-stack/api/api-audit-file/src/schema/schema.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import z from "zod";
|
||||
|
||||
/**
|
||||
* Zod schema for the `RequestAuditEnvelope` shape produced by
|
||||
* `@suaveplan/api-host/request-pipeline`. We re-validate at the sink
|
||||
* boundary because callers (host code, plugin code, tests) may hand us
|
||||
* an object that was reshaped after the pipeline emitted it. Mirrors
|
||||
* the structural type exported from `api-host`; kept in lock-step at
|
||||
* compile-time via the `AuditRecord` re-export below.
|
||||
*/
|
||||
export const AuditRecordSchema = z
|
||||
.object({
|
||||
at: z.string().min(1),
|
||||
requestId: z.string().min(1),
|
||||
traceId: z.string().nullable(),
|
||||
route: z.string().min(1),
|
||||
plugin: z.string().min(1),
|
||||
method: z.string().min(1),
|
||||
principal: z
|
||||
.object({
|
||||
id: z.string().min(1),
|
||||
scopes: z.array(z.string()).readonly(),
|
||||
})
|
||||
.nullable(),
|
||||
tenantId: z.string().nullable(),
|
||||
status: z.number().int().min(0),
|
||||
outcome: z.union([z.literal("success"), z.literal("failure")]),
|
||||
cause: z
|
||||
.object({
|
||||
code: z.string().min(1),
|
||||
message: z.string(),
|
||||
})
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
/** Structural type re-exported for downstream typing. */
|
||||
export type AuditRecord = z.infer<typeof AuditRecordSchema>;
|
||||
|
||||
/**
|
||||
* Canonicalise a record into the byte stream that gets HMAC-signed and
|
||||
* persisted. The canonical form is JSON with sorted top-level keys and
|
||||
* no trailing whitespace; sorting the keys means the same logical
|
||||
* record always serialises to the same bytes regardless of how the
|
||||
* upstream pipeline constructed it (object spread, property order, ...).
|
||||
*
|
||||
* Returns the UTF-8 bytes ready for HMAC input. The caller appends an
|
||||
* HMAC suffix when writing to disk; see {@link buildSignedLine}.
|
||||
*/
|
||||
export function canonicalise(record: AuditRecord): Uint8Array {
|
||||
const ordered: Record<string, unknown> = {};
|
||||
for (const key of (Object.keys(record) as (keyof AuditRecord)[]).sort()) {
|
||||
ordered[key] = record[key];
|
||||
}
|
||||
return new TextEncoder().encode(JSON.stringify(ordered));
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the on-disk line for a record + HMAC. The signed shape is a
|
||||
* JSON object with two top-level keys, `record` (the canonical payload
|
||||
* unwrapped) and `hmac` (lowercase hex of HMAC-SHA-256). One trailing
|
||||
* `\n` so consumers can read the file as JSON-per-line.
|
||||
*/
|
||||
export function buildSignedLine(
|
||||
canonicalBytes: Uint8Array,
|
||||
hmacHex: string
|
||||
): string {
|
||||
const recordObject = JSON.parse(new TextDecoder().decode(canonicalBytes));
|
||||
return `${JSON.stringify({ record: recordObject, hmac: hmacHex })}\n`;
|
||||
}
|
||||
6
packages/api-stack/api/api-audit-file/src/sink/index.ts
Normal file
6
packages/api-stack/api/api-audit-file/src/sink/index.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
export { createFileAuditSink } from "./sink.js";
|
||||
export type {
|
||||
AppendMode,
|
||||
CreateFileAuditSinkOptions,
|
||||
FileAuditSink,
|
||||
} from "./sink.types.js";
|
||||
128
packages/api-stack/api/api-audit-file/src/sink/sink.md
Normal file
128
packages/api-stack/api/api-audit-file/src/sink/sink.md
Normal file
@@ -0,0 +1,128 @@
|
||||
# `sink`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
The public entry point of `@suaveplan/api-audit-file`. The
|
||||
`createFileAuditSink` factory composes the schema, hmac, rotation,
|
||||
retention, instruments, and fs-helpers modules into a single object
|
||||
that implements the `RequestAuditSink` contract from
|
||||
`@suaveplan/api-host/request-pipeline`. The sink writes each accepted
|
||||
record as one JSON line with an HMAC-SHA-256 signature, rolls the file
|
||||
when size OR age limits are reached, and keeps at most `maxFiles`
|
||||
rotated siblings.
|
||||
|
||||
The sink is queue-buffered: callers `await emit()` and resolve as soon
|
||||
as the record has been appended to disk. A bounded queue protects the
|
||||
host from a stalled filesystem: records arriving while the queue is at
|
||||
`maxQueueSize` are dropped with a typed `AUDIT_FILE.QUEUE_FULL` and an
|
||||
`api.audit.dropped_total{sink="file"}` increment. `dispose()` flushes
|
||||
the queue and is idempotent.
|
||||
|
||||
## Features
|
||||
|
||||
- `createFileAuditSink({ path, hmacKey, ... })` — async factory.
|
||||
Returns a sink ready to `emit()`.
|
||||
- Schema validation at the boundary: invalid records throw
|
||||
`AUDIT_FILE.INVALID_RECORD` and increment `dropped_total`.
|
||||
- `strict-sync` (default) and `append` modes for the durability /
|
||||
throughput trade-off.
|
||||
- Boot-time `mkdir -p` of the parent directory + `W_OK` probe;
|
||||
unwriteable targets throw `AUDIT_FILE.PATH_NOT_WRITEABLE` from the
|
||||
factory itself so misconfiguration is visible at host startup.
|
||||
- Size + time rotation; `maxFiles` retention; per-eviction OTel counter.
|
||||
- Idempotent `dispose()`; observable `queueDepth` getter for probes.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { createFileAuditSink } from "@suaveplan/api-audit-file/sink";
|
||||
|
||||
const sink = await createFileAuditSink({
|
||||
path: "/var/log/suaveplan/audit.log",
|
||||
hmacKey: new TextEncoder().encode(process.env.AUDIT_HMAC_KEY ?? ""),
|
||||
});
|
||||
|
||||
await sink.emit({
|
||||
at: new Date().toISOString(),
|
||||
requestId: "r-1",
|
||||
traceId: null,
|
||||
route: "/admin/users",
|
||||
plugin: "core-admin",
|
||||
method: "POST",
|
||||
principal: { id: "user-1", scopes: ["admin"] },
|
||||
tenantId: "tenant-a",
|
||||
status: 200,
|
||||
outcome: "success",
|
||||
});
|
||||
|
||||
await sink.dispose();
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Run with a custom rotation policy, `append` mode, real `MeterProvider`,
|
||||
and an injected `FakeClock` for deterministic tests:
|
||||
|
||||
```ts
|
||||
import { FakeClock } from "@suaveplan/testing";
|
||||
import {
|
||||
MeterProvider,
|
||||
InMemoryMetricExporter,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import { createFileAuditSink } from "@suaveplan/api-audit-file/sink";
|
||||
import { createAuditFileInstruments } from "@suaveplan/api-audit-file/instruments";
|
||||
|
||||
const clock = new FakeClock(0);
|
||||
const exporter = new InMemoryMetricExporter(0);
|
||||
const reader = new PeriodicExportingMetricReader({ exporter });
|
||||
const meterProvider = new MeterProvider({ readers: [reader] });
|
||||
|
||||
const sink = await createFileAuditSink({
|
||||
path: "/tmp/audit.log",
|
||||
hmacKey: new TextEncoder().encode("secret"),
|
||||
clock: clock.read,
|
||||
rotation: { maxSizeBytes: 1024, maxAgeMs: 60_000, maxFiles: 5 },
|
||||
appendMode: "append",
|
||||
maxQueueSize: 256,
|
||||
instruments: createAuditFileInstruments({ meterProvider }),
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface CreateFileAuditSinkOptions`
|
||||
- `path: string` (required)
|
||||
- `hmacKey: Uint8Array` (required, non-empty)
|
||||
- `clock?: Clock` — default `() => Date.now()`
|
||||
- `rotation?: Partial<RotationPolicy>`
|
||||
- `appendMode?: "strict-sync" | "append"` — default `"strict-sync"`
|
||||
- `maxQueueSize?: number` — default 1024
|
||||
- `fs?: FsAdapter` — default `nodeFsAdapter`
|
||||
- `logger?: LoggerLike` — default no-op
|
||||
- `instruments?: AuditFileInstruments` — default global meter
|
||||
- `interface FileAuditSink`
|
||||
- `emit(record): Promise<void>`
|
||||
- `dispose(): Promise<void>`
|
||||
- `readonly queueDepth: number`
|
||||
- `createFileAuditSink(options): Promise<FileAuditSink>`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The drain loop is mutex-style: a single in-flight drain promise covers
|
||||
every concurrent `emit()` call so writes are serialised against the
|
||||
file handle even when callers fan out. Backpressure is signalled
|
||||
synchronously — `emit()` throws `QUEUE_FULL` before any await when the
|
||||
bounded queue is at capacity, so the host can apply load shedding
|
||||
without observing a write-latency spike. Rotation runs in the same
|
||||
drain frame as the triggering write so the rename always happens
|
||||
before the segment grows past the cap by one extra line. Retention
|
||||
runs after rotation; a retention failure is logged but does not
|
||||
propagate — the sink stays available even if `unlink` is transiently
|
||||
denied. The on-disk format is intentionally trivial (`{ record, hmac }`
|
||||
per line) so an operator with `jq` and `openssl dgst -sha256 -hmac` can
|
||||
verify integrity without any Suaveplan tooling on hand.
|
||||
@@ -0,0 +1,199 @@
|
||||
import { mkdtemp } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
InMemoryMetricExporter,
|
||||
MeterProvider,
|
||||
PeriodicExportingMetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
type AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { nodeFsAdapter } from "../fs-helpers/fs-helpers.js";
|
||||
import {
|
||||
createAuditFileInstruments,
|
||||
SINK_ATTRIBUTE,
|
||||
} from "../instruments/instruments.js";
|
||||
import { createFileAuditSink } from "./sink.js";
|
||||
|
||||
const KEY = new TextEncoder().encode("hmac-conformance-key");
|
||||
|
||||
interface Harness {
|
||||
dir: string;
|
||||
reader: PeriodicExportingMetricReader;
|
||||
instruments: ReturnType<typeof createAuditFileInstruments>;
|
||||
}
|
||||
|
||||
async function buildHarness(): Promise<Harness> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "audit-conf-"));
|
||||
const exporter = new InMemoryMetricExporter(0);
|
||||
const reader = new PeriodicExportingMetricReader({ exporter });
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
const instruments = createAuditFileInstruments({ meterProvider: provider });
|
||||
return { dir, reader, instruments };
|
||||
}
|
||||
|
||||
async function counterValue(
|
||||
reader: PeriodicExportingMetricReader,
|
||||
name: string
|
||||
): Promise<number> {
|
||||
const { resourceMetrics } = await reader.collect();
|
||||
for (const rm of resourceMetrics.scopeMetrics) {
|
||||
for (const m of rm.metrics) {
|
||||
if (m.descriptor.name !== name) continue;
|
||||
const dp = m.dataPoints.find(
|
||||
(d) =>
|
||||
(d.attributes as Readonly<Record<string, unknown>>).sink ===
|
||||
"file"
|
||||
);
|
||||
if (dp) return Number(dp.value);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
describe("api-audit-file sink — conformance", () => {
|
||||
let h: Harness;
|
||||
|
||||
beforeEach(async () => {
|
||||
h = await buildHarness();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await h.reader.shutdown();
|
||||
});
|
||||
|
||||
it("drop increments dropped_total with sink label file", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(h.dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
maxQueueSize: 1,
|
||||
instruments: h.instruments,
|
||||
fs: {
|
||||
...nodeFsAdapter,
|
||||
appendLine: () =>
|
||||
new Promise((resolve) => setTimeout(resolve, 50)),
|
||||
},
|
||||
});
|
||||
// Pre-fill queue WITHOUT waiting for drain by issuing concurrently.
|
||||
const a = sink.emit({
|
||||
at: new Date(0).toISOString(),
|
||||
requestId: "r-1",
|
||||
traceId: null,
|
||||
route: "/x",
|
||||
plugin: "p",
|
||||
method: "GET",
|
||||
principal: null,
|
||||
tenantId: null,
|
||||
status: 200,
|
||||
outcome: "success",
|
||||
});
|
||||
let caught: unknown;
|
||||
try {
|
||||
await sink.emit({
|
||||
at: new Date(1).toISOString(),
|
||||
requestId: "r-2",
|
||||
traceId: null,
|
||||
route: "/x",
|
||||
plugin: "p",
|
||||
method: "GET",
|
||||
principal: null,
|
||||
tenantId: null,
|
||||
status: 200,
|
||||
outcome: "success",
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
await a;
|
||||
await sink.dispose();
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.QUEUE_FULL
|
||||
);
|
||||
const dropped = await counterValue(h.reader, "api.audit.dropped_total");
|
||||
expect(dropped).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("records_total increments per successful write", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(h.dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
instruments: h.instruments,
|
||||
appendMode: "append",
|
||||
});
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
await sink.emit({
|
||||
at: new Date(i).toISOString(),
|
||||
requestId: `r-${i}`,
|
||||
traceId: null,
|
||||
route: "/x",
|
||||
plugin: "p",
|
||||
method: "GET",
|
||||
principal: null,
|
||||
tenantId: null,
|
||||
status: 200,
|
||||
outcome: "success",
|
||||
});
|
||||
}
|
||||
await sink.dispose();
|
||||
const records = await counterValue(h.reader, "api.audit.records_total");
|
||||
expect(records).toBe(3);
|
||||
});
|
||||
|
||||
it("concurrent emits share a single drain promise", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(h.dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
instruments: h.instruments,
|
||||
maxQueueSize: 10,
|
||||
appendMode: "append",
|
||||
fs: {
|
||||
...nodeFsAdapter,
|
||||
appendLine: (path, line) =>
|
||||
nodeFsAdapter.appendLine(path, line, false),
|
||||
},
|
||||
});
|
||||
const pending: Promise<void>[] = [];
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
pending.push(
|
||||
sink.emit({
|
||||
at: new Date(i).toISOString(),
|
||||
requestId: `r-${i}`,
|
||||
traceId: null,
|
||||
route: "/x",
|
||||
plugin: "p",
|
||||
method: "GET",
|
||||
principal: null,
|
||||
tenantId: null,
|
||||
status: 200,
|
||||
outcome: "success",
|
||||
})
|
||||
);
|
||||
}
|
||||
await Promise.all(pending);
|
||||
await sink.dispose();
|
||||
const records = await counterValue(h.reader, "api.audit.records_total");
|
||||
expect(records).toBe(5);
|
||||
});
|
||||
|
||||
it("SINK_ATTRIBUTE is the frozen `{ sink: 'file' }` literal", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(SINK_ATTRIBUTE).toEqual({ sink: "file" });
|
||||
expect(Object.isFrozen(SINK_ATTRIBUTE)).toBe(true);
|
||||
});
|
||||
});
|
||||
319
packages/api-stack/api/api-audit-file/src/sink/sink.test.ts
Normal file
319
packages/api-stack/api/api-audit-file/src/sink/sink.test.ts
Normal file
@@ -0,0 +1,319 @@
|
||||
import { chmod, mkdtemp, readdir, readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { verifyRecord } from "../hmac/hmac.js";
|
||||
import {
|
||||
type AuditRecord,
|
||||
AuditRecordSchema,
|
||||
canonicalise,
|
||||
} from "../schema/schema.js";
|
||||
import { createFileAuditSink } from "./sink.js";
|
||||
|
||||
const KEY = new TextEncoder().encode("super-secret-rotating-key");
|
||||
|
||||
async function freshDir(): Promise<string> {
|
||||
return mkdtemp(join(tmpdir(), "audit-sink-"));
|
||||
}
|
||||
|
||||
function makeRecord(i: number): AuditRecord {
|
||||
return {
|
||||
at: new Date(i * 1000).toISOString(),
|
||||
requestId: `r-${i}`,
|
||||
traceId: null,
|
||||
route: "/users",
|
||||
plugin: "core",
|
||||
method: "GET",
|
||||
principal: { id: "u-1", scopes: ["read"] },
|
||||
tenantId: "t-1",
|
||||
status: 200,
|
||||
outcome: "success",
|
||||
};
|
||||
}
|
||||
|
||||
describe("api-audit-file sink", () => {
|
||||
it("writes 1000 records as 1000 verified JSON lines", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const path = join(dir, "audit.log");
|
||||
const sink = await createFileAuditSink({
|
||||
path,
|
||||
hmacKey: KEY,
|
||||
appendMode: "append",
|
||||
});
|
||||
for (let i = 0; i < 1000; i += 1) {
|
||||
await sink.emit(makeRecord(i));
|
||||
}
|
||||
await sink.dispose();
|
||||
const text = await readFile(path, "utf8");
|
||||
const lines = text.trim().split("\n");
|
||||
expect(lines).toHaveLength(1000);
|
||||
for (const raw of lines) {
|
||||
const { record, hmac } = JSON.parse(raw) as {
|
||||
record: unknown;
|
||||
hmac: string;
|
||||
};
|
||||
const parsed = AuditRecordSchema.parse(record);
|
||||
expect(verifyRecord(KEY, canonicalise(parsed), hmac)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("unwriteable path raises AUDIT_FILE_PATH_NOT_WRITEABLE", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
await chmod(dir, 0o400);
|
||||
let caught: unknown;
|
||||
try {
|
||||
await createFileAuditSink({
|
||||
path: join(dir, "sub", "audit.log"),
|
||||
hmacKey: KEY,
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
} finally {
|
||||
await chmod(dir, 0o700);
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.PATH_NOT_WRITEABLE
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an empty hmacKey at construction", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
let caught: unknown;
|
||||
try {
|
||||
await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: new Uint8Array(0),
|
||||
});
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
});
|
||||
|
||||
it("rejects an invalid record with INVALID_RECORD", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
});
|
||||
const bad = { not: "an envelope" } as unknown as AuditRecord;
|
||||
let caught: unknown;
|
||||
try {
|
||||
await sink.emit(bad);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.INVALID_RECORD
|
||||
);
|
||||
await sink.dispose();
|
||||
});
|
||||
|
||||
it("strict-sync mode persists data on dispose", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const path = join(dir, "audit.log");
|
||||
const sink = await createFileAuditSink({
|
||||
path,
|
||||
hmacKey: KEY,
|
||||
appendMode: "strict-sync",
|
||||
});
|
||||
await sink.emit(makeRecord(1));
|
||||
await sink.dispose();
|
||||
const text = await readFile(path, "utf8");
|
||||
expect(text.split("\n").filter(Boolean)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("rotation produces a sibling and continues writing", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const path = join(dir, "audit.log");
|
||||
const sink = await createFileAuditSink({
|
||||
path,
|
||||
hmacKey: KEY,
|
||||
rotation: { maxSizeBytes: 200, maxFiles: 5 },
|
||||
appendMode: "append",
|
||||
});
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
await sink.emit(makeRecord(i));
|
||||
}
|
||||
await sink.dispose();
|
||||
const entries = (await readdir(dir)).filter((f) => f.endsWith(".log"));
|
||||
expect(entries.some((f) => f !== "audit.log")).toBe(true);
|
||||
const current = await readFile(path, "utf8");
|
||||
expect(current.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("queueDepth getter reports zero after drain", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
});
|
||||
await sink.emit(makeRecord(1));
|
||||
expect(sink.queueDepth).toBe(0);
|
||||
await sink.dispose();
|
||||
});
|
||||
|
||||
it("dispose is idempotent and drops emits afterward", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
});
|
||||
await sink.dispose();
|
||||
await sink.dispose();
|
||||
// post-dispose emit should be a silent drop (no throw)
|
||||
await sink.emit(makeRecord(1));
|
||||
});
|
||||
|
||||
it("writer failure surfaces WRITER_FAILED and increments dropped", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
let attempt = 0;
|
||||
const failingFs = {
|
||||
...(await import("../fs-helpers/fs-helpers.js")).nodeFsAdapter,
|
||||
async appendLine() {
|
||||
attempt += 1;
|
||||
throw new Error("EIO");
|
||||
},
|
||||
};
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
fs: failingFs,
|
||||
});
|
||||
let caught: unknown;
|
||||
try {
|
||||
await sink.emit(makeRecord(1));
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.WRITER_FAILED
|
||||
);
|
||||
expect(attempt).toBe(1);
|
||||
await sink.dispose();
|
||||
});
|
||||
|
||||
it("propagates AuditFileError class directly from writer", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const nfa = (await import("../fs-helpers/fs-helpers.js")).nodeFsAdapter;
|
||||
const failingFs = {
|
||||
...nfa,
|
||||
async appendLine() {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.WRITER_FAILED,
|
||||
"error.audit.file.writer_failed"
|
||||
);
|
||||
},
|
||||
};
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
fs: failingFs,
|
||||
});
|
||||
let caught: unknown;
|
||||
try {
|
||||
await sink.emit(makeRecord(1));
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AuditFileError);
|
||||
await sink.dispose();
|
||||
});
|
||||
|
||||
it("rotation failure surfaces ROTATION_FAILED through emit", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const nfa = (await import("../fs-helpers/fs-helpers.js")).nodeFsAdapter;
|
||||
const failingFs = {
|
||||
...nfa,
|
||||
async rename() {
|
||||
throw new Error("EBUSY");
|
||||
},
|
||||
};
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
rotation: { maxAgeMs: 1 },
|
||||
clock: (() => {
|
||||
let t = 0;
|
||||
return () => {
|
||||
t += 1000;
|
||||
return t;
|
||||
};
|
||||
})(),
|
||||
fs: failingFs,
|
||||
});
|
||||
let caught: unknown;
|
||||
try {
|
||||
await sink.emit(makeRecord(1));
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect((caught as AuditFileError).code).toBe(
|
||||
AUDIT_FILE_CODES.ROTATION_FAILED
|
||||
);
|
||||
await sink.dispose();
|
||||
});
|
||||
|
||||
it("retention failure is logged but not surfaced from emit", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const dir = await freshDir();
|
||||
const errors: string[] = [];
|
||||
const logger = {
|
||||
debug() {},
|
||||
info() {},
|
||||
warn() {},
|
||||
error(msg: string) {
|
||||
errors.push(msg);
|
||||
},
|
||||
};
|
||||
const nfa = (await import("../fs-helpers/fs-helpers.js")).nodeFsAdapter;
|
||||
const failingFs = {
|
||||
...nfa,
|
||||
async unlink() {
|
||||
throw new Error("EACCES");
|
||||
},
|
||||
};
|
||||
const sink = await createFileAuditSink({
|
||||
path: join(dir, "audit.log"),
|
||||
hmacKey: KEY,
|
||||
rotation: { maxSizeBytes: 10, maxFiles: 1 },
|
||||
logger,
|
||||
fs: failingFs,
|
||||
appendMode: "append",
|
||||
});
|
||||
for (let i = 0; i < 8; i += 1) {
|
||||
await sink.emit(makeRecord(i));
|
||||
}
|
||||
await sink.dispose();
|
||||
expect(errors.some((m) => m.includes("retention"))).toBe(true);
|
||||
});
|
||||
});
|
||||
226
packages/api-stack/api/api-audit-file/src/sink/sink.ts
Normal file
226
packages/api-stack/api/api-audit-file/src/sink/sink.ts
Normal file
@@ -0,0 +1,226 @@
|
||||
import { dirname } from "node:path";
|
||||
import type { Clock } from "@suaveplan/types/clock";
|
||||
import type { LoggerLike } from "@suaveplan/types/logger-like";
|
||||
import {
|
||||
AUDIT_FILE_CODES,
|
||||
AuditFileError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { type FsAdapter, nodeFsAdapter } from "../fs-helpers/fs-helpers.js";
|
||||
import { signRecord } from "../hmac/hmac.js";
|
||||
import {
|
||||
type AuditFileInstruments,
|
||||
createAuditFileInstruments,
|
||||
SINK_ATTRIBUTE,
|
||||
} from "../instruments/instruments.js";
|
||||
import { enforceRetention } from "../retention/retention.js";
|
||||
import {
|
||||
type RotationState,
|
||||
resolveRotationPolicy,
|
||||
rotate,
|
||||
shouldRotate,
|
||||
} from "../rotation/rotation.js";
|
||||
import {
|
||||
type AuditRecord,
|
||||
AuditRecordSchema,
|
||||
buildSignedLine,
|
||||
canonicalise,
|
||||
} from "../schema/schema.js";
|
||||
import type {
|
||||
AppendMode,
|
||||
CreateFileAuditSinkOptions,
|
||||
FileAuditSink,
|
||||
} from "./sink.types.js";
|
||||
|
||||
const NOOP_LOG = (): void => undefined;
|
||||
const NOOP_LOGGER: LoggerLike = {
|
||||
debug: NOOP_LOG,
|
||||
info: NOOP_LOG,
|
||||
warn: NOOP_LOG,
|
||||
error: NOOP_LOG,
|
||||
};
|
||||
|
||||
const DEFAULT_CLOCK: Clock = () => Date.now();
|
||||
const DEFAULT_QUEUE_SIZE = 1024;
|
||||
|
||||
/**
|
||||
* Build a file-rotating, HMAC-signed audit sink that implements the
|
||||
* `RequestAuditSink` contract from `@suaveplan/api-host/request-pipeline`.
|
||||
* The sink fails fast on a non-writeable target directory and writes
|
||||
* JSON-per-line with HMAC-SHA-256 over canonical bytes. Size + time
|
||||
* rotation rolls the active file to a timestamp-suffixed sibling;
|
||||
* retention prunes the oldest siblings to keep at most `maxFiles`.
|
||||
*/
|
||||
export async function createFileAuditSink(
|
||||
options: CreateFileAuditSinkOptions
|
||||
): Promise<FileAuditSink> {
|
||||
if (options.hmacKey.byteLength === 0) {
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.INVALID_RECORD,
|
||||
"error.audit.file.invalid_options",
|
||||
{ context: { field: "hmacKey", reason: "empty" } }
|
||||
);
|
||||
}
|
||||
const fs = options.fs ?? nodeFsAdapter;
|
||||
await fs.ensureWriteableDir(dirname(options.path));
|
||||
const clock = options.clock ?? DEFAULT_CLOCK;
|
||||
const policy = resolveRotationPolicy(options.rotation);
|
||||
const appendMode: AppendMode = options.appendMode ?? "strict-sync";
|
||||
const maxQueueSize = options.maxQueueSize ?? DEFAULT_QUEUE_SIZE;
|
||||
const logger = options.logger ?? NOOP_LOGGER;
|
||||
const instruments = options.instruments ?? createAuditFileInstruments();
|
||||
const state: RotationState = {
|
||||
openedAtMs: clock(),
|
||||
bytesWritten: await fs.fileSize(options.path),
|
||||
policy,
|
||||
};
|
||||
return new SinkImpl({
|
||||
path: options.path,
|
||||
hmacKey: options.hmacKey,
|
||||
clock,
|
||||
appendMode,
|
||||
maxQueueSize,
|
||||
fs,
|
||||
logger,
|
||||
instruments,
|
||||
state,
|
||||
});
|
||||
}
|
||||
|
||||
interface SinkContext {
|
||||
readonly path: string;
|
||||
readonly hmacKey: Uint8Array;
|
||||
readonly clock: Clock;
|
||||
readonly appendMode: AppendMode;
|
||||
readonly maxQueueSize: number;
|
||||
readonly fs: FsAdapter;
|
||||
readonly logger: LoggerLike;
|
||||
readonly instruments: AuditFileInstruments;
|
||||
readonly state: RotationState;
|
||||
}
|
||||
|
||||
class SinkImpl implements FileAuditSink {
|
||||
private readonly ctx: SinkContext;
|
||||
private queue: AuditRecord[] = [];
|
||||
private disposed = false;
|
||||
private drainPromise: Promise<void> | null = null;
|
||||
|
||||
public constructor(ctx: SinkContext) {
|
||||
this.ctx = ctx;
|
||||
}
|
||||
|
||||
public get queueDepth(): number {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
public async emit(record: AuditRecord): Promise<void> {
|
||||
if (this.disposed) {
|
||||
this.dropAndCount();
|
||||
return;
|
||||
}
|
||||
let validated: AuditRecord;
|
||||
try {
|
||||
validated = AuditRecordSchema.parse(record);
|
||||
} catch (cause) {
|
||||
this.ctx.instruments.droppedTotal.add(1, SINK_ATTRIBUTE);
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.INVALID_RECORD,
|
||||
"error.audit.file.invalid_record",
|
||||
{ cause }
|
||||
);
|
||||
}
|
||||
if (this.queue.length >= this.ctx.maxQueueSize) {
|
||||
this.dropAndCount();
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.QUEUE_FULL,
|
||||
"error.audit.file.queue_full",
|
||||
{ context: { depth: this.queue.length } }
|
||||
);
|
||||
}
|
||||
this.queue.push(validated);
|
||||
await this.drain();
|
||||
}
|
||||
|
||||
public async dispose(): Promise<void> {
|
||||
if (this.disposed) return;
|
||||
this.disposed = true;
|
||||
await this.drain();
|
||||
}
|
||||
|
||||
private dropAndCount(): void {
|
||||
this.ctx.instruments.droppedTotal.add(1, SINK_ATTRIBUTE);
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
const existing = this.drainPromise;
|
||||
if (existing !== null) return existing;
|
||||
const next = this.drainLoop().finally(() => {
|
||||
this.drainPromise = null;
|
||||
});
|
||||
this.drainPromise = next;
|
||||
return next;
|
||||
}
|
||||
|
||||
private async drainLoop(): Promise<void> {
|
||||
while (this.queue.length > 0) {
|
||||
const next = this.queue[0] as AuditRecord;
|
||||
try {
|
||||
await this.writeOne(next);
|
||||
this.queue.shift();
|
||||
} catch (err) {
|
||||
this.ctx.logger.error("audit.file.write_failed", {
|
||||
error: (err as Error).message,
|
||||
});
|
||||
this.queue.shift();
|
||||
this.dropAndCount();
|
||||
if (err instanceof AuditFileError) {
|
||||
throw err;
|
||||
}
|
||||
throw new AuditFileError(
|
||||
AUDIT_FILE_CODES.WRITER_FAILED,
|
||||
"error.audit.file.writer_failed",
|
||||
{ cause: err }
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async writeOne(record: AuditRecord): Promise<void> {
|
||||
await this.maybeRotate();
|
||||
const bytes = canonicalise(record);
|
||||
const hmacHex = signRecord(this.ctx.hmacKey, bytes);
|
||||
const line = buildSignedLine(bytes, hmacHex);
|
||||
const sync = this.ctx.appendMode === "strict-sync";
|
||||
await this.ctx.fs.appendLine(this.ctx.path, line, sync);
|
||||
this.ctx.state.bytesWritten += Buffer.byteLength(line, "utf8");
|
||||
this.ctx.instruments.recordsTotal.add(1, SINK_ATTRIBUTE);
|
||||
}
|
||||
|
||||
private async maybeRotate(): Promise<void> {
|
||||
if (!shouldRotate(this.ctx.state, this.ctx.clock())) return;
|
||||
try {
|
||||
await rotate(
|
||||
this.ctx.fs,
|
||||
this.ctx.path,
|
||||
this.ctx.state,
|
||||
this.ctx.clock
|
||||
);
|
||||
} catch (err) {
|
||||
this.ctx.logger.error("audit.file.rotation_failed", {
|
||||
error: (err as Error).message,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
await enforceRetention(
|
||||
this.ctx.fs,
|
||||
this.ctx.path,
|
||||
this.ctx.state.policy.maxFiles,
|
||||
this.ctx.instruments
|
||||
);
|
||||
} catch (err) {
|
||||
this.ctx.logger.error("audit.file.retention_failed", {
|
||||
error: (err as Error).message,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user