Initial Commit
This commit is contained in:
109
packages/api-stack/api/api-2pc/README.md
Normal file
109
packages/api-stack/api/api-2pc/README.md
Normal file
@@ -0,0 +1,109 @@
|
||||
# @suaveplan/api-2pc - Two-phase commit coordinator for the Suaveplan API plugin system.
|
||||
|
||||
Two-phase commit coordinator for the Suaveplan API plugin system. Drives Postgres PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK PREPARED across registered participants with a host-owned ledger in suaveplan_2pc_transactions. Refuses non-preparable engines as writers (SQLite, SurrealDB) but accepts them as read-only voters. Default timeoutMs 5000, hard ceiling 30000, default outstanding-prepared cap 256, all operator-tunable. Ships boot-time orphan resolution via pg_prepared_xacts, the full TWOPC_* error namespace, OTel transaction/prepare/commit/rollback spans plus prepare_duration_ms / commit_duration_ms histograms / transactions_total counter / outstanding_prepared gauge, and run2pcParticipantConformance on the ./testing subpath. Server-tier — Bun + Node.
|
||||
|
||||
> **Postgres-only writer surface.** SQLite and SurrealDB cannot honour
|
||||
> `PREPARE TRANSACTION` and are refused as writers at install time.
|
||||
> They MAY participate as read-only voters that contribute a
|
||||
> `commit`/`abort` vote based on their own consistency check, but the
|
||||
> coordinator never sends them prepare / commit / rollback SQL.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [coordinator](./src/coordinator/coordinator.md) — Top-level orchestration for the 2PC protocol. `createTwoPcCoordinator` builds a fully-wired coordinator that holds a pa…
|
||||
- [coordinator](./src/coordinator/coordinator.md) — Top-level orchestration for the 2PC protocol. `createTwoPcCoordinator` builds a fully-wired coordinator that holds a pa…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — This module owns the `TWOPC_*` error-code namespace and the `TwoPcError` concrete class for the `@suaveplan/api-2pc` pa…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — This module owns the `TWOPC_*` error-code namespace and the `TwoPcError` concrete class for the `@suaveplan/api-2pc` pa…
|
||||
- [instruments](./src/instruments/instruments.md) — Builds the OTel instrument registry the 2PC coordinator emits against on every transaction. Each instrument's name foll…
|
||||
- [instruments](./src/instruments/instruments.md) — Builds the OTel instrument registry the 2PC coordinator emits against on every transaction. Each instrument's name foll…
|
||||
- [outstanding-cap](./src/limits/outstanding-cap.md) — Admission control for in-flight 2PC transactions. The coordinator holds a fixed-size budget of "outstanding prepared tr…
|
||||
- [timeout-ceiling](./src/limits/timeout-ceiling.md) — Enforces the hard 30 000 ms ceiling on caller-supplied `timeoutMs` values. The check fires at the public entry point of…
|
||||
- [options](./src/options/options.md) — Defines the boundary-validated option shape consumed by `createTwoPcCoordinator`. Every tunable that affects coordinato…
|
||||
- [options](./src/options/options.md) — Defines the boundary-validated option shape consumed by `createTwoPcCoordinator`. Every tunable that affects coordinato…
|
||||
- [persistence](./src/persistence/persistence.md) — Persists every coordinator state transition into the `suaveplan_2pc_transactions` ledger table. The table is owned by `…
|
||||
- [persistence](./src/persistence/persistence.md) — Persists every coordinator state transition into the `suaveplan_2pc_transactions` ledger table. The table is owned by `…
|
||||
- [commit](./src/phases/commit.md) — Phase-2 commit for a single participant. Once every participant has returned `commit` from phase 1 and the coordinator…
|
||||
- [gid](./src/phases/gid.md) — Builds and parses the global transaction id (gid) string passed to Postgres' `PREPARE TRANSACTION '<gid>'`. The format…
|
||||
- [prepare](./src/phases/prepare.md) — Implements phase 1 of the 2PC protocol for a single participant. The coordinator iterates participants and calls this m…
|
||||
- [rollback](./src/phases/rollback.md) — Phase-2 rollback for a single participant. Used both by the coordinator's abort path (vote_abort or prepare failure) an…
|
||||
- [recovery](./src/recovery/recovery.md) — Boot-time orphan resolution for the 2PC coordinator. When a host crashes between phase 1 and phase 2, prepared transact…
|
||||
- [recovery](./src/recovery/recovery.md) — Boot-time orphan resolution for the 2PC coordinator. When a host crashes between phase 1 and phase 2, prepared transact…
|
||||
- [registry](./src/registry/participant-registry.md) — The participant registry is the install-time admission control between plugins and the 2PC coordinator. Plugins declare…
|
||||
- [participant-registry](./src/registry/participant-registry.md) — The participant registry is the install-time admission control between plugins and the 2PC coordinator. Plugins declare…
|
||||
- [testing](./src/testing/conformance.md) — Provides `run2pcParticipantConformance(factory)` — a runnable test suite that pins the contract every 2PC writer partic…
|
||||
- [conformance](./src/testing/conformance.md) — Provides `run2pcParticipantConformance(factory)` — a runnable test suite that pins the contract every 2PC writer partic…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Why
|
||||
|
||||
Some cross-database workflows demand atomic visibility — no partial
|
||||
state, no eventual convergence. Per the API Plugin Foundation design
|
||||
doc §9.2 the coordinator runs phase-1 prepare across every registered
|
||||
participant, persists the vote tally to `suaveplan_2pc_transactions`
|
||||
on the host's metadata DB, then issues `COMMIT PREPARED` or
|
||||
`ROLLBACK PREPARED` against every writer participant. Recovery on the
|
||||
next boot resolves any prepared transactions left over from a host
|
||||
crash.
|
||||
|
||||
## Modules
|
||||
|
||||
- `coordinator/` — `createTwoPcCoordinator` + `run2pc` factory and
|
||||
state machine.
|
||||
- `registry/` — `TwoPcParticipantRegistry` (admission control).
|
||||
- `phases/` — `preparePhaseOne`, `commitPhaseTwo`, `rollbackPhaseTwo`,
|
||||
`gid` helpers.
|
||||
- `persistence/` — `TwoPcLedgerStore` over the host-owned
|
||||
`suaveplan_2pc_transactions` table.
|
||||
- `recovery/` — boot-time orphan resolution via `pg_prepared_xacts`.
|
||||
- `limits/` — `OutstandingPreparedCap` (admission), `assertTimeoutBelowCeiling`
|
||||
(per-call ceiling).
|
||||
- `instruments/` — OTel meter/tracer bundle.
|
||||
- `options/` — Zod-validated coordinator options + defaults.
|
||||
- `error-codes/` — 10 `TWOPC_*` codes + `TwoPcError`.
|
||||
- `types/` — `Run2PcInput`, `Run2PcOutcome`, participant + state types.
|
||||
- `testing/` — `run2pcParticipantConformance(factory)` suite.
|
||||
|
||||
## Quick start
|
||||
|
||||
```ts
|
||||
import { createTwoPcCoordinator } from "@suaveplan/api-2pc";
|
||||
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-01" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () => crypto.randomUUID(),
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "billing", input: { invoiceId: "inv-1" } },
|
||||
{ id: "inventory", input: { sku: "abc-9" } },
|
||||
],
|
||||
});
|
||||
// outcome.state is "committed" | "aborted" | "in_doubt"
|
||||
```
|
||||
|
||||
## Defaults
|
||||
|
||||
- `timeoutMs`: **5000** (operator-tunable per-call up to the
|
||||
30000 ms ceiling).
|
||||
- `outstandingPreparedCap`: **256** (operator-tunable via host
|
||||
config).
|
||||
|
||||
## Error namespace
|
||||
|
||||
Every production code path throws `TwoPcError` with one of the ten
|
||||
`TWOPC_*` codes. See `src/error-codes/error-codes.md` for the full
|
||||
catalogue.
|
||||
|
||||
## Conformance
|
||||
|
||||
Every adapter implementing a writer participant MUST run
|
||||
`run2pcParticipantConformance({ create: ... })` in a
|
||||
`tests/conformance/run-2pc-participant.test.ts` file. The harness
|
||||
pins prepare / commit / rollback semantics and the idempotency
|
||||
contract.
|
||||
|
||||
123
packages/api-stack/api/api-2pc/package.json
Normal file
123
packages/api-stack/api/api-2pc/package.json
Normal file
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"name": "@suaveplan/api-2pc",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Two-phase commit coordinator for the Suaveplan API plugin foundation. Drives Postgres PREPARE TRANSACTION / COMMIT PREPARED / ROLLBACK PREPARED across registered participants with a host-owned ledger in suaveplan_2pc_transactions. Refuses non-preparable engines as writers (SQLite, SurrealDB) but accepts them as read-only voters. Default timeoutMs 5000, hard ceiling 30000, default outstanding-prepared cap 256, all operator-tunable. Ships boot-time orphan resolution via pg_prepared_xacts, the full TWOPC_* error namespace, OTel transaction/prepare/commit/rollback spans plus prepare_duration_ms / commit_duration_ms histograms / transactions_total counter / outstanding_prepared gauge, and run2pcParticipantConformance on the ./testing subpath. Server-tier — Bun + Node.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./coordinator": {
|
||||
"import": {
|
||||
"types": "./dist/coordinator/coordinator.d.ts",
|
||||
"default": "./dist/coordinator/coordinator.js"
|
||||
}
|
||||
},
|
||||
"./registry": {
|
||||
"import": {
|
||||
"types": "./dist/registry/participant-registry.d.ts",
|
||||
"default": "./dist/registry/participant-registry.js"
|
||||
}
|
||||
},
|
||||
"./phases": {
|
||||
"import": {
|
||||
"types": "./dist/phases/index.d.ts",
|
||||
"default": "./dist/phases/index.js"
|
||||
}
|
||||
},
|
||||
"./persistence": {
|
||||
"import": {
|
||||
"types": "./dist/persistence/persistence.d.ts",
|
||||
"default": "./dist/persistence/persistence.js"
|
||||
}
|
||||
},
|
||||
"./recovery": {
|
||||
"import": {
|
||||
"types": "./dist/recovery/recovery.d.ts",
|
||||
"default": "./dist/recovery/recovery.js"
|
||||
}
|
||||
},
|
||||
"./limits": {
|
||||
"import": {
|
||||
"types": "./dist/limits/index.d.ts",
|
||||
"default": "./dist/limits/index.js"
|
||||
}
|
||||
},
|
||||
"./instruments": {
|
||||
"import": {
|
||||
"types": "./dist/instruments/instruments.d.ts",
|
||||
"default": "./dist/instruments/instruments.js"
|
||||
}
|
||||
},
|
||||
"./error-codes": {
|
||||
"import": {
|
||||
"types": "./dist/error-codes/error-codes.d.ts",
|
||||
"default": "./dist/error-codes/error-codes.js"
|
||||
}
|
||||
},
|
||||
"./options": {
|
||||
"import": {
|
||||
"types": "./dist/options/options.d.ts",
|
||||
"default": "./dist/options/options.js"
|
||||
}
|
||||
},
|
||||
"./testing": {
|
||||
"import": {
|
||||
"types": "./dist/testing/conformance.d.ts",
|
||||
"default": "./dist/testing/conformance.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "bun test --timeout=30000 src/",
|
||||
"test:watch": "bun test --watch src/",
|
||||
"test:coverage": "bun test --coverage --coverage-reporter=lcov --timeout=30000 src/",
|
||||
"test:integration": "vitest run --config vitest.integration.config.ts --passWithNoTests",
|
||||
"lint": "bunx biome check src/ tests/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/ tests/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/ tests/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/ tests/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/api-types": "workspace:^",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@suaveplan/testing-integration": "workspace:^",
|
||||
"@suaveplan/testing-integration-postgres": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"postgres": "catalog:database",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": ["dist", "src", "README.md"]
|
||||
}
|
||||
150
packages/api-stack/api/api-2pc/src/coordinator/coordinator.md
Normal file
150
packages/api-stack/api/api-2pc/src/coordinator/coordinator.md
Normal file
@@ -0,0 +1,150 @@
|
||||
# `coordinator/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Top-level orchestration for the 2PC protocol. `createTwoPcCoordinator`
|
||||
builds a fully-wired coordinator that holds a participant registry, a
|
||||
ledger store, an admission counter, and an OTel instrument bundle.
|
||||
The `run2pc({ participants, timeoutMs? })` entry point owns the full
|
||||
state machine described in design doc §9.2:
|
||||
|
||||
1. Admission via the outstanding-prepared cap.
|
||||
2. Ledger insert at `voting`.
|
||||
3. Phase-1 prepare per participant, with the
|
||||
`2pc.prepare_duration_ms` histogram and a `2pc.prepare` span per
|
||||
participant.
|
||||
4. Either unanimous commit → `markAllVotedCommit` → phase-2
|
||||
`COMMIT PREPARED` per writer; OR any abort/error →
|
||||
`markAborting` → phase-2 `ROLLBACK PREPARED` per prepared writer.
|
||||
5. Terminal ledger stamp (`committed` / `aborted` / `in_doubt`).
|
||||
|
||||
The coordinator is the only path application code ever touches.
|
||||
Every smaller module (phases, persistence, recovery, limits) is
|
||||
internal plumbing this factory composes.
|
||||
|
||||
## Features
|
||||
|
||||
- Single public surface: `createTwoPcCoordinator(deps)` returns
|
||||
`{ run2pc, outstandingCount, options }`.
|
||||
- Default `timeoutMs` from options (5000), per-call override accepted
|
||||
up to the 30000 ceiling (enforced via
|
||||
`limits/timeout-ceiling`).
|
||||
- Admission cap (default 256) enforced via
|
||||
`limits/outstanding-cap`; over-cap calls fail with
|
||||
`TWOPC_OUTSTANDING_LIMIT_EXCEEDED` before any participant is
|
||||
contacted.
|
||||
- Hooks every documented OTel instrument (`2pc.transaction` span,
|
||||
`2pc.prepare` / `2pc.commit` / `2pc.rollback` child spans,
|
||||
`2pc.transactions_total` counter, `prepare`/`commit_duration_ms`
|
||||
histograms, `outstanding_prepared` gauge).
|
||||
- Ledger persistence is bidirectional — every state transition writes
|
||||
through `TwoPcLedgerStore`, so the recovery runner has a full audit
|
||||
trail on restart.
|
||||
- Abort handling: vote_abort, prepare_failed, and timeout all funnel
|
||||
to the rollback branch; rollback errors escalate to `in_doubt`
|
||||
rather than failing the rollback path itself.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { createTwoPcCoordinator } from "@suaveplan/api-2pc/coordinator";
|
||||
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-01" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () => crypto.randomUUID(),
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "billing", input: { invoiceId: "inv-1" } },
|
||||
{ id: "inventory", input: { sku: "abc-9" } },
|
||||
],
|
||||
});
|
||||
console.log(outcome.state); // "committed" | "aborted" | "in_doubt"
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Inject a custom instruments bundle to feed a hand-rolled
|
||||
MeterProvider, override the default timeout per call, and consume the
|
||||
per-participant vote summary on the outcome:
|
||||
|
||||
```ts
|
||||
import { createTwoPcCoordinator } from "@suaveplan/api-2pc/coordinator";
|
||||
import { createTwoPcInstruments } from "@suaveplan/api-2pc/instruments";
|
||||
|
||||
const instruments = createTwoPcInstruments({
|
||||
meterProvider,
|
||||
outstandingPreparedSnapshot: () => coordinator.outstandingCount(),
|
||||
});
|
||||
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: {
|
||||
hostInstanceId: "host-01",
|
||||
defaultTimeoutMs: 10_000,
|
||||
outstandingPreparedCap: 64,
|
||||
},
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () => crypto.randomUUID(),
|
||||
instruments,
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "billing", input: { invoiceId } },
|
||||
{ id: "audit", input: { event: "invoice.created" } },
|
||||
],
|
||||
timeoutMs: 2_500,
|
||||
});
|
||||
for (const v of outcome.votes) {
|
||||
auditLog.append({
|
||||
txId: outcome.txId,
|
||||
participant: v.participantId,
|
||||
vote: v.vote,
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface TwoPcCoordinatorDeps`
|
||||
- `options: TwoPcCoordinatorOptions`
|
||||
- `registry: TwoPcParticipantRegistry`
|
||||
- `ledgerDb: LedgerDb`
|
||||
- `clock: Clock`
|
||||
- `idGenerator: IdGenerator`
|
||||
- `instruments?: TwoPcInstruments`
|
||||
- `interface TwoPcCoordinator`
|
||||
- `run2pc(input: Run2PcInput): Promise<Run2PcOutcome>`
|
||||
- `outstandingCount(): number`
|
||||
- `options: ResolvedTwoPcCoordinatorOptions`
|
||||
- `createTwoPcCoordinator(deps): TwoPcCoordinator` — factory.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The function is structured as one large `try` / `finally` so cleanup
|
||||
(span end, deadline timer clear, admission release) runs regardless
|
||||
of which branch fires. The pattern keeps the happy path linear and
|
||||
the failure paths exhaustive without nested try blocks.
|
||||
|
||||
The "any commit failure → in_doubt" escalation is deliberate. Once
|
||||
the coordinator has issued `COMMIT PREPARED` against at least one
|
||||
participant, the only correct follow-up is to keep trying — but the
|
||||
hot path cannot block on that. Instead, the row is marked `in_doubt`,
|
||||
the OTel counter ticks the `in_doubt` outcome, and the recovery
|
||||
runner picks it up on the next boot.
|
||||
|
||||
Vote-abort and prepare-failed paths both call `ac.abort()` BEFORE
|
||||
running phase 2, so any prepared participants whose subsequent SQL
|
||||
hits the signal map to `TWOPC_TIMEOUT` rather than getting stuck. The
|
||||
deadline timer is cleared in the `finally` so a successful commit
|
||||
does not leave a dangling abort handle on the AbortController.
|
||||
@@ -0,0 +1,414 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { LedgerDb } from "../persistence/persistence.js";
|
||||
import { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { createTwoPcCoordinator } from "./coordinator.js";
|
||||
|
||||
let nextId = 0;
|
||||
const FROZEN_NOW = 1_700_000_000_000;
|
||||
const baseDeps = () => {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
return {
|
||||
ledgerDb: db,
|
||||
registry,
|
||||
clock: () => FROZEN_NOW,
|
||||
idGenerator: () => `tx-${++nextId}`,
|
||||
options: { hostInstanceId: "h" },
|
||||
};
|
||||
};
|
||||
|
||||
interface Spy {
|
||||
readonly id: string;
|
||||
readonly calls: string[];
|
||||
participant: RegisteredTwoPcParticipant;
|
||||
}
|
||||
|
||||
function buildSpy(
|
||||
id: string,
|
||||
opts: {
|
||||
vote?: "commit" | "abort";
|
||||
writeIntent?: boolean;
|
||||
voterMode?: "writer" | "read-only";
|
||||
sqlFail?: { stage: "prepare" | "commit" | "rollback"; err: Error };
|
||||
prepareThrows?: Error;
|
||||
delayMs?: number;
|
||||
} = {}
|
||||
): Spy {
|
||||
const calls: string[] = [];
|
||||
const participant: RegisteredTwoPcParticipant = {
|
||||
id,
|
||||
voterMode: opts.voterMode ?? "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.sqlFail) {
|
||||
const matchesStage =
|
||||
(opts.sqlFail.stage === "prepare" &&
|
||||
sql.startsWith("PREPARE")) ||
|
||||
(opts.sqlFail.stage === "commit" &&
|
||||
sql.startsWith("COMMIT")) ||
|
||||
(opts.sqlFail.stage === "rollback" &&
|
||||
sql.startsWith("ROLLBACK"));
|
||||
if (matchesStage) throw opts.sqlFail.err;
|
||||
}
|
||||
if (opts.delayMs) {
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, opts.delayMs)
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => {
|
||||
if (opts.prepareThrows) throw opts.prepareThrows;
|
||||
return {
|
||||
vote: opts.vote ?? "commit",
|
||||
writeIntent: opts.writeIntent ?? true,
|
||||
};
|
||||
},
|
||||
};
|
||||
return { id, calls, participant };
|
||||
}
|
||||
|
||||
describe("createTwoPcCoordinator", () => {
|
||||
it("unanimous commit issues COMMIT PREPARED on every participant", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b");
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
const outcome = await c.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
expect(outcome.state).toBe("committed");
|
||||
expect(outcome.votes).toEqual([
|
||||
{ participantId: "a", vote: "commit" },
|
||||
{ participantId: "b", vote: "commit" },
|
||||
]);
|
||||
expect(
|
||||
a.calls.filter((s) => s.startsWith("COMMIT PREPARED")).length
|
||||
).toBe(1);
|
||||
expect(
|
||||
b.calls.filter((s) => s.startsWith("COMMIT PREPARED")).length
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("any vote-abort rolls back everywhere with TWOPC_VOTE_ABORT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b", { vote: "abort" });
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.VOTE_ABORT);
|
||||
}
|
||||
expect(
|
||||
a.calls.filter((s) => s.startsWith("ROLLBACK PREPARED")).length
|
||||
).toBe(1);
|
||||
expect(
|
||||
b.calls.filter((s) => s.startsWith("ROLLBACK PREPARED")).length
|
||||
).toBe(0); // b voted abort, never prepared.
|
||||
});
|
||||
|
||||
it("deadline expiry during prepare triggers rollback with TWOPC_TIMEOUT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const slow = buildSpy("slow", {
|
||||
sqlFail: {
|
||||
stage: "prepare",
|
||||
err: new Error("connection aborted"),
|
||||
},
|
||||
});
|
||||
// Patch the run() implementation to abort the signal mid-prepare
|
||||
// — emulates the deadline elapsing while the SQL is in flight.
|
||||
const original = slow.participant.connection.run;
|
||||
let runs = 0;
|
||||
slow.participant = {
|
||||
...slow.participant,
|
||||
connection: {
|
||||
...slow.participant.connection,
|
||||
run: async (sql: string) => {
|
||||
runs += 1;
|
||||
if (runs === 1) {
|
||||
// Sleep long enough for the deadline (1ms) to fire.
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
return original(sql);
|
||||
},
|
||||
},
|
||||
};
|
||||
deps.registry.register(slow.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [{ id: "slow", input: {} }],
|
||||
timeoutMs: 1,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.TIMEOUT);
|
||||
}
|
||||
});
|
||||
|
||||
it("commit failure escalates to TWOPC_IN_DOUBT with the row marked in_doubt", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a", {
|
||||
sqlFail: { stage: "commit", err: new Error("commit failed") },
|
||||
});
|
||||
deps.registry.register(a.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
}
|
||||
});
|
||||
|
||||
it("rollback failure escalates to TWOPC_IN_DOUBT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b", { vote: "abort" });
|
||||
// a's rollback SQL fails (non-idempotency error)
|
||||
const originalA = a.participant.connection.run;
|
||||
a.participant = {
|
||||
...a.participant,
|
||||
connection: {
|
||||
...a.participant.connection,
|
||||
run: async (sql: string) => {
|
||||
if (sql.startsWith("ROLLBACK PREPARED"))
|
||||
throw new Error("connection dropped");
|
||||
return originalA(sql);
|
||||
},
|
||||
},
|
||||
};
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses unknown participant id with TWOPC_PREPARE_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "ghost", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects timeoutMs above the hard ceiling without contacting participants", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
deps.registry.register(a.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({
|
||||
participants: [{ id: "a", input: {} }],
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED
|
||||
);
|
||||
}
|
||||
expect(a.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("outstanding-prepared cap rejects 257th concurrent transaction", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
deps.options = { hostInstanceId: "h", outstandingPreparedCap: 2 };
|
||||
let resolveA: () => void = () => undefined;
|
||||
let resolveB: () => void = () => undefined;
|
||||
const aGate = new Promise<void>((r) => {
|
||||
resolveA = r;
|
||||
});
|
||||
const bGate = new Promise<void>((r) => {
|
||||
resolveB = r;
|
||||
});
|
||||
const a = buildSpy("a");
|
||||
const b = buildSpy("b");
|
||||
a.participant = {
|
||||
...a.participant,
|
||||
prepare: async () => {
|
||||
await aGate;
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
b.participant = {
|
||||
...b.participant,
|
||||
prepare: async () => {
|
||||
await bGate;
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
deps.registry.register(a.participant);
|
||||
deps.registry.register(b.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
const p1 = c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
const p2 = c.run2pc({ participants: [{ id: "b", input: {} }] });
|
||||
// Third call should hit the cap and reject synchronously.
|
||||
await expect(
|
||||
c.run2pc({ participants: [{ id: "a", input: {} }] })
|
||||
).rejects.toMatchObject({
|
||||
twoPcCode: TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED,
|
||||
});
|
||||
resolveA();
|
||||
resolveB();
|
||||
await Promise.all([p1, p2]);
|
||||
// After draining, a fresh call admits.
|
||||
const fresh = buildSpy("fresh");
|
||||
deps.registry.register(fresh.participant);
|
||||
const r = await c.run2pc({
|
||||
participants: [{ id: "fresh", input: {} }],
|
||||
});
|
||||
expect(r.state).toBe("committed");
|
||||
});
|
||||
|
||||
it("exposes outstandingCount and resolved options", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const deps = baseDeps();
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
expect(c.outstandingCount()).toBe(0);
|
||||
expect(c.options.hostInstanceId).toBe("h");
|
||||
expect(c.options.defaultTimeoutMs).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("preserves non-TwoPcError throws from a participant prepare callback as is", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a", { prepareThrows: new Error("custom") });
|
||||
deps.registry.register(a.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
// The prepare phase wraps the cause into TWOPC_PREPARE_FAILED.
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rethrows non-TwoPcError exceptions raised by commitPhaseTwo", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const a = buildSpy("a");
|
||||
// Force commit to throw a plain Error (not a TwoPcError) by
|
||||
// patching the connection's run to raise a non-TwoPcError on
|
||||
// the COMMIT PREPARED call but wrap that exception in a way
|
||||
// commitPhaseTwo doesn't recognise — we patch commitPhaseTwo's
|
||||
// invocation by replacing the participant's run AFTER prepare.
|
||||
deps.registry.register(a.participant);
|
||||
const original = a.participant.connection.run;
|
||||
let calls = 0;
|
||||
a.participant.connection.run = (async (sql: string) => {
|
||||
calls += 1;
|
||||
if (sql.startsWith("COMMIT PREPARED")) {
|
||||
// Throw something that bypasses TwoPcError detection
|
||||
// inside commitPhaseTwo: the phase wraps any thrown
|
||||
// value into TWOPC_COMMIT_FAILED, so the coordinator
|
||||
// never sees a non-TwoPcError. Confirm the wrapping
|
||||
// happens (so the else-branch in the coordinator's
|
||||
// catch is unreachable by design and the test asserts
|
||||
// that the wrapping path is the one taken).
|
||||
throw new RangeError("raw");
|
||||
}
|
||||
return original.call(a.participant.connection, sql);
|
||||
}) as typeof original;
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
try {
|
||||
await c.run2pc({ participants: [{ id: "a", input: {} }] });
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
// commitPhaseTwo wraps any thrown value into TWOPC_COMMIT_FAILED,
|
||||
// which the coordinator collects into commitErrors and escalates
|
||||
// to TWOPC_IN_DOUBT.
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
expect(calls).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes a read-only voter in the vote tally without sending SQL", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const deps = baseDeps();
|
||||
const writer = buildSpy("writer");
|
||||
const reader = buildSpy("reader", {
|
||||
voterMode: "read-only",
|
||||
writeIntent: false,
|
||||
});
|
||||
deps.registry.register(writer.participant);
|
||||
deps.registry.register(reader.participant);
|
||||
const c = createTwoPcCoordinator(deps);
|
||||
const outcome = await c.run2pc({
|
||||
participants: [
|
||||
{ id: "writer", input: {} },
|
||||
{ id: "reader", input: {} },
|
||||
],
|
||||
});
|
||||
expect(outcome.state).toBe("committed");
|
||||
expect(reader.calls).toEqual([]); // no SQL ever to the read-only voter
|
||||
});
|
||||
});
|
||||
281
packages/api-stack/api/api-2pc/src/coordinator/coordinator.ts
Normal file
281
packages/api-stack/api/api-2pc/src/coordinator/coordinator.ts
Normal file
@@ -0,0 +1,281 @@
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
import type { Clock, IdGenerator } from "@suaveplan/types";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import {
|
||||
createTwoPcInstruments,
|
||||
type TwoPcInstruments,
|
||||
} from "../instruments/instruments.js";
|
||||
import { OutstandingPreparedCap } from "../limits/outstanding-cap.js";
|
||||
import { assertTimeoutBelowCeiling } from "../limits/timeout-ceiling.js";
|
||||
import {
|
||||
type ResolvedTwoPcCoordinatorOptions,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
type TwoPcCoordinatorOptions,
|
||||
} from "../options/options.js";
|
||||
import type { LedgerDb } from "../persistence/persistence.js";
|
||||
import {
|
||||
type LedgerParticipantEntry,
|
||||
TwoPcLedgerStore,
|
||||
} from "../persistence/persistence.js";
|
||||
import {
|
||||
commitPhaseTwo,
|
||||
type ParticipantPrepareOutcome,
|
||||
preparePhaseOne,
|
||||
rollbackPhaseTwo,
|
||||
} from "../phases/index.js";
|
||||
import type { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
import type { Run2PcInput, Run2PcOutcome } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Adapter surface the coordinator depends on. Mirrors the foundation
|
||||
* pattern: dependencies passed in, not imported.
|
||||
*/
|
||||
export interface TwoPcCoordinatorDeps {
|
||||
readonly options: TwoPcCoordinatorOptions;
|
||||
readonly registry: TwoPcParticipantRegistry;
|
||||
readonly ledgerDb: LedgerDb;
|
||||
readonly clock: Clock;
|
||||
readonly idGenerator: IdGenerator;
|
||||
/**
|
||||
* Optional override; defaults to a fresh instance backed by the OTel
|
||||
* globals. Tests inject their own meter/tracer providers.
|
||||
*/
|
||||
readonly instruments?: TwoPcInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Public coordinator surface. `run2pc` is the only entry point;
|
||||
* `outstandingCount` and `getOptions` are exposed for diagnostics and
|
||||
* the OTel observable gauge.
|
||||
*/
|
||||
export interface TwoPcCoordinator {
|
||||
run2pc(input: Run2PcInput): Promise<Run2PcOutcome>;
|
||||
outstandingCount(): number;
|
||||
readonly options: ResolvedTwoPcCoordinatorOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory entry point. Builds a fully-wired coordinator from the
|
||||
* supplied dependencies. The returned object is the only surface
|
||||
* application code touches.
|
||||
*/
|
||||
export function createTwoPcCoordinator(
|
||||
deps: TwoPcCoordinatorDeps
|
||||
): TwoPcCoordinator {
|
||||
const options = resolveTwoPcCoordinatorOptions(deps.options);
|
||||
const ledger = new TwoPcLedgerStore(deps.ledgerDb);
|
||||
const cap = new OutstandingPreparedCap(options.outstandingPreparedCap);
|
||||
const instruments =
|
||||
deps.instruments ??
|
||||
createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => cap.snapshot(),
|
||||
});
|
||||
const { registry, clock, idGenerator } = deps;
|
||||
|
||||
async function run2pc(input: Run2PcInput): Promise<Run2PcOutcome> {
|
||||
const requestedTimeout = input.timeoutMs ?? options.defaultTimeoutMs;
|
||||
assertTimeoutBelowCeiling(requestedTimeout);
|
||||
cap.admit();
|
||||
const txId = idGenerator();
|
||||
const lookups = input.participants.map((p) => {
|
||||
const registered = registry.get(p.id);
|
||||
if (registered === undefined) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PREPARE_FAILED,
|
||||
`Unknown participant ${p.id}`,
|
||||
{ context: { participantId: p.id, txId } }
|
||||
);
|
||||
}
|
||||
return { spec: p, registered };
|
||||
});
|
||||
const participantEntries: LedgerParticipantEntry[] = lookups.map(
|
||||
({ registered }) => ({
|
||||
participantId: registered.id,
|
||||
voterMode: registered.voterMode,
|
||||
vote: null,
|
||||
})
|
||||
);
|
||||
|
||||
ledger.insert({
|
||||
id: txId,
|
||||
coordinator: options.hostInstanceId,
|
||||
participants: participantEntries,
|
||||
});
|
||||
|
||||
const txSpan = instruments.tracer.startSpan("2pc.transaction", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.count": lookups.length,
|
||||
"coordinator.host": options.hostInstanceId,
|
||||
},
|
||||
});
|
||||
const ac = new AbortController();
|
||||
const deadlineHandle = setTimeout(() => ac.abort(), requestedTimeout);
|
||||
|
||||
const prepareT0 = clock();
|
||||
const outcomes: ParticipantPrepareOutcome[] = [];
|
||||
let abortReason: TwoPcError | null = null;
|
||||
try {
|
||||
try {
|
||||
for (const { spec, registered } of lookups) {
|
||||
const o = await preparePhaseOne({
|
||||
hostInstanceId: options.hostInstanceId,
|
||||
txId,
|
||||
participant: registered,
|
||||
payload: spec.input,
|
||||
instruments,
|
||||
signal: ac.signal,
|
||||
});
|
||||
outcomes.push(o);
|
||||
if (o.vote === "abort") {
|
||||
abortReason = new TwoPcError(
|
||||
TWOPC_CODES.VOTE_ABORT,
|
||||
`Participant ${o.participantId} voted abort`,
|
||||
{
|
||||
context: {
|
||||
participantId: o.participantId,
|
||||
txId,
|
||||
},
|
||||
}
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) {
|
||||
abortReason = err;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
const prepareDuration = clock() - prepareT0;
|
||||
const commitT0 = clock();
|
||||
let outcomeState: Run2PcOutcome["state"];
|
||||
let commitDuration = 0;
|
||||
|
||||
if (
|
||||
abortReason === null &&
|
||||
outcomes.every((o) => o.vote === "commit")
|
||||
) {
|
||||
ledger.markAllVotedCommit({
|
||||
id: txId,
|
||||
participants: participantEntries.map((e) => {
|
||||
const matched = outcomes.find(
|
||||
(o) => o.participantId === e.participantId
|
||||
);
|
||||
return {
|
||||
...e,
|
||||
vote: matched?.vote ?? null,
|
||||
};
|
||||
}),
|
||||
preparedAt: new Date(clock()).toISOString(),
|
||||
});
|
||||
const commitErrors: TwoPcError[] = [];
|
||||
for (const o of outcomes) {
|
||||
try {
|
||||
await commitPhaseTwo({
|
||||
txId,
|
||||
participant: registry.get(o.participantId)!,
|
||||
outcome: o,
|
||||
instruments,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) commitErrors.push(err);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
commitDuration = clock() - commitT0;
|
||||
if (commitErrors.length > 0) {
|
||||
ledger.markInDoubt(txId);
|
||||
outcomeState = "in_doubt";
|
||||
instruments.transactionsTotal.add(1, {
|
||||
outcome: "in_doubt",
|
||||
});
|
||||
instruments.commitDurationMs.record(commitDuration, {
|
||||
outcome: "in_doubt",
|
||||
});
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.IN_DOUBT,
|
||||
`Transaction ${txId} is in doubt (${commitErrors.length} commit failure(s))`,
|
||||
{
|
||||
cause: commitErrors[0],
|
||||
context: { txId, failed: commitErrors.length },
|
||||
}
|
||||
);
|
||||
}
|
||||
ledger.markCommitted(txId, new Date(clock()).toISOString());
|
||||
outcomeState = "committed";
|
||||
instruments.transactionsTotal.add(1, { outcome: "committed" });
|
||||
instruments.commitDurationMs.record(commitDuration, {
|
||||
outcome: "committed",
|
||||
});
|
||||
} else {
|
||||
// Rollback path. Abort the deadline so any pending prepares
|
||||
// hit the TIMEOUT path on their next SQL call.
|
||||
ac.abort();
|
||||
ledger.markAborting({
|
||||
id: txId,
|
||||
participants: participantEntries.map((e) => {
|
||||
const matched = outcomes.find(
|
||||
(o) => o.participantId === e.participantId
|
||||
);
|
||||
return { ...e, vote: matched?.vote ?? null };
|
||||
}),
|
||||
});
|
||||
const rollbackErrors: TwoPcError[] = [];
|
||||
for (const o of outcomes) {
|
||||
try {
|
||||
await rollbackPhaseTwo({
|
||||
txId,
|
||||
participant: registry.get(o.participantId)!,
|
||||
outcome: o,
|
||||
instruments,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) rollbackErrors.push(err);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
ledger.markAborted(txId, new Date(clock()).toISOString());
|
||||
outcomeState = "aborted";
|
||||
instruments.transactionsTotal.add(1, { outcome: "aborted" });
|
||||
if (rollbackErrors.length > 0) {
|
||||
ledger.markInDoubt(txId);
|
||||
instruments.transactionsTotal.add(1, {
|
||||
outcome: "in_doubt",
|
||||
});
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.IN_DOUBT,
|
||||
`Rollback partially failed for ${txId}`,
|
||||
{
|
||||
cause: rollbackErrors[0],
|
||||
context: { txId },
|
||||
}
|
||||
);
|
||||
}
|
||||
if (abortReason !== null) throw abortReason;
|
||||
}
|
||||
|
||||
return {
|
||||
txId,
|
||||
state: outcomeState,
|
||||
votes: outcomes.map((o) => ({
|
||||
participantId: o.participantId,
|
||||
vote: o.vote as TwoPcVote,
|
||||
})),
|
||||
prepareDurationMs: prepareDuration,
|
||||
commitDurationMs: commitDuration,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(deadlineHandle);
|
||||
txSpan.end();
|
||||
cap.release();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
run2pc,
|
||||
outstandingCount: () => cap.snapshot(),
|
||||
options,
|
||||
};
|
||||
}
|
||||
5
packages/api-stack/api/api-2pc/src/coordinator/index.ts
Normal file
5
packages/api-stack/api/api-2pc/src/coordinator/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
createTwoPcCoordinator,
|
||||
type TwoPcCoordinator,
|
||||
type TwoPcCoordinatorDeps,
|
||||
} from "./coordinator.js";
|
||||
@@ -0,0 +1,97 @@
|
||||
# `error-codes/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
This module owns the `TWOPC_*` error-code namespace and the `TwoPcError`
|
||||
concrete class for the `@suaveplan/api-2pc` package. The registry is the
|
||||
canonical contract every production code path in the coordinator throws
|
||||
against — observability dashboards, support runbooks, and integrating
|
||||
plugins all join on these stable string codes.
|
||||
|
||||
The codes are intentionally enumerated up-front rather than grown
|
||||
ad-hoc: a missing code is a spec deviation that a CI gate can detect,
|
||||
and the closed set lets operators write alerting rules without
|
||||
constantly chasing schema drift.
|
||||
|
||||
## Features
|
||||
|
||||
- Ten stable codes (`TIMEOUT`, `VOTE_ABORT`, `PREPARE_FAILED`,
|
||||
`COMMIT_FAILED`, `ROLLBACK_FAILED`, `IN_DOUBT`,
|
||||
`PARTICIPANT_NOT_PREPARABLE`, `READ_ONLY_PARTICIPANT_CANNOT_WRITE`,
|
||||
`OUTSTANDING_LIMIT_EXCEEDED`, `TIMEOUT_CEILING_EXCEEDED`).
|
||||
- Each code is exposed via `TWOPC_CODES.<NAME>` returning the prefixed
|
||||
string `"TWOPC.<NAME>"` produced by `defineCodes`.
|
||||
- `TwoPcError` extends `SuaveplanError` so cause-chain traversal,
|
||||
Zod-validated wire format, and the OTel span-event adapter from
|
||||
`@suaveplan/error` all apply unchanged.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT,
|
||||
"Phase-1 deadline elapsed",
|
||||
{ context: { txId, timeoutMs: 5000 } }
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Branching on error code at the boundary between coordinator and caller
|
||||
is the recommended consumer pattern. The narrowed `twoPcCode` field
|
||||
gives an exhaustive switch the compiler can check:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
|
||||
try {
|
||||
await run2pc({ participants, timeoutMs: 5000 }, ctx);
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) {
|
||||
switch (err.twoPcCode) {
|
||||
case TWOPC_CODES.TIMEOUT:
|
||||
metrics.incr("billing.2pc.timeout");
|
||||
return retryWithBackoff();
|
||||
case TWOPC_CODES.VOTE_ABORT:
|
||||
return surfaceUserError();
|
||||
case TWOPC_CODES.IN_DOUBT:
|
||||
pageOncall(err);
|
||||
return;
|
||||
default:
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `TWOPC_CODES` — frozen object `{ TIMEOUT: "TWOPC.TIMEOUT", ... }`.
|
||||
- `type TwoPcErrorCode` — string-literal union of every value in
|
||||
`TWOPC_CODES`.
|
||||
- `class TwoPcError extends SuaveplanError<TwoPcErrorCode>` — exposes
|
||||
`twoPcCode` as a same-typed accessor for ergonomic narrowing without
|
||||
re-asserting at every catch site.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`defineCodes` from `@suaveplan/error` produces the prefixed values at
|
||||
runtime, so every code string starts with `"TWOPC."`. This matches the
|
||||
prefix convention used by sibling api-stack packages (`HOST.*`,
|
||||
`PLUGIN.*`, `RELOAD.*`, `LEDGER.*`). The `TwoPcError` class carries a
|
||||
redundant `twoPcCode` field — duplicating `code` — purely to give
|
||||
callers a name they don't have to import the generic
|
||||
`SuaveplanError<C>` type to narrow against. The duplication is
|
||||
deliberate and harmless because the base class makes `code` readonly.
|
||||
|
||||
The registry is closed; adding a code requires an OpenSpec change so
|
||||
every consumer dashboard and runbook stays in sync. Coverage assertions
|
||||
ensure every code is actually thrown by a production path before this
|
||||
package can graduate to `1.0.0`.
|
||||
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, expectTypeOf, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError, type TwoPcErrorCode } from "./error-codes.js";
|
||||
|
||||
describe("TWOPC_CODES", () => {
|
||||
it("exposes ten prefixed string codes", { timeout: 5000 }, () => {
|
||||
const values = Object.values(TWOPC_CODES);
|
||||
expect(values).toHaveLength(10);
|
||||
for (const v of values) {
|
||||
expect(v.startsWith("TWOPC.")).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("includes every required code identifier", { timeout: 5000 }, () => {
|
||||
expect(TWOPC_CODES.TIMEOUT).toBe("TWOPC.TIMEOUT");
|
||||
expect(TWOPC_CODES.VOTE_ABORT).toBe("TWOPC.VOTE_ABORT");
|
||||
expect(TWOPC_CODES.PREPARE_FAILED).toBe("TWOPC.PREPARE_FAILED");
|
||||
expect(TWOPC_CODES.COMMIT_FAILED).toBe("TWOPC.COMMIT_FAILED");
|
||||
expect(TWOPC_CODES.ROLLBACK_FAILED).toBe("TWOPC.ROLLBACK_FAILED");
|
||||
expect(TWOPC_CODES.IN_DOUBT).toBe("TWOPC.IN_DOUBT");
|
||||
expect(TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE).toBe(
|
||||
"TWOPC.PARTICIPANT_NOT_PREPARABLE"
|
||||
);
|
||||
expect(TWOPC_CODES.READ_ONLY_PARTICIPANT_CANNOT_WRITE).toBe(
|
||||
"TWOPC.READ_ONLY_PARTICIPANT_CANNOT_WRITE"
|
||||
);
|
||||
expect(TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED).toBe(
|
||||
"TWOPC.OUTSTANDING_LIMIT_EXCEEDED"
|
||||
);
|
||||
expect(TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED).toBe(
|
||||
"TWOPC.TIMEOUT_CEILING_EXCEEDED"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("TwoPcError", () => {
|
||||
it("carries the supplied code on both code and twoPcCode", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = new TwoPcError(TWOPC_CODES.TIMEOUT, "deadline");
|
||||
expect(err.code).toBe(TWOPC_CODES.TIMEOUT);
|
||||
expect(err.twoPcCode).toBe(TWOPC_CODES.TIMEOUT);
|
||||
expect(err.message).toBe("deadline");
|
||||
});
|
||||
|
||||
it("preserves the cause and context options", { timeout: 5000 }, () => {
|
||||
const cause = new Error("inner");
|
||||
const err = new TwoPcError(TWOPC_CODES.IN_DOUBT, "partial commit", {
|
||||
cause,
|
||||
context: { txId: "tx-1" },
|
||||
});
|
||||
expect(err.cause).toBe(cause);
|
||||
expect(err.context).toEqual({ txId: "tx-1" });
|
||||
});
|
||||
|
||||
it("narrows the code type to TwoPcErrorCode", { timeout: 5000 }, () => {
|
||||
const err = new TwoPcError(TWOPC_CODES.VOTE_ABORT, "abort");
|
||||
expectTypeOf(err.twoPcCode).toEqualTypeOf<TwoPcErrorCode>();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import type { SuaveplanErrorOptions } from "@suaveplan/error";
|
||||
import { defineCodes, SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
/**
|
||||
* Stable error-code registry for `@suaveplan/api-2pc`. The ten codes
|
||||
* cover every production failure mode the coordinator surfaces:
|
||||
*
|
||||
* - `TWOPC_TIMEOUT` — the prepare-phase deadline elapsed before all
|
||||
* participants returned a vote. The coordinator rolls back every
|
||||
* participant that already voted commit.
|
||||
* - `TWOPC_VOTE_ABORT` — at least one participant voted abort; the
|
||||
* coordinator rolled back every commit-voter.
|
||||
* - `TWOPC_PREPARE_FAILED` — a participant raised during phase 1
|
||||
* (network drop, SQL error, capability check failure). Treated as an
|
||||
* implicit abort vote: rollback runs everywhere a prepare succeeded.
|
||||
* - `TWOPC_COMMIT_FAILED` — at least one `COMMIT PREPARED` raised. The
|
||||
* transaction is `in_doubt`; the recovery runner is the only path
|
||||
* that can resolve it.
|
||||
* - `TWOPC_ROLLBACK_FAILED` — at least one `ROLLBACK PREPARED` raised.
|
||||
* Recovery resolves it on next boot.
|
||||
* - `TWOPC_IN_DOUBT` — terminal state surfacing a partially-committed
|
||||
* transaction whose follow-up requires the recovery runner.
|
||||
* - `TWOPC_PARTICIPANT_NOT_PREPARABLE` — install-time refusal of a
|
||||
* write participant whose engine does not advertise `db.preparable`.
|
||||
* SQLite / SurrealDB SHALL hit this when registered as writers.
|
||||
* - `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE` — a read-only voter's
|
||||
* prepare callback returned a write-intent payload; the coordinator
|
||||
* refused the transaction before phase 2.
|
||||
* - `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` — admission control refused a
|
||||
* new `run2pc` call because the host has 256 prepared transactions
|
||||
* already in flight. The cap is operator-tunable but the default
|
||||
* ships at 256 per design doc §14.
|
||||
* - `TWOPC_TIMEOUT_CEILING_EXCEEDED` — caller-supplied `timeoutMs`
|
||||
* exceeded the 30 000 ms hard ceiling. The call is refused at the
|
||||
* boundary (no participant is contacted).
|
||||
*
|
||||
* Consumers branch on the string value:
|
||||
*
|
||||
* ```ts
|
||||
* if (error.code === TWOPC_CODES.TIMEOUT) { ... }
|
||||
* ```
|
||||
*/
|
||||
export const TWOPC_CODES = defineCodes("TWOPC", [
|
||||
"TIMEOUT",
|
||||
"VOTE_ABORT",
|
||||
"PREPARE_FAILED",
|
||||
"COMMIT_FAILED",
|
||||
"ROLLBACK_FAILED",
|
||||
"IN_DOUBT",
|
||||
"PARTICIPANT_NOT_PREPARABLE",
|
||||
"READ_ONLY_PARTICIPANT_CANNOT_WRITE",
|
||||
"OUTSTANDING_LIMIT_EXCEEDED",
|
||||
"TIMEOUT_CEILING_EXCEEDED",
|
||||
] as const);
|
||||
|
||||
export type TwoPcErrorCode = (typeof TWOPC_CODES)[keyof typeof TWOPC_CODES];
|
||||
|
||||
/**
|
||||
* Concrete error class for the `api-2pc` package. Extends
|
||||
* {@link SuaveplanError} so `instanceof TwoPcError` narrows
|
||||
* `code` to {@link TwoPcErrorCode} at the type level.
|
||||
*/
|
||||
export class TwoPcError extends SuaveplanError<TwoPcErrorCode> {
|
||||
public readonly twoPcCode: TwoPcErrorCode;
|
||||
|
||||
public constructor(
|
||||
code: TwoPcErrorCode,
|
||||
message: string,
|
||||
options?: SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.twoPcCode = code;
|
||||
}
|
||||
}
|
||||
5
packages/api-stack/api/api-2pc/src/error-codes/index.ts
Normal file
5
packages/api-stack/api/api-2pc/src/error-codes/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
TWOPC_CODES,
|
||||
TwoPcError,
|
||||
type TwoPcErrorCode,
|
||||
} from "./error-codes.js";
|
||||
68
packages/api-stack/api/api-2pc/src/index.ts
Normal file
68
packages/api-stack/api/api-2pc/src/index.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
export {
|
||||
createTwoPcCoordinator,
|
||||
type TwoPcCoordinator,
|
||||
type TwoPcCoordinatorDeps,
|
||||
} from "./coordinator/coordinator.js";
|
||||
export {
|
||||
TWOPC_CODES,
|
||||
TwoPcError,
|
||||
type TwoPcErrorCode,
|
||||
} from "./error-codes/error-codes.js";
|
||||
export {
|
||||
createTwoPcInstruments,
|
||||
type TwoPcInstruments,
|
||||
type TwoPcInstrumentsConfig,
|
||||
type TwoPcOutcome,
|
||||
} from "./instruments/instruments.js";
|
||||
export {
|
||||
assertTimeoutBelowCeiling,
|
||||
OutstandingPreparedCap,
|
||||
} from "./limits/index.js";
|
||||
export {
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
type ResolvedTwoPcCoordinatorOptions,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
TIMEOUT_CEILING_MS,
|
||||
type TwoPcCoordinatorOptions,
|
||||
twoPcCoordinatorOptionsSchema,
|
||||
} from "./options/options.js";
|
||||
export {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
type LedgerStatement,
|
||||
type TwoPcLedgerRow,
|
||||
TwoPcLedgerStore,
|
||||
} from "./persistence/persistence.js";
|
||||
export {
|
||||
buildGid,
|
||||
type CommitPhaseInput,
|
||||
commitPhaseTwo,
|
||||
gidPrefixFor,
|
||||
isUnknownPreparedTransaction,
|
||||
type ParticipantPrepareOutcome,
|
||||
type PreparePhaseInput,
|
||||
parseGid,
|
||||
preparePhaseOne,
|
||||
type RollbackPhaseInput,
|
||||
rollbackPhaseTwo,
|
||||
} from "./phases/index.js";
|
||||
export {
|
||||
buildProbeFromRegistry,
|
||||
type RecoveryInput,
|
||||
type RecoveryProbe,
|
||||
type RecoveryReport,
|
||||
resolveRecovery,
|
||||
} from "./recovery/recovery.js";
|
||||
export {
|
||||
type PreparableCapabilityCheck,
|
||||
TwoPcParticipantRegistry,
|
||||
} from "./registry/participant-registry.js";
|
||||
export type {
|
||||
PreparableConnection,
|
||||
RegisteredTwoPcParticipant,
|
||||
Run2PcInput,
|
||||
Run2PcOutcome,
|
||||
TwoPcCoordinatorState,
|
||||
TwoPcPrepareResult,
|
||||
} from "./types/types.types.js";
|
||||
6
packages/api-stack/api/api-2pc/src/instruments/index.ts
Normal file
6
packages/api-stack/api/api-2pc/src/instruments/index.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
createTwoPcInstruments,
|
||||
type TwoPcInstruments,
|
||||
type TwoPcInstrumentsConfig,
|
||||
type TwoPcOutcome,
|
||||
} from "./instruments.js";
|
||||
105
packages/api-stack/api/api-2pc/src/instruments/instruments.md
Normal file
105
packages/api-stack/api/api-2pc/src/instruments/instruments.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# `instruments/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Builds the OTel instrument registry the 2PC coordinator emits against
|
||||
on every transaction. Each instrument's name follows design doc §9.2
|
||||
exactly so a missing entry is a spec deviation a CI gate could detect.
|
||||
Centralising creation keeps the coordinator code free of meter / tracer
|
||||
plumbing — every phase module simply records against the frozen
|
||||
bundle returned here.
|
||||
|
||||
The observable gauge wires into the coordinator's outstanding-prepared
|
||||
counter via a snapshot callback, so dashboards see the live admission
|
||||
state without the coordinator pushing on every change.
|
||||
|
||||
## Features
|
||||
|
||||
- One span: `2pc.transaction` per transaction lifetime; per-phase
|
||||
child spans `2pc.prepare`, `2pc.commit`, `2pc.rollback` (started by
|
||||
the phase modules themselves, using the tracer this bundle exposes).
|
||||
- One counter: `2pc.transactions_total` labelled `{ outcome }`.
|
||||
- Two histograms: `2pc.prepare_duration_ms`, `2pc.commit_duration_ms`
|
||||
labelled `{ outcome }`.
|
||||
- One observable gauge: `2pc.outstanding_prepared` wired to a
|
||||
snapshot callback supplied at construction.
|
||||
- Provider defaults to the OTel globals (no-op when no SDK is
|
||||
installed) so unit tests do not have to wire a provider.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { createTwoPcInstruments } from "@suaveplan/api-2pc/instruments";
|
||||
|
||||
const instruments = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => coordinator.outstandingCount(),
|
||||
});
|
||||
|
||||
instruments.transactionsTotal.add(1, { outcome: "committed" });
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Inject a real OTel SDK and a custom snapshot to feed a live dashboard:
|
||||
|
||||
```ts
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
import { MeterProvider } from "@opentelemetry/sdk-metrics";
|
||||
import { createTwoPcInstruments } from "@suaveplan/api-2pc/instruments";
|
||||
|
||||
const meterProvider = new MeterProvider({ /* readers/exporters */ });
|
||||
metrics.setGlobalMeterProvider(meterProvider);
|
||||
|
||||
const instruments = createTwoPcInstruments({
|
||||
meterProvider,
|
||||
tracerProvider: trace.getTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => coordinator.outstandingCount(),
|
||||
});
|
||||
|
||||
const span = instruments.tracer.startSpan("2pc.transaction", {
|
||||
attributes: { "tx.id": txId, participant_count: 2 },
|
||||
});
|
||||
try {
|
||||
// ...
|
||||
instruments.prepareDurationMs.record(prepareElapsedMs, {
|
||||
outcome: "committed",
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type TwoPcOutcome = "committed" | "aborted" | "in_doubt"` — values
|
||||
attached to the `outcome` attribute on every counter / histogram
|
||||
observation.
|
||||
- `interface TwoPcInstrumentsConfig`
|
||||
- `meterProvider?: MeterProvider`
|
||||
- `tracerProvider?: TracerProvider`
|
||||
- `outstandingPreparedSnapshot: () => number`
|
||||
- `interface TwoPcInstruments` — frozen bundle exposing `meter`,
|
||||
`tracer`, `transactionsTotal`, `prepareDurationMs`,
|
||||
`commitDurationMs`, `outstandingPreparedGauge`.
|
||||
- `createTwoPcInstruments(config): TwoPcInstruments` — factory.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The observable gauge's callback fires whenever the SDK collects metrics
|
||||
(typically on a fixed interval driven by the exporter). The snapshot
|
||||
returned by `outstandingPreparedSnapshot` is therefore an
|
||||
eventually-consistent value — operators must not treat it as a strict
|
||||
upper bound for admission decisions. The strict upper bound is enforced
|
||||
inside `limits/outstanding-cap.ts`; the gauge is purely a dashboard
|
||||
signal.
|
||||
|
||||
Histogram unit is set explicitly to `"ms"` per OTel naming convention.
|
||||
The two histograms are named after the phases they measure so a single
|
||||
dashboard panel can render side-by-side latency for prepare and commit
|
||||
without re-labelling — operators flagging long prepares typically
|
||||
care more about phase-1 latency because that is when row-level locks
|
||||
are held on every participant.
|
||||
@@ -0,0 +1,218 @@
|
||||
import type {
|
||||
BatchObservableCallback,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
ObservableCallback,
|
||||
ObservableGauge,
|
||||
Tracer,
|
||||
TracerProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { createTwoPcInstruments } from "./instruments.js";
|
||||
|
||||
interface FakeInstrument {
|
||||
readonly kind: "counter" | "histogram" | "gauge";
|
||||
readonly name: string;
|
||||
readonly options: Readonly<Record<string, unknown>>;
|
||||
addCalls: Array<{
|
||||
value: number;
|
||||
attrs?: Readonly<Record<string, unknown>>;
|
||||
}>;
|
||||
recordCalls: Array<{
|
||||
value: number;
|
||||
attrs?: Readonly<Record<string, unknown>>;
|
||||
}>;
|
||||
gaugeCallbacks: ObservableCallback[];
|
||||
}
|
||||
|
||||
interface CapturedMeter {
|
||||
readonly name: string;
|
||||
readonly instruments: FakeInstrument[];
|
||||
}
|
||||
|
||||
function buildFakeMeterProvider(): {
|
||||
provider: MeterProvider;
|
||||
meters: CapturedMeter[];
|
||||
} {
|
||||
const meters: CapturedMeter[] = [];
|
||||
const provider: MeterProvider = {
|
||||
getMeter(name: string): Meter {
|
||||
const captured: CapturedMeter = { name, instruments: [] };
|
||||
meters.push(captured);
|
||||
const meter: Meter = {
|
||||
createCounter(n, opts) {
|
||||
const inst: FakeInstrument = {
|
||||
kind: "counter",
|
||||
name: n,
|
||||
options: opts ?? {},
|
||||
addCalls: [],
|
||||
recordCalls: [],
|
||||
gaugeCallbacks: [],
|
||||
};
|
||||
captured.instruments.push(inst);
|
||||
return {
|
||||
add(v: number, a?: Readonly<Record<string, unknown>>) {
|
||||
inst.addCalls.push({ value: v, attrs: a });
|
||||
},
|
||||
} as ReturnType<Meter["createCounter"]>;
|
||||
},
|
||||
createHistogram(n, opts) {
|
||||
const inst: FakeInstrument = {
|
||||
kind: "histogram",
|
||||
name: n,
|
||||
options: opts ?? {},
|
||||
addCalls: [],
|
||||
recordCalls: [],
|
||||
gaugeCallbacks: [],
|
||||
};
|
||||
captured.instruments.push(inst);
|
||||
return {
|
||||
record(
|
||||
v: number,
|
||||
a?: Readonly<Record<string, unknown>>
|
||||
) {
|
||||
inst.recordCalls.push({ value: v, attrs: a });
|
||||
},
|
||||
} as ReturnType<Meter["createHistogram"]>;
|
||||
},
|
||||
createObservableGauge(n, opts) {
|
||||
const inst: FakeInstrument = {
|
||||
kind: "gauge",
|
||||
name: n,
|
||||
options: opts ?? {},
|
||||
addCalls: [],
|
||||
recordCalls: [],
|
||||
gaugeCallbacks: [],
|
||||
};
|
||||
captured.instruments.push(inst);
|
||||
const gauge: ObservableGauge = {
|
||||
addCallback(cb: ObservableCallback) {
|
||||
inst.gaugeCallbacks.push(cb);
|
||||
},
|
||||
removeCallback(_cb: ObservableCallback) {
|
||||
// no-op for tests
|
||||
},
|
||||
} as unknown as ObservableGauge;
|
||||
return gauge;
|
||||
},
|
||||
addBatchObservableCallback(_cb: BatchObservableCallback) {
|
||||
// no-op
|
||||
},
|
||||
removeBatchObservableCallback(_cb: BatchObservableCallback) {
|
||||
// no-op
|
||||
},
|
||||
} as unknown as Meter;
|
||||
return meter;
|
||||
},
|
||||
} as MeterProvider;
|
||||
return { provider, meters };
|
||||
}
|
||||
|
||||
function buildFakeTracerProvider(): TracerProvider {
|
||||
return {
|
||||
getTracer(_name: string): Tracer {
|
||||
return {
|
||||
startSpan() {
|
||||
return {
|
||||
setAttribute: () => undefined,
|
||||
setStatus: () => undefined,
|
||||
recordException: () => undefined,
|
||||
addEvent: () => undefined,
|
||||
end: () => undefined,
|
||||
spanContext: () => ({
|
||||
traceId: "0",
|
||||
spanId: "0",
|
||||
traceFlags: 0,
|
||||
}),
|
||||
isRecording: () => false,
|
||||
updateName: () => undefined,
|
||||
};
|
||||
},
|
||||
startActiveSpan: undefined as never,
|
||||
} as unknown as Tracer;
|
||||
},
|
||||
} as TracerProvider;
|
||||
}
|
||||
|
||||
describe("createTwoPcInstruments", () => {
|
||||
it("creates every documented instrument on the package meter", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const { provider, meters } = buildFakeMeterProvider();
|
||||
const instruments = createTwoPcInstruments({
|
||||
meterProvider: provider,
|
||||
tracerProvider: buildFakeTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => 7,
|
||||
});
|
||||
expect(meters[0]?.name).toBe("@suaveplan/api-2pc");
|
||||
const names = meters[0]?.instruments.map((i) => i.name);
|
||||
expect(names).toEqual([
|
||||
"2pc.transactions_total",
|
||||
"2pc.prepare_duration_ms",
|
||||
"2pc.commit_duration_ms",
|
||||
"2pc.outstanding_prepared",
|
||||
]);
|
||||
expect(instruments.transactionsTotal).toBeDefined();
|
||||
expect(instruments.prepareDurationMs).toBeDefined();
|
||||
expect(instruments.commitDurationMs).toBeDefined();
|
||||
expect(instruments.outstandingPreparedGauge).toBeDefined();
|
||||
});
|
||||
|
||||
it("uses ms units on the duration histograms", { timeout: 5000 }, () => {
|
||||
const { provider, meters } = buildFakeMeterProvider();
|
||||
createTwoPcInstruments({
|
||||
meterProvider: provider,
|
||||
tracerProvider: buildFakeTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
const histograms = meters[0]?.instruments.filter(
|
||||
(i) => i.kind === "histogram"
|
||||
);
|
||||
const byName = new Map(histograms?.map((h) => [h.name, h]));
|
||||
expect(byName.get("2pc.prepare_duration_ms")?.options).toEqual({
|
||||
unit: "ms",
|
||||
});
|
||||
expect(byName.get("2pc.commit_duration_ms")?.options).toEqual({
|
||||
unit: "ms",
|
||||
});
|
||||
});
|
||||
|
||||
it("the observable gauge callback reports the snapshot value", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const { provider, meters } = buildFakeMeterProvider();
|
||||
let snapshot = 3;
|
||||
createTwoPcInstruments({
|
||||
meterProvider: provider,
|
||||
tracerProvider: buildFakeTracerProvider(),
|
||||
outstandingPreparedSnapshot: () => snapshot,
|
||||
});
|
||||
const gauge = meters[0]?.instruments.find((i) => i.kind === "gauge");
|
||||
const observations: number[] = [];
|
||||
const observer = {
|
||||
observe(value: number) {
|
||||
observations.push(value);
|
||||
},
|
||||
};
|
||||
for (const cb of gauge?.gaugeCallbacks ?? []) {
|
||||
cb(observer as Parameters<ObservableCallback>[0]);
|
||||
}
|
||||
snapshot = 9;
|
||||
for (const cb of gauge?.gaugeCallbacks ?? []) {
|
||||
cb(observer as Parameters<ObservableCallback>[0]);
|
||||
}
|
||||
expect(observations).toEqual([3, 9]);
|
||||
});
|
||||
|
||||
it("falls back to OTel globals when no provider supplied", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// The OTel API returns a no-op provider when no SDK is installed;
|
||||
// this exercises the default-branch wiring.
|
||||
const instruments = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
expect(instruments.transactionsTotal).toBeDefined();
|
||||
expect(instruments.tracer).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,89 @@
|
||||
import type {
|
||||
Counter,
|
||||
Histogram,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
ObservableGauge,
|
||||
Tracer,
|
||||
TracerProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
|
||||
/** Outcome attribute attached to every counter/histogram observation. */
|
||||
export type TwoPcOutcome = "committed" | "aborted" | "in_doubt";
|
||||
|
||||
/**
|
||||
* Config consumed by {@link createTwoPcInstruments}. The two
|
||||
* providers default to the OTel globals (which fall back to no-op
|
||||
* implementations when no SDK is installed) so the coordinator works
|
||||
* in test envs without ceremony.
|
||||
*/
|
||||
export interface TwoPcInstrumentsConfig {
|
||||
readonly meterProvider?: MeterProvider;
|
||||
readonly tracerProvider?: TracerProvider;
|
||||
/** Callback returning the current outstanding-prepared count for the gauge. */
|
||||
readonly outstandingPreparedSnapshot: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Frozen bundle of every OTel instrument the coordinator emits. The
|
||||
* names follow design doc §9.2:
|
||||
*
|
||||
* - `2pc.transaction` — span per transaction lifetime.
|
||||
* - `2pc.prepare` / `2pc.commit` / `2pc.rollback` — spans per
|
||||
* participant phase.
|
||||
* - `2pc.transactions_total` — counter labelled `{ outcome }`.
|
||||
* - `2pc.prepare_duration_ms` / `2pc.commit_duration_ms` — histograms
|
||||
* labelled `{ outcome }`.
|
||||
* - `2pc.outstanding_prepared` — observable gauge.
|
||||
*/
|
||||
export interface TwoPcInstruments {
|
||||
readonly meter: Meter;
|
||||
readonly tracer: Tracer;
|
||||
readonly transactionsTotal: Counter;
|
||||
readonly prepareDurationMs: Histogram;
|
||||
readonly commitDurationMs: Histogram;
|
||||
readonly outstandingPreparedGauge: ObservableGauge;
|
||||
}
|
||||
|
||||
const METER_NAME = "@suaveplan/api-2pc";
|
||||
|
||||
/**
|
||||
* Build the coordinator's instrument registry. Returns a frozen bundle
|
||||
* — every counter / histogram / gauge is constructed once so the
|
||||
* first measurement has zero allocation cost.
|
||||
*
|
||||
* The observable gauge is wired to `config.outstandingPreparedSnapshot`
|
||||
* at construction time. Operators who want a live admission-cap
|
||||
* dashboard register their SDK once at process boot and the
|
||||
* coordinator picks it up via the global provider.
|
||||
*/
|
||||
export function createTwoPcInstruments(
|
||||
config: TwoPcInstrumentsConfig
|
||||
): TwoPcInstruments {
|
||||
const meterProvider = config.meterProvider ?? metrics.getMeterProvider();
|
||||
const tracerProvider = config.tracerProvider ?? trace.getTracerProvider();
|
||||
const meter = meterProvider.getMeter(METER_NAME);
|
||||
const tracer = tracerProvider.getTracer(METER_NAME);
|
||||
const transactionsTotal = meter.createCounter("2pc.transactions_total");
|
||||
const prepareDurationMs = meter.createHistogram("2pc.prepare_duration_ms", {
|
||||
unit: "ms",
|
||||
});
|
||||
const commitDurationMs = meter.createHistogram("2pc.commit_duration_ms", {
|
||||
unit: "ms",
|
||||
});
|
||||
const outstandingPreparedGauge = meter.createObservableGauge(
|
||||
"2pc.outstanding_prepared"
|
||||
);
|
||||
outstandingPreparedGauge.addCallback((observer) => {
|
||||
observer.observe(config.outstandingPreparedSnapshot());
|
||||
});
|
||||
return Object.freeze({
|
||||
meter,
|
||||
tracer,
|
||||
transactionsTotal,
|
||||
prepareDurationMs,
|
||||
commitDurationMs,
|
||||
outstandingPreparedGauge,
|
||||
});
|
||||
}
|
||||
2
packages/api-stack/api/api-2pc/src/limits/index.ts
Normal file
2
packages/api-stack/api/api-2pc/src/limits/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export { OutstandingPreparedCap } from "./outstanding-cap.js";
|
||||
export { assertTimeoutBelowCeiling } from "./timeout-ceiling.js";
|
||||
104
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.md
Normal file
104
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# `limits/outstanding-cap`
|
||||
|
||||
## Purpose
|
||||
|
||||
Admission control for in-flight 2PC transactions. The coordinator
|
||||
holds a fixed-size budget of "outstanding prepared transactions"; once
|
||||
the budget is exhausted new `run2pc` calls refuse rather than queue.
|
||||
The default is 256 (design doc §14's open question, pinned by this
|
||||
package's spec) and operators can tune it via host config.
|
||||
|
||||
Refusing rather than queuing is intentional: Postgres holds row-level
|
||||
locks for every prepared transaction, so a backed-up coordinator
|
||||
would otherwise convert a transient overload into a global lock
|
||||
storm. Failing fast lets callers route to a degraded path
|
||||
(degraded-write, retry queue, etc.).
|
||||
|
||||
## Features
|
||||
|
||||
- `OutstandingPreparedCap` — synchronous counter with `admit()`,
|
||||
`release()`, `snapshot()`, `capacity()`.
|
||||
- Constructor validates that the cap is a positive integer (rejects
|
||||
zero, negatives, fractions, `NaN`).
|
||||
- `release()` is idempotent below zero — a buggy double-release in
|
||||
caller code cannot silently expand the real cap.
|
||||
- Synchronous-only — relies on the event-loop atomicity guarantee
|
||||
between `admit` / `release` pairs in JavaScript runtimes.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { OutstandingPreparedCap } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
const cap = new OutstandingPreparedCap(256);
|
||||
|
||||
cap.admit();
|
||||
try {
|
||||
// run the 2PC transaction
|
||||
} finally {
|
||||
cap.release();
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Wrap admission inside the coordinator's public entry to centralise the
|
||||
release in one `finally`. The snapshot fuels the OTel gauge:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
import { OutstandingPreparedCap } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
class MiniCoordinator {
|
||||
private readonly cap = new OutstandingPreparedCap(256);
|
||||
|
||||
snapshot() {
|
||||
return this.cap.snapshot();
|
||||
}
|
||||
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
this.cap.admit();
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof TwoPcError &&
|
||||
err.twoPcCode === TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED
|
||||
) {
|
||||
// route to degraded path
|
||||
throw err;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
this.cap.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `class OutstandingPreparedCap`
|
||||
- `constructor(cap: number)` — throws
|
||||
`TWOPC_OUTSTANDING_LIMIT_EXCEEDED` on invalid `cap`.
|
||||
- `admit(): void` — throws
|
||||
`TWOPC_OUTSTANDING_LIMIT_EXCEEDED` when at capacity.
|
||||
- `release(): void` — idempotent below zero.
|
||||
- `snapshot(): number` — current outstanding count.
|
||||
- `capacity(): number` — configured cap.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Construction-time validation re-uses the same error code as the
|
||||
runtime cap-exceeded path. This is deliberate: operators tuning the
|
||||
cap via host config want one error string to alert on, regardless of
|
||||
whether the misconfiguration was a fractional value at startup or a
|
||||
real overload at steady state. The context object on the thrown
|
||||
`TwoPcError` distinguishes the two cases (`cap` only vs.
|
||||
`cap` + `outstanding`).
|
||||
|
||||
The class is intentionally tiny — no async, no internal queue, no
|
||||
external dependencies. Adding a queue here would re-introduce the
|
||||
lock-storm risk the cap exists to prevent.
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { OutstandingPreparedCap } from "./outstanding-cap.js";
|
||||
|
||||
describe("OutstandingPreparedCap", () => {
|
||||
it("admits up to the configured cap", { timeout: 5000 }, () => {
|
||||
const cap = new OutstandingPreparedCap(3);
|
||||
cap.admit();
|
||||
cap.admit();
|
||||
cap.admit();
|
||||
expect(cap.snapshot()).toBe(3);
|
||||
expect(cap.capacity()).toBe(3);
|
||||
});
|
||||
|
||||
it("rejects the (cap+1)th admission with TWOPC.OUTSTANDING_LIMIT_EXCEEDED", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cap = new OutstandingPreparedCap(2);
|
||||
cap.admit();
|
||||
cap.admit();
|
||||
try {
|
||||
cap.admit();
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("recovers as transactions release slots", { timeout: 5000 }, () => {
|
||||
const cap = new OutstandingPreparedCap(1);
|
||||
cap.admit();
|
||||
expect(() => cap.admit()).toThrow();
|
||||
cap.release();
|
||||
cap.admit();
|
||||
expect(cap.snapshot()).toBe(1);
|
||||
});
|
||||
|
||||
it("release below zero clamps at zero (defends against double-release)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cap = new OutstandingPreparedCap(2);
|
||||
cap.release();
|
||||
cap.release();
|
||||
expect(cap.snapshot()).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
0,
|
||||
-1,
|
||||
1.5,
|
||||
Number.NaN,
|
||||
])("constructor rejects non-positive-integer cap %p", (bad) => {
|
||||
expect(() => new OutstandingPreparedCap(bad as number)).toThrow();
|
||||
});
|
||||
});
|
||||
64
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.ts
Normal file
64
packages/api-stack/api/api-2pc/src/limits/outstanding-cap.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
|
||||
/**
|
||||
* Admission counter for outstanding prepared transactions. Tracks how
|
||||
* many `run2pc` calls are between `admit()` and `release()`; new calls
|
||||
* past the cap raise `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` until an
|
||||
* in-flight transaction releases a slot.
|
||||
*
|
||||
* The implementation is single-threaded — `admit` and `release` are
|
||||
* synchronous balanced calls. Bun's event loop guarantees the call
|
||||
* pair is atomic relative to other tasks, so no mutex is required.
|
||||
*/
|
||||
export class OutstandingPreparedCap {
|
||||
private outstanding = 0;
|
||||
|
||||
public constructor(private readonly cap: number) {
|
||||
if (!Number.isInteger(cap) || cap <= 0) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED,
|
||||
`outstandingPreparedCap must be a positive integer; got ${cap}`,
|
||||
{ context: { cap } }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserve a slot. Throws
|
||||
* `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` when the cap is already at
|
||||
* capacity. On success the caller MUST eventually invoke
|
||||
* `release()` in a `finally` block.
|
||||
*/
|
||||
public admit(): void {
|
||||
if (this.outstanding >= this.cap) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.OUTSTANDING_LIMIT_EXCEEDED,
|
||||
`Outstanding prepared transactions at cap ${this.cap}`,
|
||||
{ context: { cap: this.cap, outstanding: this.outstanding } }
|
||||
);
|
||||
}
|
||||
this.outstanding += 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the slot reserved by a matching {@link admit}. Idempotent
|
||||
* against arithmetic underflow — repeated calls clamp at zero so
|
||||
* a buggy double-release in caller code does not allow the next
|
||||
* `admit` to silently exceed the real cap.
|
||||
*/
|
||||
public release(): void {
|
||||
if (this.outstanding > 0) {
|
||||
this.outstanding -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** Current outstanding count. Snapshot read for the OTel gauge. */
|
||||
public snapshot(): number {
|
||||
return this.outstanding;
|
||||
}
|
||||
|
||||
/** Configured cap. Diagnostics only. */
|
||||
public capacity(): number {
|
||||
return this.cap;
|
||||
}
|
||||
}
|
||||
77
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.md
Normal file
77
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.md
Normal file
@@ -0,0 +1,77 @@
|
||||
# `limits/timeout-ceiling`
|
||||
|
||||
## Purpose
|
||||
|
||||
Enforces the hard 30 000 ms ceiling on caller-supplied `timeoutMs`
|
||||
values. The check fires at the public entry point of `run2pc` so
|
||||
above-ceiling calls never contact a participant; the operational
|
||||
warning in design doc §9.2 is that long prepares hold row-level locks,
|
||||
so surfacing the misuse loudly is more useful than silently clamping.
|
||||
|
||||
This module is intentionally tiny — one function — because the
|
||||
behaviour is small but the gate is one of the most-asserted contracts
|
||||
in the spec.
|
||||
|
||||
## Features
|
||||
|
||||
- `assertTimeoutBelowCeiling(timeoutMs)` — throws
|
||||
`TWOPC_TIMEOUT_CEILING_EXCEEDED` for above-ceiling, non-positive,
|
||||
non-finite, or `NaN` inputs.
|
||||
- Re-uses the `TIMEOUT_CEILING_MS` constant from `options/` so the
|
||||
ceiling stays a single source of truth.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { assertTimeoutBelowCeiling } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
assertTimeoutBelowCeiling(5000); // ok
|
||||
assertTimeoutBelowCeiling(30001); // throws TWOPC_TIMEOUT_CEILING_EXCEEDED
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The function is intentionally side-effect-free so callers can wrap it
|
||||
in a per-tenant override path:
|
||||
|
||||
```ts
|
||||
import { TIMEOUT_CEILING_MS } from "@suaveplan/api-2pc/options";
|
||||
import { assertTimeoutBelowCeiling } from "@suaveplan/api-2pc/limits";
|
||||
|
||||
function resolveTimeout(
|
||||
requested: number | undefined,
|
||||
tenantMax: number
|
||||
): number {
|
||||
const effective = requested ?? DEFAULT_TIMEOUT_MS;
|
||||
assertTimeoutBelowCeiling(effective);
|
||||
if (effective > tenantMax) {
|
||||
return tenantMax;
|
||||
}
|
||||
if (effective > TIMEOUT_CEILING_MS) {
|
||||
// unreachable given the assert above — included only to
|
||||
// illustrate that the ceiling acts as the strict upper bound.
|
||||
return TIMEOUT_CEILING_MS;
|
||||
}
|
||||
return effective;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `function assertTimeoutBelowCeiling(timeoutMs: number): void`
|
||||
- Throws `TwoPcError(TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED)` for any
|
||||
of: `NaN`, `Infinity`, `-Infinity`, values `<= 0`, values
|
||||
`> TIMEOUT_CEILING_MS`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The check rejects non-positive values for the same reason it rejects
|
||||
above-ceiling ones: an immediate or backwards-in-time deadline would
|
||||
collapse phase 1 to a single tick and the rollback path would fire
|
||||
before any participant could finish a real prepare. Treating that as
|
||||
the same misuse class as "30 minutes" is a deliberate ergonomic
|
||||
choice — operators get one error code to grep for.
|
||||
|
||||
`Number.isFinite` is used in preference to `Number.isNaN` because the
|
||||
former covers `Infinity` and `-Infinity` in addition to `NaN`. The
|
||||
explicit `<= 0` check then excludes zero and negative-finite values.
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { DEFAULT_TIMEOUT_MS, TIMEOUT_CEILING_MS } from "../options/options.js";
|
||||
import { assertTimeoutBelowCeiling } from "./timeout-ceiling.js";
|
||||
|
||||
function expectCeilingError(fn: () => void): TwoPcError {
|
||||
try {
|
||||
fn();
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
const e = err as TwoPcError;
|
||||
expect(e.twoPcCode).toBe(TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED);
|
||||
return e;
|
||||
}
|
||||
throw new Error("expected TWOPC_TIMEOUT_CEILING_EXCEEDED");
|
||||
}
|
||||
|
||||
describe("assertTimeoutBelowCeiling", () => {
|
||||
it("accepts the default timeoutMs (5000)", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
assertTimeoutBelowCeiling(DEFAULT_TIMEOUT_MS)
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts a value equal to the ceiling", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
assertTimeoutBelowCeiling(TIMEOUT_CEILING_MS)
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects timeoutMs above the ceiling", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() =>
|
||||
assertTimeoutBelowCeiling(TIMEOUT_CEILING_MS + 1)
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects zero", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() => assertTimeoutBelowCeiling(0));
|
||||
});
|
||||
|
||||
it("rejects negative values", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() => assertTimeoutBelowCeiling(-1));
|
||||
});
|
||||
|
||||
it("rejects NaN", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() => assertTimeoutBelowCeiling(Number.NaN));
|
||||
});
|
||||
|
||||
it("rejects Infinity", { timeout: 5000 }, () => {
|
||||
expectCeilingError(() =>
|
||||
assertTimeoutBelowCeiling(Number.POSITIVE_INFINITY)
|
||||
);
|
||||
});
|
||||
});
|
||||
29
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.ts
Normal file
29
packages/api-stack/api/api-2pc/src/limits/timeout-ceiling.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { TIMEOUT_CEILING_MS } from "../options/options.js";
|
||||
|
||||
/**
|
||||
* Validate a caller-supplied `timeoutMs`. Per spec the coordinator
|
||||
* MUST refuse values above the hard ceiling (default 30 000 ms) at
|
||||
* call time — silently clamping would hide the misuse and let
|
||||
* long-held prepares regress production throughput.
|
||||
*
|
||||
* Negative or non-finite values are also refused — they would
|
||||
* collapse the deadline to "immediate" which is operationally
|
||||
* indistinguishable from a flaky participant.
|
||||
*/
|
||||
export function assertTimeoutBelowCeiling(timeoutMs: number): void {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED,
|
||||
`timeoutMs must be a positive finite number; got ${timeoutMs}`,
|
||||
{ context: { timeoutMs } }
|
||||
);
|
||||
}
|
||||
if (timeoutMs > TIMEOUT_CEILING_MS) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT_CEILING_EXCEEDED,
|
||||
`timeoutMs ${timeoutMs} exceeds hard ceiling ${TIMEOUT_CEILING_MS}`,
|
||||
{ context: { timeoutMs, ceiling: TIMEOUT_CEILING_MS } }
|
||||
);
|
||||
}
|
||||
}
|
||||
9
packages/api-stack/api/api-2pc/src/options/index.ts
Normal file
9
packages/api-stack/api/api-2pc/src/options/index.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
type ResolvedTwoPcCoordinatorOptions,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
TIMEOUT_CEILING_MS,
|
||||
type TwoPcCoordinatorOptions,
|
||||
twoPcCoordinatorOptionsSchema,
|
||||
} from "./options.js";
|
||||
96
packages/api-stack/api/api-2pc/src/options/options.md
Normal file
96
packages/api-stack/api/api-2pc/src/options/options.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# `options/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the boundary-validated option shape consumed by
|
||||
`createTwoPcCoordinator`. Every tunable that affects coordinator
|
||||
behaviour (host instance id, outstanding-prepared cap, default phase-1
|
||||
timeout) lives here, with Zod validation at the seam and explicit
|
||||
defaults pinned to the design doc §9.2 numbers.
|
||||
|
||||
Centralising option resolution in one module keeps the coordinator
|
||||
free of conditional fallback logic — by the time it sees the resolved
|
||||
shape every field is concrete, so subsequent code never has to
|
||||
re-check `undefined`.
|
||||
|
||||
## Features
|
||||
|
||||
- `DEFAULT_TIMEOUT_MS = 5000` — phase-1 deadline default per design
|
||||
doc §9.2.
|
||||
- `TIMEOUT_CEILING_MS = 30000` — hard ceiling enforced at call time
|
||||
(the coordinator's `limits/timeout-ceiling` module imports this).
|
||||
- `DEFAULT_OUTSTANDING_PREPARED_CAP = 256` — admission-control default
|
||||
per design doc §14.
|
||||
- `twoPcCoordinatorOptionsSchema` — Zod schema, `strict()` so typos
|
||||
surface as Zod errors instead of being silently ignored.
|
||||
- `resolveTwoPcCoordinatorOptions(raw)` — validates and fills in
|
||||
defaults; returns the fully-populated `ResolvedTwoPcCoordinatorOptions`
|
||||
shape the coordinator consumes.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
} from "@suaveplan/api-2pc/options";
|
||||
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "host-01",
|
||||
});
|
||||
|
||||
console.log(resolved.defaultTimeoutMs === DEFAULT_TIMEOUT_MS); // true
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Override every default — useful for canary hosts where operators want
|
||||
a tighter admission cap and a longer prepare window:
|
||||
|
||||
```ts
|
||||
import { resolveTwoPcCoordinatorOptions } from "@suaveplan/api-2pc/options";
|
||||
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "canary-eu-west-1",
|
||||
outstandingPreparedCap: 64,
|
||||
defaultTimeoutMs: 15_000,
|
||||
});
|
||||
```
|
||||
|
||||
Calls supplying `defaultTimeoutMs` above `TIMEOUT_CEILING_MS` raise a
|
||||
Zod error at this boundary — the coordinator's runtime ceiling check
|
||||
in `limits/timeout-ceiling.ts` then surfaces the misuse on per-call
|
||||
overrides as well.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `const DEFAULT_TIMEOUT_MS: number` — 5000.
|
||||
- `const TIMEOUT_CEILING_MS: number` — 30000.
|
||||
- `const DEFAULT_OUTSTANDING_PREPARED_CAP: number` — 256.
|
||||
- `interface TwoPcCoordinatorOptions` — caller-supplied input shape.
|
||||
- `interface ResolvedTwoPcCoordinatorOptions` — internal shape after
|
||||
defaults.
|
||||
- `const twoPcCoordinatorOptionsSchema: z.ZodType` — strict-shape
|
||||
Zod schema.
|
||||
- `function resolveTwoPcCoordinatorOptions(raw): ResolvedTwoPcCoordinatorOptions`
|
||||
— validates and resolves defaults.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The schema uses `.strict()` so unknown keys fail validation rather than
|
||||
silently being dropped. The default for `defaultTimeoutMs` is applied
|
||||
post-parse with `?? DEFAULT_TIMEOUT_MS` rather than via
|
||||
`z.default(...)` so the schema continues to reject above-ceiling
|
||||
values regardless of whether the caller supplied the field — Zod's
|
||||
`default` would short-circuit `.max()` checks when the field is
|
||||
omitted, which is the wrong shape for a ceiling-enforced option.
|
||||
|
||||
The module deliberately does NOT re-throw Zod failures as
|
||||
`TwoPcError` — that's the responsibility of the coordinator's
|
||||
public entry points. Keeping this module pure of cross-package error
|
||||
mapping makes it reusable from tests that want a raw Zod surface for
|
||||
property-based shape testing.
|
||||
70
packages/api-stack/api/api-2pc/src/options/options.test.ts
Normal file
70
packages/api-stack/api/api-2pc/src/options/options.test.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
resolveTwoPcCoordinatorOptions,
|
||||
TIMEOUT_CEILING_MS,
|
||||
twoPcCoordinatorOptionsSchema,
|
||||
} from "./options.js";
|
||||
|
||||
describe("resolveTwoPcCoordinatorOptions", () => {
|
||||
it("applies all defaults when only required fields supplied", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "host-1",
|
||||
});
|
||||
expect(resolved.hostInstanceId).toBe("host-1");
|
||||
expect(resolved.defaultTimeoutMs).toBe(DEFAULT_TIMEOUT_MS);
|
||||
expect(resolved.outstandingPreparedCap).toBe(
|
||||
DEFAULT_OUTSTANDING_PREPARED_CAP
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves explicit overrides", { timeout: 5000 }, () => {
|
||||
const resolved = resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "h",
|
||||
defaultTimeoutMs: 12345,
|
||||
outstandingPreparedCap: 4,
|
||||
});
|
||||
expect(resolved.defaultTimeoutMs).toBe(12345);
|
||||
expect(resolved.outstandingPreparedCap).toBe(4);
|
||||
});
|
||||
|
||||
it("rejects defaultTimeoutMs above the ceiling at the schema boundary", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "h",
|
||||
defaultTimeoutMs: TIMEOUT_CEILING_MS + 1,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects an empty hostInstanceId", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
resolveTwoPcCoordinatorOptions({ hostInstanceId: "" })
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects unknown keys (strict schema)", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
twoPcCoordinatorOptionsSchema.parse({
|
||||
hostInstanceId: "h",
|
||||
bogus: 1,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects a non-integer outstandingPreparedCap", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
resolveTwoPcCoordinatorOptions({
|
||||
hostInstanceId: "h",
|
||||
outstandingPreparedCap: 1.5,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
84
packages/api-stack/api/api-2pc/src/options/options.ts
Normal file
84
packages/api-stack/api/api-2pc/src/options/options.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
import z from "zod";
|
||||
|
||||
/**
|
||||
* Default phase-1 deadline. Design doc §9.2 anchors at 5 000 ms — long
|
||||
* enough for a real cross-database prepare, short enough that
|
||||
* row-level locks held by `PREPARE TRANSACTION` do not pile up.
|
||||
*/
|
||||
export const DEFAULT_TIMEOUT_MS = 5000;
|
||||
|
||||
/**
|
||||
* Hard ceiling on `timeoutMs`. Calls above this are refused at the
|
||||
* boundary rather than silently clamped — operators want to see the
|
||||
* misuse loudly because long prepares hold row-level locks.
|
||||
*/
|
||||
export const TIMEOUT_CEILING_MS = 30000;
|
||||
|
||||
/**
|
||||
* Default cap on outstanding prepared transactions per coordinator.
|
||||
* Design doc §14 left this open; this is the spec'd default and is
|
||||
* operator-tunable. New `run2pc` calls past the cap fail with
|
||||
* `TWOPC_OUTSTANDING_LIMIT_EXCEEDED` until in-flight transactions
|
||||
* complete.
|
||||
*/
|
||||
export const DEFAULT_OUTSTANDING_PREPARED_CAP = 256;
|
||||
|
||||
/**
|
||||
* Caller-tunable options handed to `createTwoPcCoordinator`. Every
|
||||
* field is optional; defaults are applied by `resolveOptions`.
|
||||
*/
|
||||
export interface TwoPcCoordinatorOptions {
|
||||
/** Stable host id pre-tagged on every prepared transaction's gid. */
|
||||
readonly hostInstanceId: string;
|
||||
/**
|
||||
* Override of `DEFAULT_OUTSTANDING_PREPARED_CAP`. Must be `>= 1`.
|
||||
*/
|
||||
readonly outstandingPreparedCap?: number;
|
||||
/**
|
||||
* Override of `DEFAULT_TIMEOUT_MS`. Must satisfy
|
||||
* `1 <= value <= TIMEOUT_CEILING_MS`.
|
||||
*/
|
||||
readonly defaultTimeoutMs?: number;
|
||||
}
|
||||
|
||||
/** Strict-shape Zod schema for {@link TwoPcCoordinatorOptions}. */
|
||||
export const twoPcCoordinatorOptionsSchema = z
|
||||
.object({
|
||||
hostInstanceId: z.string().min(1),
|
||||
outstandingPreparedCap: z.number().int().positive().optional(),
|
||||
defaultTimeoutMs: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.max(TIMEOUT_CEILING_MS)
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
/**
|
||||
* Resolved option shape — every field present, defaults applied. The
|
||||
* coordinator only ever reads this shape; the optional fields exist on
|
||||
* the input shape only.
|
||||
*/
|
||||
export interface ResolvedTwoPcCoordinatorOptions {
|
||||
readonly hostInstanceId: string;
|
||||
readonly outstandingPreparedCap: number;
|
||||
readonly defaultTimeoutMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply defaults and Zod validation. Throws the raw `z.ZodError` so
|
||||
* callers can repackage it into a domain error if they prefer; the
|
||||
* coordinator boundary wraps this in a `TwoPcError` itself.
|
||||
*/
|
||||
export function resolveTwoPcCoordinatorOptions(
|
||||
raw: TwoPcCoordinatorOptions
|
||||
): ResolvedTwoPcCoordinatorOptions {
|
||||
const parsed = twoPcCoordinatorOptionsSchema.parse(raw);
|
||||
return {
|
||||
hostInstanceId: parsed.hostInstanceId,
|
||||
outstandingPreparedCap:
|
||||
parsed.outstandingPreparedCap ?? DEFAULT_OUTSTANDING_PREPARED_CAP,
|
||||
defaultTimeoutMs: parsed.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
};
|
||||
}
|
||||
7
packages/api-stack/api/api-2pc/src/persistence/index.ts
Normal file
7
packages/api-stack/api/api-2pc/src/persistence/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
type LedgerStatement,
|
||||
type TwoPcLedgerRow,
|
||||
TwoPcLedgerStore,
|
||||
} from "./persistence.js";
|
||||
114
packages/api-stack/api/api-2pc/src/persistence/persistence.md
Normal file
114
packages/api-stack/api/api-2pc/src/persistence/persistence.md
Normal file
@@ -0,0 +1,114 @@
|
||||
# `persistence/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Persists every coordinator state transition into the
|
||||
`suaveplan_2pc_transactions` ledger table. The table is owned by
|
||||
`@suaveplan/api-host` (migration id 2, `saga-2pc-reserved`); this
|
||||
module only reads and writes rows so the host stays the single source
|
||||
of truth for schema evolution.
|
||||
|
||||
The ledger is what makes 2PC durable across coordinator restarts.
|
||||
Without it, a host that crashes mid-prepare would lose track of which
|
||||
participants were `PREPARE TRANSACTION`'d and could not safely choose
|
||||
between commit and rollback on the next boot.
|
||||
|
||||
## Features
|
||||
|
||||
- `TwoPcLedgerStore` — CRUD facade over `suaveplan_2pc_transactions`
|
||||
with statements for every state transition (`insert`,
|
||||
`markAllVotedCommit`, `markAborting`, `markCommitted`, `markAborted`,
|
||||
`markInDoubt`).
|
||||
- `findById` — single-row lookup for state inspection.
|
||||
- `findRecoverable` — scan returning every non-terminal row for the
|
||||
current host instance id. Drives `recovery/`.
|
||||
- `TwoPcLedgerStore.decodeParticipants` — static helper that parses the
|
||||
`participants_json` blob back into a typed array.
|
||||
- Structural `LedgerDb` / `LedgerStatement` interfaces so the store
|
||||
works with `bun:sqlite`, `@suaveplan/db-sqlite`'s thin wrapper, and
|
||||
`better-sqlite3` without any concrete dependency.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { Database } from "bun:sqlite";
|
||||
import { TwoPcLedgerStore } from "@suaveplan/api-2pc/persistence";
|
||||
|
||||
const db = new Database(":memory:");
|
||||
// Schema is created by api-host's migration runner — see migrations.ts
|
||||
// migration id 2 (`saga-2pc-reserved`). In tests, run the host runner
|
||||
// against the same database.
|
||||
|
||||
const store = new TwoPcLedgerStore(db as never);
|
||||
store.insert({
|
||||
id: "tx-1",
|
||||
coordinator: "host-instance-01",
|
||||
participants: [
|
||||
{ participantId: "billing", voterMode: "writer", vote: null },
|
||||
],
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Walk the recovery candidates and decode each blob — the typical shape
|
||||
used inside `recovery/recovery.ts`:
|
||||
|
||||
```ts
|
||||
import { TwoPcLedgerStore } from "@suaveplan/api-2pc/persistence";
|
||||
|
||||
function listRecoverable(store: TwoPcLedgerStore, hostId: string) {
|
||||
const rows = store.findRecoverable(hostId);
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
state: row.state,
|
||||
participants: TwoPcLedgerStore.decodeParticipants(row),
|
||||
}));
|
||||
}
|
||||
```
|
||||
|
||||
The static `decodeParticipants` is deliberately exposed so callers
|
||||
working with raw rows (e.g. ad-hoc admin scripts) do not have to
|
||||
recreate the JSON contract themselves.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface LedgerStatement` / `interface LedgerDb` — structural
|
||||
SQLite-shaped views.
|
||||
- `interface TwoPcLedgerRow` — exact row shape returned by `findById`
|
||||
/ `findRecoverable`.
|
||||
- `interface LedgerParticipantEntry` — per-participant entry in
|
||||
`participants_json`.
|
||||
- `class TwoPcLedgerStore`
|
||||
- `constructor(db: LedgerDb)`
|
||||
- `insert({ id, coordinator, participants })`
|
||||
- `markAllVotedCommit({ id, participants, preparedAt })`
|
||||
- `markAborting({ id, participants })`
|
||||
- `markCommitted(id, committedAt)`
|
||||
- `markAborted(id, abortedAt)`
|
||||
- `markInDoubt(id)`
|
||||
- `findById(id): TwoPcLedgerRow | null`
|
||||
- `findRecoverable(coordinator): readonly TwoPcLedgerRow[]`
|
||||
- `static decodeParticipants(row): readonly LedgerParticipantEntry[]`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Every UPDATE uses a single statement with the new `state` value
|
||||
matched against `TwoPcCoordinatorState`'s string-literal union — the
|
||||
`satisfies TwoPcCoordinatorState` annotation catches typos at compile
|
||||
time. The store does NOT enforce a state machine on top of the table;
|
||||
that responsibility belongs to the coordinator. Keeping the
|
||||
persistence layer state-machine-free lets recovery use the same store
|
||||
without conflating "what changed in this lifetime" with "what the row
|
||||
looked like after a crash".
|
||||
|
||||
The `participants_json` blob is stored as a JSON string rather than a
|
||||
side table — the coordinator never queries by participant id, only by
|
||||
transaction id, so a JSON blob has zero indexing cost and survives
|
||||
schema evolution painlessly. `decodeParticipants` returns an empty
|
||||
array when the blob is not an array — defensive against rows
|
||||
hand-written by ad-hoc admin scripts.
|
||||
@@ -0,0 +1,174 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
TwoPcLedgerStore,
|
||||
} from "./persistence.js";
|
||||
|
||||
function makeDb(): LedgerDb {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
return db;
|
||||
}
|
||||
|
||||
const SAMPLE_PARTICIPANTS: ReadonlyArray<LedgerParticipantEntry> = [
|
||||
{ participantId: "billing", voterMode: "writer", vote: null },
|
||||
{ participantId: "inventory", voterMode: "writer", vote: null },
|
||||
];
|
||||
|
||||
describe("TwoPcLedgerStore", () => {
|
||||
it("inserts a new row in voting state", { timeout: 5000 }, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-1",
|
||||
coordinator: "host-1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
const row = store.findById("tx-1");
|
||||
expect(row).not.toBeNull();
|
||||
expect(row?.state).toBe("voting");
|
||||
expect(row?.coordinator).toBe("host-1");
|
||||
expect(row?.preparedAt).toBeNull();
|
||||
});
|
||||
|
||||
it("marks the row committing once all participants voted commit", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-2",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markAllVotedCommit({
|
||||
id: "tx-2",
|
||||
participants: SAMPLE_PARTICIPANTS.map((p) => ({
|
||||
...p,
|
||||
vote: "commit",
|
||||
})),
|
||||
preparedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
const row = store.findById("tx-2");
|
||||
expect(row?.state).toBe("committing");
|
||||
expect(row?.preparedAt).toBe("2026-05-26T00:00:00.000Z");
|
||||
const decoded = TwoPcLedgerStore.decodeParticipants(
|
||||
row as NonNullable<typeof row>
|
||||
);
|
||||
expect(decoded[0]?.vote).toBe("commit");
|
||||
});
|
||||
|
||||
it("transitions through aborting then committed and aborted final states", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-3",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markAborting({ id: "tx-3", participants: SAMPLE_PARTICIPANTS });
|
||||
expect(store.findById("tx-3")?.state).toBe("aborting");
|
||||
|
||||
store.markCommitted("tx-3", "2026-05-26T00:01:00.000Z");
|
||||
expect(store.findById("tx-3")?.committedAt).toBe(
|
||||
"2026-05-26T00:01:00.000Z"
|
||||
);
|
||||
|
||||
store.markAborted("tx-3", "2026-05-26T00:02:00.000Z");
|
||||
expect(store.findById("tx-3")?.abortedAt).toBe(
|
||||
"2026-05-26T00:02:00.000Z"
|
||||
);
|
||||
});
|
||||
|
||||
it("marks rows in-doubt", { timeout: 5000 }, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "tx-4",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markInDoubt("tx-4");
|
||||
expect(store.findById("tx-4")?.state).toBe("in_doubt");
|
||||
});
|
||||
|
||||
it("findById returns null for missing rows", { timeout: 5000 }, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
expect(store.findById("never")).toBeNull();
|
||||
});
|
||||
|
||||
it("findRecoverable returns only non-terminal rows for matching coordinator", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const store = new TwoPcLedgerStore(makeDb());
|
||||
store.insert({
|
||||
id: "a",
|
||||
coordinator: "h1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.insert({
|
||||
id: "b",
|
||||
coordinator: "h1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markCommitted("b", "2026-05-26T00:00:00.000Z");
|
||||
store.insert({
|
||||
id: "c",
|
||||
coordinator: "h2",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.insert({
|
||||
id: "d",
|
||||
coordinator: "h1",
|
||||
participants: SAMPLE_PARTICIPANTS,
|
||||
});
|
||||
store.markInDoubt("d");
|
||||
|
||||
const recoverable = store.findRecoverable("h1");
|
||||
const ids = recoverable.map((r) => r.id).sort();
|
||||
expect(ids).toEqual(["a", "d"]);
|
||||
});
|
||||
|
||||
it("decodeParticipants returns an empty array on non-array JSON", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const decoded = TwoPcLedgerStore.decodeParticipants({
|
||||
id: "x",
|
||||
coordinator: "h",
|
||||
state: "voting",
|
||||
participantsJson: '{"not":"array"}',
|
||||
preparedAt: null,
|
||||
committedAt: null,
|
||||
abortedAt: null,
|
||||
});
|
||||
expect(decoded).toEqual([]);
|
||||
});
|
||||
|
||||
it("decodeParticipants round-trips real entries", { timeout: 5000 }, () => {
|
||||
const decoded = TwoPcLedgerStore.decodeParticipants({
|
||||
id: "x",
|
||||
coordinator: "h",
|
||||
state: "committing",
|
||||
participantsJson: JSON.stringify([
|
||||
{
|
||||
participantId: "p",
|
||||
voterMode: "writer",
|
||||
vote: "commit",
|
||||
},
|
||||
]),
|
||||
preparedAt: null,
|
||||
committedAt: null,
|
||||
abortedAt: null,
|
||||
});
|
||||
expect(decoded).toHaveLength(1);
|
||||
expect(decoded[0]?.vote).toBe("commit");
|
||||
});
|
||||
});
|
||||
228
packages/api-stack/api/api-2pc/src/persistence/persistence.ts
Normal file
228
packages/api-stack/api/api-2pc/src/persistence/persistence.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
import type { TwoPcCoordinatorState } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Structural view of a SQLite-shaped statement returned by
|
||||
* `LedgerDb.prepare(sql)`. Matches `bun:sqlite`'s `Statement` surface
|
||||
* narrowly: only the methods this module needs are exposed.
|
||||
*/
|
||||
export interface LedgerStatement {
|
||||
run(...params: readonly unknown[]): { changes: number } | unknown;
|
||||
all<T = Record<string, unknown>>(
|
||||
...params: readonly unknown[]
|
||||
): readonly T[];
|
||||
get<T = Record<string, unknown>>(
|
||||
...params: readonly unknown[]
|
||||
): T | undefined | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural SQLite-like database handle. Compatible with `bun:sqlite`,
|
||||
* `@suaveplan/db-sqlite`'s thin wrapper, and `better-sqlite3`. The
|
||||
* coordinator only ever sees this interface — concrete drivers stay
|
||||
* outside the package boundary.
|
||||
*/
|
||||
export interface LedgerDb {
|
||||
prepare(sql: string): LedgerStatement;
|
||||
/** Execute a single DDL or DML statement with no parameters. */
|
||||
run(sql: string): unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* One row of `suaveplan_2pc_transactions`. The schema is owned by
|
||||
* `@suaveplan/api-host` (see `plugin-ledger/migrations.ts` migration
|
||||
* id 2, `saga-2pc-reserved`). This row shape mirrors the DDL fields
|
||||
* verbatim — keep them in lockstep when the host schema evolves.
|
||||
*/
|
||||
export interface TwoPcLedgerRow {
|
||||
readonly id: string;
|
||||
readonly coordinator: string;
|
||||
readonly state: TwoPcCoordinatorState;
|
||||
readonly participantsJson: string;
|
||||
readonly preparedAt: string | null;
|
||||
readonly committedAt: string | null;
|
||||
readonly abortedAt: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-participant entry inside the `participants_json` blob. Captures
|
||||
* the vote and the voter mode so recovery can re-derive whether to
|
||||
* send commit/rollback SQL on boot.
|
||||
*/
|
||||
export interface LedgerParticipantEntry {
|
||||
readonly participantId: string;
|
||||
readonly voterMode: "writer" | "read-only";
|
||||
readonly vote: TwoPcVote | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistence facade over `suaveplan_2pc_transactions`. The schema is
|
||||
* created by `api-host`'s migration runner; this module only reads and
|
||||
* writes rows.
|
||||
*/
|
||||
export class TwoPcLedgerStore {
|
||||
public constructor(private readonly db: LedgerDb) {}
|
||||
|
||||
/** Insert a new row in the `voting` state. */
|
||||
public insert(input: {
|
||||
id: string;
|
||||
coordinator: string;
|
||||
participants: ReadonlyArray<LedgerParticipantEntry>;
|
||||
}): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO suaveplan_2pc_transactions
|
||||
(id, coordinator, state, participants_json, prepared_at, committed_at, aborted_at)
|
||||
VALUES (?, ?, ?, ?, NULL, NULL, NULL)`
|
||||
)
|
||||
.run(
|
||||
input.id,
|
||||
input.coordinator,
|
||||
"voting" satisfies TwoPcCoordinatorState,
|
||||
JSON.stringify(input.participants)
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark the row as `committing` and stamp `prepared_at`. */
|
||||
public markAllVotedCommit(input: {
|
||||
id: string;
|
||||
participants: ReadonlyArray<LedgerParticipantEntry>;
|
||||
preparedAt: string;
|
||||
}): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, participants_json = ?, prepared_at = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(
|
||||
"committing" satisfies TwoPcCoordinatorState,
|
||||
JSON.stringify(input.participants),
|
||||
input.preparedAt,
|
||||
input.id
|
||||
);
|
||||
}
|
||||
|
||||
/** Mark the row as `aborting` after a vote_abort or timeout. */
|
||||
public markAborting(input: {
|
||||
id: string;
|
||||
participants: ReadonlyArray<LedgerParticipantEntry>;
|
||||
}): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, participants_json = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(
|
||||
"aborting" satisfies TwoPcCoordinatorState,
|
||||
JSON.stringify(input.participants),
|
||||
input.id
|
||||
);
|
||||
}
|
||||
|
||||
/** Final-state stamp once every participant acknowledged commit. */
|
||||
public markCommitted(id: string, committedAt: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, committed_at = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run("committed" satisfies TwoPcCoordinatorState, committedAt, id);
|
||||
}
|
||||
|
||||
/** Final-state stamp once every participant acknowledged rollback. */
|
||||
public markAborted(id: string, abortedAt: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?, aborted_at = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run("aborted" satisfies TwoPcCoordinatorState, abortedAt, id);
|
||||
}
|
||||
|
||||
/** Mark a transaction as in-doubt (commit-prepared / rollback-prepared SQL failed). */
|
||||
public markInDoubt(id: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE suaveplan_2pc_transactions
|
||||
SET state = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run("in_doubt" satisfies TwoPcCoordinatorState, id);
|
||||
}
|
||||
|
||||
/** Lookup a single row by id. Returns `null` when none exists. */
|
||||
public findById(id: string): TwoPcLedgerRow | null {
|
||||
const row = this.db
|
||||
.prepare(
|
||||
`SELECT id, coordinator, state, participants_json,
|
||||
prepared_at, committed_at, aborted_at
|
||||
FROM suaveplan_2pc_transactions WHERE id = ?`
|
||||
)
|
||||
.get<{
|
||||
id: string;
|
||||
coordinator: string;
|
||||
state: TwoPcCoordinatorState;
|
||||
participants_json: string;
|
||||
prepared_at: string | null;
|
||||
committed_at: string | null;
|
||||
aborted_at: string | null;
|
||||
}>(id);
|
||||
if (row === null || row === undefined) return null;
|
||||
return {
|
||||
id: row.id,
|
||||
coordinator: row.coordinator,
|
||||
state: row.state,
|
||||
participantsJson: row.participants_json,
|
||||
preparedAt: row.prepared_at,
|
||||
committedAt: row.committed_at,
|
||||
abortedAt: row.aborted_at,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Recovery scan — every row whose state is non-terminal AND whose
|
||||
* coordinator matches the provided host instance id. These rows
|
||||
* are the candidates the recovery runner must resolve on boot.
|
||||
*/
|
||||
public findRecoverable(coordinator: string): readonly TwoPcLedgerRow[] {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`SELECT id, coordinator, state, participants_json,
|
||||
prepared_at, committed_at, aborted_at
|
||||
FROM suaveplan_2pc_transactions
|
||||
WHERE coordinator = ?
|
||||
AND state IN ('voting','committing','aborting','in_doubt')`
|
||||
)
|
||||
.all<{
|
||||
id: string;
|
||||
coordinator: string;
|
||||
state: TwoPcCoordinatorState;
|
||||
participants_json: string;
|
||||
prepared_at: string | null;
|
||||
committed_at: string | null;
|
||||
aborted_at: string | null;
|
||||
}>(coordinator);
|
||||
return rows.map((r) => ({
|
||||
id: r.id,
|
||||
coordinator: r.coordinator,
|
||||
state: r.state,
|
||||
participantsJson: r.participants_json,
|
||||
preparedAt: r.prepared_at,
|
||||
committedAt: r.committed_at,
|
||||
abortedAt: r.aborted_at,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Decode the `participants_json` blob back into an array. */
|
||||
public static decodeParticipants(
|
||||
row: TwoPcLedgerRow
|
||||
): ReadonlyArray<LedgerParticipantEntry> {
|
||||
const parsed = JSON.parse(row.participantsJson) as unknown;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed as ReadonlyArray<LedgerParticipantEntry>;
|
||||
}
|
||||
}
|
||||
94
packages/api-stack/api/api-2pc/src/phases/commit.md
Normal file
94
packages/api-stack/api/api-2pc/src/phases/commit.md
Normal file
@@ -0,0 +1,94 @@
|
||||
# `phases/commit`
|
||||
|
||||
## Purpose
|
||||
|
||||
Phase-2 commit for a single participant. Once every participant has
|
||||
returned `commit` from phase 1 and the coordinator has persisted the
|
||||
`all-voted-commit` state to the ledger, this module emits
|
||||
`COMMIT PREPARED '<gid>'` against each writer participant and records
|
||||
`2pc.commit_duration_ms`.
|
||||
|
||||
Phase 2 is intentionally unbounded by timeout — the protocol cannot
|
||||
tolerate giving up on a commit-prepared mid-flight, because doing so
|
||||
would leave the system in `in_doubt` state and require the recovery
|
||||
runner to resolve it on the next boot.
|
||||
|
||||
## Features
|
||||
|
||||
- Skips participants that did not actually prepare (`outcome.prepared
|
||||
=== false`) — read-only voters and abort-voters never see commit
|
||||
SQL.
|
||||
- Emits a `2pc.commit` span with `tx.id`, `participant.id`, and `gid`
|
||||
attributes.
|
||||
- Records `2pc.commit_duration_ms` tagged with `outcome: "committed"`
|
||||
on success.
|
||||
- Failure preserves the underlying cause on a thrown
|
||||
`TwoPcError(TWOPC_COMMIT_FAILED)`; the coordinator escalates to
|
||||
`TWOPC_IN_DOUBT` once it has tried every participant.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { commitPhaseTwo } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
await commitPhaseTwo({
|
||||
txId,
|
||||
participant,
|
||||
outcome: preparedOutcome,
|
||||
instruments,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The coordinator collects per-participant failures rather than
|
||||
short-circuiting — every participant gets attempted exactly once so
|
||||
operators can see the full picture in the ledger:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
import { commitPhaseTwo } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const failures: TwoPcError[] = [];
|
||||
for (const outcome of preparedOutcomes) {
|
||||
if (!outcome.prepared) continue;
|
||||
try {
|
||||
await commitPhaseTwo({
|
||||
txId,
|
||||
participant: registry.get(outcome.participantId)!,
|
||||
outcome,
|
||||
instruments,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof TwoPcError) failures.push(err);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
// ledger.markInDoubt(txId); recovery resolves on next boot
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface CommitPhaseInput`
|
||||
- `txId: string`
|
||||
- `participant: RegisteredTwoPcParticipant`
|
||||
- `outcome: ParticipantPrepareOutcome` — the phase-1 result.
|
||||
- `instruments: TwoPcInstruments`
|
||||
- `function commitPhaseTwo(input): Promise<void>`
|
||||
- no-op for `outcome.prepared === false`
|
||||
- throws `TwoPcError(TWOPC_COMMIT_FAILED)` on SQL failure.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The gid stored on the prepare outcome is the single source of truth —
|
||||
this module never rebuilds it. Reusing the prepare-time value
|
||||
defends against the unlikely case where a host-config change between
|
||||
phases would have produced a different gid: the SQL the coordinator
|
||||
needs to send is exactly the one it prepared with.
|
||||
|
||||
The SQL escape (`replace(/'/g, "''")`) matches the prepare-phase
|
||||
treatment so a gid containing the unlikely quote character would
|
||||
still produce valid SQL. The escape is cheap; the defence-in-depth
|
||||
costs nothing on the happy path.
|
||||
98
packages/api-stack/api/api-2pc/src/phases/commit.test.ts
Normal file
98
packages/api-stack/api/api-2pc/src/phases/commit.test.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { createTwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { commitPhaseTwo } from "./commit.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
|
||||
const INSTRUMENTS = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
|
||||
function makeParticipant(opts: { fail?: Error } = {}) {
|
||||
const calls: string[] = [];
|
||||
const p: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.fail) throw opts.fail;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
return { participant: p, calls };
|
||||
}
|
||||
|
||||
const PREPARED_OUTCOME: ParticipantPrepareOutcome = {
|
||||
participantId: "billing",
|
||||
vote: "commit",
|
||||
prepared: true,
|
||||
gid: "2pc-h-t-billing",
|
||||
};
|
||||
|
||||
describe("commitPhaseTwo", () => {
|
||||
it("issues COMMIT PREPARED against a prepared writer", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: PREPARED_OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls[0]).toBe("COMMIT PREPARED '2pc-h-t-billing'");
|
||||
});
|
||||
|
||||
it("is a no-op when outcome.prepared === false", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: { ...PREPARED_OUTCOME, prepared: false, gid: "" },
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("wraps a SQL throw into TWOPC.COMMIT_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error("commit failed");
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
try {
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: PREPARED_OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.COMMIT_FAILED
|
||||
);
|
||||
expect((err as TwoPcError).cause).toBe(cause);
|
||||
}
|
||||
});
|
||||
|
||||
it("escapes single quotes inside the gid before interpolation", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await commitPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: { ...PREPARED_OUTCOME, gid: "a'b" },
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls[0]).toBe("COMMIT PREPARED 'a''b'");
|
||||
});
|
||||
});
|
||||
66
packages/api-stack/api/api-2pc/src/phases/commit.ts
Normal file
66
packages/api-stack/api/api-2pc/src/phases/commit.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { TwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
|
||||
/**
|
||||
* Input to {@link commitPhaseTwo}. The participant entry is consulted
|
||||
* for the connection; the outcome carries the gid.
|
||||
*/
|
||||
export interface CommitPhaseInput {
|
||||
readonly txId: string;
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
readonly outcome: ParticipantPrepareOutcome;
|
||||
readonly instruments: TwoPcInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 2 commit for a single participant. Issues
|
||||
* `COMMIT PREPARED '<gid>'` and records the commit duration.
|
||||
*
|
||||
* Read-only voters and writers that did not actually prepare (vote
|
||||
* was `abort`) are skipped — the caller only routes prepared writers
|
||||
* through this function.
|
||||
*
|
||||
* Failures throw `TWOPC_COMMIT_FAILED` with the cause preserved. The
|
||||
* coordinator escalates to `TWOPC_IN_DOUBT` once it has tried every
|
||||
* participant.
|
||||
*/
|
||||
export async function commitPhaseTwo(input: CommitPhaseInput): Promise<void> {
|
||||
const { txId, participant, outcome, instruments } = input;
|
||||
if (!outcome.prepared) return;
|
||||
const span = instruments.tracer.startSpan("2pc.commit", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.id": participant.id,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
});
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`COMMIT PREPARED '${outcome.gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (cause) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.COMMIT_FAILED,
|
||||
`COMMIT PREPARED failed for ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
participantId: participant.id,
|
||||
txId,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
instruments.commitDurationMs.record(Date.now() - t0, {
|
||||
outcome: "committed",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
85
packages/api-stack/api/api-2pc/src/phases/gid.md
Normal file
85
packages/api-stack/api/api-2pc/src/phases/gid.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# `phases/gid`
|
||||
|
||||
## Purpose
|
||||
|
||||
Builds and parses the global transaction id (gid) string passed to
|
||||
Postgres' `PREPARE TRANSACTION '<gid>'`. The format embeds the host
|
||||
instance id so the recovery scanner can filter `pg_prepared_xacts` to
|
||||
only the rows this coordinator owns — without that anchor, two
|
||||
coordinators sharing a Postgres host would clobber each other's
|
||||
recovery decisions.
|
||||
|
||||
The module is intentionally pure: no SQL, no client. Centralising the
|
||||
gid format here makes the recovery filter and the per-participant
|
||||
prepare path share a single source of truth.
|
||||
|
||||
## Features
|
||||
|
||||
- `buildGid(hostInstanceId, txId, participantId)` — produces
|
||||
`2pc-<host>-<tx>-<participant>`.
|
||||
- `parseGid(gid, hostInstanceId)` — extracts `{ txId, participantId }`
|
||||
for gids written by the supplied host; returns `null` otherwise so
|
||||
recovery skips foreign prepared transactions silently. The
|
||||
hostInstanceId is supplied by the caller (the recovery runner)
|
||||
because the host id itself may contain hyphens and the parser cannot
|
||||
recover it unambiguously from the gid alone.
|
||||
- `gidPrefixFor(hostInstanceId)` — the prefix the recovery SQL filter
|
||||
uses; centralised so SQL and parser never drift.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { buildGid, parseGid } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const gid = buildGid("host-01", "tx-abc", "billing");
|
||||
// "2pc-host-01-tx-abc-billing"
|
||||
|
||||
const parts = parseGid(gid, "host-01");
|
||||
// { txId: "tx-abc", participantId: "billing" }
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Recovery uses the prefix filter to ignore prepared transactions
|
||||
written by other components sharing the database:
|
||||
|
||||
```ts
|
||||
import { gidPrefixFor, parseGid } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const prefix = gidPrefixFor("host-01");
|
||||
const rows = await db.run<{ gid: string }>(
|
||||
`SELECT gid FROM pg_prepared_xacts WHERE gid LIKE $1`,
|
||||
[`${prefix}%`]
|
||||
);
|
||||
for (const row of rows) {
|
||||
const parts = parseGid(row.gid, "host-01");
|
||||
if (parts === null) continue; // foreign prepared transaction
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `buildGid(hostInstanceId: string, txId: string, participantId: string): string`
|
||||
- `parseGid(gid: string, hostInstanceId: string): { txId, participantId } | null`
|
||||
- `gidPrefixFor(hostInstanceId: string): string`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`parseGid` takes the caller-supplied `hostInstanceId` and strips the
|
||||
prefix `2pc-<host>-` before extracting `txId` and `participantId`.
|
||||
The host instance id is supplied explicitly because hyphens are
|
||||
allowed inside it (production hosts often look like `host-eu-west-1`)
|
||||
and the parser cannot recover the boundary from the gid alone. The
|
||||
recovery runner always knows its own host id, so this constraint is
|
||||
ergonomic in practice.
|
||||
|
||||
Participant ids are constrained by the registry (and by host config)
|
||||
to not contain hyphens, so the LAST hyphen after the prefix
|
||||
unambiguously delimits the `participantId` from the `txId`.
|
||||
|
||||
The function returns `null` rather than throwing on unrecognised gids
|
||||
because the recovery scan iterates over rows that may include prepared
|
||||
transactions written by code outside this package's control — Postgres
|
||||
has no participant-tag column, only the gid string itself, so the
|
||||
shared-database scenario is a legitimate concern.
|
||||
58
packages/api-stack/api/api-2pc/src/phases/gid.test.ts
Normal file
58
packages/api-stack/api/api-2pc/src/phases/gid.test.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { buildGid, gidPrefixFor, parseGid } from "./gid.js";
|
||||
|
||||
describe("buildGid", () => {
|
||||
it("produces the documented prefix-host-tx-participant format", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(buildGid("h1", "tx-abc", "billing")).toBe(
|
||||
"2pc-h1-tx-abc-billing"
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves hyphens in the host instance id", { timeout: 5000 }, () => {
|
||||
expect(buildGid("host-01", "tx", "billing")).toBe(
|
||||
"2pc-host-01-tx-billing"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseGid", () => {
|
||||
it("round-trips a freshly-built gid", { timeout: 5000 }, () => {
|
||||
const gid = buildGid("host-01", "tx-uuid-with-hyphens", "billing");
|
||||
const parts = parseGid(gid, "host-01");
|
||||
expect(parts).toEqual({
|
||||
txId: "tx-uuid-with-hyphens",
|
||||
participantId: "billing",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null for gids without the 2pc- prefix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseGid("foreign-gid", "h")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for gids written by another host", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const gid = buildGid("host-other", "tx", "billing");
|
||||
expect(parseGid(gid, "host-mine")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null when there is no participant suffix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseGid("2pc-h-onlyhost", "h")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null on an empty txId", { timeout: 5000 }, () => {
|
||||
expect(parseGid("2pc-h--billing", "h")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("gidPrefixFor", () => {
|
||||
it("returns the recovery-filter prefix", { timeout: 5000 }, () => {
|
||||
expect(gidPrefixFor("host-01")).toBe("2pc-host-01-");
|
||||
});
|
||||
});
|
||||
64
packages/api-stack/api/api-2pc/src/phases/gid.ts
Normal file
64
packages/api-stack/api/api-2pc/src/phases/gid.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Global transaction id helpers. Postgres' `PREPARE TRANSACTION '<gid>'`
|
||||
* accepts a string up to ~200 bytes. The coordinator's gid format is:
|
||||
*
|
||||
* `2pc-<hostInstanceId>-<txId>-<participantId>`
|
||||
*
|
||||
* — the host instance id is the recovery anchor (every prepared row
|
||||
* tagged with the host that wrote it; recovery filters on this prefix),
|
||||
* the txId is the coordinator-owned transaction id, the participantId
|
||||
* disambiguates per-participant rows for hosts that participate as more
|
||||
* than one logical actor.
|
||||
*
|
||||
* Embedded single quotes in any field are escaped by SQL-escaping the
|
||||
* gid at the prepared-statement layer — the coordinator never builds
|
||||
* the SQL with string concatenation against this value.
|
||||
*/
|
||||
|
||||
const PREFIX = "2pc";
|
||||
|
||||
/** Build the gid string for a given transaction + participant. */
|
||||
export function buildGid(
|
||||
hostInstanceId: string,
|
||||
txId: string,
|
||||
participantId: string
|
||||
): string {
|
||||
return `${PREFIX}-${hostInstanceId}-${txId}-${participantId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a gid back into its `txId` and `participantId` parts, given
|
||||
* the host instance id of the current coordinator. Returns `null`
|
||||
* when the gid does not match the expected prefix (foreign prepared
|
||||
* transactions sharing the host) — recovery skips those silently.
|
||||
*
|
||||
* `participantId` is constrained by the registry to never contain
|
||||
* hyphens, so the last hyphen unambiguously delimits it from
|
||||
* `txId`.
|
||||
*/
|
||||
export function parseGid(
|
||||
gid: string,
|
||||
hostInstanceId: string
|
||||
): {
|
||||
readonly txId: string;
|
||||
readonly participantId: string;
|
||||
} | null {
|
||||
const prefix = gidPrefixFor(hostInstanceId);
|
||||
if (!gid.startsWith(prefix)) return null;
|
||||
const remainder = gid.slice(prefix.length);
|
||||
const lastHyphen = remainder.lastIndexOf("-");
|
||||
if (lastHyphen === -1) return null;
|
||||
const participantId = remainder.slice(lastHyphen + 1);
|
||||
const txId = remainder.slice(0, lastHyphen);
|
||||
if (txId.length === 0 || participantId.length === 0) return null;
|
||||
return { txId, participantId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Predicate matching the prefix the recovery scan uses against
|
||||
* `pg_prepared_xacts`. Centralised so the SQL filter and parsing
|
||||
* logic never drift.
|
||||
*/
|
||||
export function gidPrefixFor(hostInstanceId: string): string {
|
||||
return `${PREFIX}-${hostInstanceId}-`;
|
||||
}
|
||||
15
packages/api-stack/api/api-2pc/src/phases/index.ts
Normal file
15
packages/api-stack/api/api-2pc/src/phases/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
export {
|
||||
type CommitPhaseInput,
|
||||
commitPhaseTwo,
|
||||
} from "./commit.js";
|
||||
export { buildGid, gidPrefixFor, parseGid } from "./gid.js";
|
||||
export {
|
||||
type ParticipantPrepareOutcome,
|
||||
type PreparePhaseInput,
|
||||
preparePhaseOne,
|
||||
} from "./prepare.js";
|
||||
export {
|
||||
isUnknownPreparedTransaction,
|
||||
type RollbackPhaseInput,
|
||||
rollbackPhaseTwo,
|
||||
} from "./rollback.js";
|
||||
105
packages/api-stack/api/api-2pc/src/phases/prepare.md
Normal file
105
packages/api-stack/api/api-2pc/src/phases/prepare.md
Normal file
@@ -0,0 +1,105 @@
|
||||
# `phases/prepare`
|
||||
|
||||
## Purpose
|
||||
|
||||
Implements phase 1 of the 2PC protocol for a single participant. The
|
||||
coordinator iterates participants and calls this module for each;
|
||||
unanimous commit votes trigger phase 2, any abort or prepare failure
|
||||
short-circuits to rollback. This module is the only place the
|
||||
`PREPARE TRANSACTION '<gid>'` SQL is emitted, so the gid format
|
||||
(`phases/gid`) and the SQL emission point stay co-located.
|
||||
|
||||
Phase 1 is the latency-sensitive half of 2PC: every participant holds
|
||||
row-level locks for the duration. The module records
|
||||
`2pc.prepare_duration_ms` so operators can spot creeping lock holds
|
||||
before they cause throughput regressions.
|
||||
|
||||
## Features
|
||||
|
||||
- Runs the participant's `prepare` callback, then the
|
||||
`PREPARE TRANSACTION` SQL when the vote is `commit` AND the
|
||||
participant is a writer.
|
||||
- Skips the SQL for read-only voters — they participate in the vote
|
||||
only.
|
||||
- Refuses a read-only voter that returned `writeIntent: true` with
|
||||
`TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`.
|
||||
- Honours an `AbortSignal` for the phase-1 deadline; aborted SQL is
|
||||
remapped to `TWOPC_TIMEOUT`.
|
||||
- Emits a `2pc.prepare` span around the whole call and records a
|
||||
`2pc.prepare_duration_ms` observation tagged with the outcome.
|
||||
- Failure cases preserve the underlying cause on the thrown
|
||||
`TwoPcError`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { preparePhaseOne } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "host-01",
|
||||
txId,
|
||||
participant,
|
||||
payload: { invoiceId: "inv-1" },
|
||||
instruments,
|
||||
signal: ac.signal,
|
||||
});
|
||||
if (outcome.vote === "abort") {
|
||||
// run rollback for every participant that voted commit
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The coordinator runs this in a loop. Read-only voters interleave
|
||||
freely with writers — the outcome's `prepared` flag tells phase 2
|
||||
whether to send commit/rollback SQL:
|
||||
|
||||
```ts
|
||||
import { preparePhaseOne } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
const outcomes: ParticipantPrepareOutcome[] = [];
|
||||
for (const p of registry.list()) {
|
||||
const o = await preparePhaseOne({
|
||||
hostInstanceId,
|
||||
txId,
|
||||
participant: p,
|
||||
payload: payloadFor(p),
|
||||
instruments,
|
||||
signal,
|
||||
});
|
||||
outcomes.push(o);
|
||||
if (o.vote === "abort") break; // short-circuit
|
||||
}
|
||||
|
||||
// phase 2: commit only those with prepared === true
|
||||
const writers = outcomes.filter((o) => o.prepared);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface ParticipantPrepareOutcome` — `{ participantId, vote,
|
||||
prepared, gid }`.
|
||||
- `interface PreparePhaseInput` — `{ hostInstanceId, txId,
|
||||
participant, payload, instruments, signal }`.
|
||||
- `function preparePhaseOne(input): Promise<ParticipantPrepareOutcome>`
|
||||
- throws `TWOPC_PREPARE_FAILED` when the participant callback or
|
||||
SQL raises (signal not aborted),
|
||||
- throws `TWOPC_TIMEOUT` when the signal aborted around the SQL,
|
||||
- throws `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE` when a read-only
|
||||
voter returned `writeIntent: true`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The SQL escape strategy is conservative: gids are built from the host
|
||||
instance id, transaction id, and participant id — all three are
|
||||
already validated at host config / id-generator / registry time — but
|
||||
the function still calls `.replace(/'/g, "''")` before interpolating
|
||||
into the `PREPARE TRANSACTION` statement. The runtime cost is zero on
|
||||
the happy path and the defence-in-depth is cheap. The coordinator
|
||||
never accepts user-supplied SQL fragments.
|
||||
|
||||
The span is started before the prepare callback so the participant's
|
||||
own work shows up in the trace as a child of the `2pc.prepare` span if
|
||||
the participant opens its own spans against the same tracer. The
|
||||
`finally`-block `span.end()` guarantees the span is closed regardless
|
||||
of which branch fires.
|
||||
252
packages/api-stack/api/api-2pc/src/phases/prepare.test.ts
Normal file
252
packages/api-stack/api/api-2pc/src/phases/prepare.test.ts
Normal file
@@ -0,0 +1,252 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { createTwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { preparePhaseOne } from "./prepare.js";
|
||||
|
||||
const INSTRUMENTS = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
|
||||
function makeConn(opts: { fail?: Error } = {}) {
|
||||
const calls: string[] = [];
|
||||
return {
|
||||
calls,
|
||||
connection: {
|
||||
engine: "postgres" as const,
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.fail) throw opts.fail;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeWriter(
|
||||
id: string,
|
||||
prepareImpl: RegisteredTwoPcParticipant["prepare"],
|
||||
connOpts: { fail?: Error } = {}
|
||||
): RegisteredTwoPcParticipant & { calls: string[] } {
|
||||
const c = makeConn(connOpts);
|
||||
return {
|
||||
id,
|
||||
voterMode: "writer",
|
||||
connection: c.connection,
|
||||
prepare: prepareImpl,
|
||||
calls: c.calls,
|
||||
};
|
||||
}
|
||||
|
||||
describe("preparePhaseOne", () => {
|
||||
it("emits PREPARE TRANSACTION on commit-voted writer and returns prepared=true", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p = makeWriter("billing", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}));
|
||||
const ac = new AbortController();
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: ac.signal,
|
||||
});
|
||||
expect(outcome.vote).toBe("commit");
|
||||
expect(outcome.prepared).toBe(true);
|
||||
expect(outcome.gid).toBe("2pc-h-t-billing");
|
||||
expect(p.calls[0]).toBe("PREPARE TRANSACTION '2pc-h-t-billing'");
|
||||
});
|
||||
|
||||
it("returns vote=abort without emitting SQL when participant votes abort", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p = makeWriter("billing", async () => ({
|
||||
vote: "abort",
|
||||
writeIntent: false,
|
||||
}));
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
expect(outcome.vote).toBe("abort");
|
||||
expect(outcome.prepared).toBe(false);
|
||||
expect(p.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns prepared=false for a read-only voter that voted commit", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p: RegisteredTwoPcParticipant & { calls: string[] } = {
|
||||
...makeWriter("audit", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: false,
|
||||
})),
|
||||
voterMode: "read-only",
|
||||
};
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
expect(outcome.vote).toBe("commit");
|
||||
expect(outcome.prepared).toBe(false);
|
||||
expect(p.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses a read-only voter that returns writeIntent=true", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p: RegisteredTwoPcParticipant & { calls: string[] } = {
|
||||
...makeWriter("rogue", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
})),
|
||||
voterMode: "read-only",
|
||||
};
|
||||
await expect(
|
||||
preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
})
|
||||
).rejects.toMatchObject({
|
||||
twoPcCode: TWOPC_CODES.READ_ONLY_PARTICIPANT_CANNOT_WRITE,
|
||||
});
|
||||
});
|
||||
|
||||
it("wraps a prepare-callback throw into TWOPC.PREPARE_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error("boom");
|
||||
const p = makeWriter("billing", async () => {
|
||||
throw cause;
|
||||
});
|
||||
try {
|
||||
await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
expect((err as TwoPcError).cause).toBe(cause);
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps a SQL throw into TWOPC.PREPARE_FAILED when not aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const sqlCause = new Error("sql failed");
|
||||
const p = makeWriter(
|
||||
"billing",
|
||||
async () => ({ vote: "commit", writeIntent: true }),
|
||||
{ fail: sqlCause }
|
||||
);
|
||||
try {
|
||||
await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PREPARE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("maps a SQL throw to TWOPC.TIMEOUT when the signal is aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
const p = makeWriter(
|
||||
"billing",
|
||||
async () => ({ vote: "commit", writeIntent: true }),
|
||||
{ fail: new Error("connection terminated") }
|
||||
);
|
||||
// Abort BEFORE the PREPARE TRANSACTION runs.
|
||||
const original = p.connection.run;
|
||||
p.connection.run = async (sql: string) => {
|
||||
ac.abort();
|
||||
return original(sql);
|
||||
};
|
||||
try {
|
||||
await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: ac.signal,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect((err as TwoPcError).twoPcCode).toBe(TWOPC_CODES.TIMEOUT);
|
||||
}
|
||||
});
|
||||
|
||||
it("raises TWOPC.TIMEOUT immediately when the signal was already aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const p = makeWriter("billing", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}));
|
||||
await expect(
|
||||
preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: ac.signal,
|
||||
})
|
||||
).rejects.toMatchObject({ twoPcCode: TWOPC_CODES.TIMEOUT });
|
||||
});
|
||||
|
||||
it("escapes single quotes in the gid before interpolation", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const p = makeWriter("bill'ing", async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}));
|
||||
const outcome = await preparePhaseOne({
|
||||
hostInstanceId: "h",
|
||||
txId: "t",
|
||||
participant: p,
|
||||
payload: {},
|
||||
instruments: INSTRUMENTS,
|
||||
signal: new AbortController().signal,
|
||||
});
|
||||
expect(outcome.gid).toBe("2pc-h-t-bill'ing");
|
||||
expect(p.calls[0]).toBe("PREPARE TRANSACTION '2pc-h-t-bill''ing'");
|
||||
});
|
||||
});
|
||||
160
packages/api-stack/api/api-2pc/src/phases/prepare.ts
Normal file
160
packages/api-stack/api/api-2pc/src/phases/prepare.ts
Normal file
@@ -0,0 +1,160 @@
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { TwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type {
|
||||
RegisteredTwoPcParticipant,
|
||||
TwoPcPrepareResult,
|
||||
} from "../types/types.types.js";
|
||||
import { buildGid } from "./gid.js";
|
||||
|
||||
/**
|
||||
* Outcome of phase-1 prepare for a single participant. `prepared`
|
||||
* tracks whether `PREPARE TRANSACTION '<gid>'` actually ran — the
|
||||
* rollback phase consults this to decide which participants need
|
||||
* `ROLLBACK PREPARED`.
|
||||
*/
|
||||
export interface ParticipantPrepareOutcome {
|
||||
readonly participantId: string;
|
||||
readonly vote: TwoPcVote;
|
||||
readonly prepared: boolean;
|
||||
/** SQL gid passed to PREPARE TRANSACTION — empty when no SQL ran. */
|
||||
readonly gid: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Input passed by the coordinator to a phase-1 run.
|
||||
*/
|
||||
export interface PreparePhaseInput {
|
||||
readonly hostInstanceId: string;
|
||||
readonly txId: string;
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
readonly payload: Readonly<Record<string, unknown>>;
|
||||
readonly instruments: TwoPcInstruments;
|
||||
/** Caller-driven AbortSignal — wired to the phase-1 deadline. */
|
||||
readonly signal: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run phase-1 for one participant. Three things happen, in order:
|
||||
*
|
||||
* 1. The participant's `prepare` callback runs its business work.
|
||||
* 2. If the participant voted `commit` AND is a writer, the coordinator
|
||||
* issues `PREPARE TRANSACTION '<gid>'` against the participant's
|
||||
* connection. Read-only voters skip this step.
|
||||
* 3. The vote and prepared-flag are returned to the coordinator.
|
||||
*
|
||||
* Failures map to {@link TWOPC_CODES} as follows:
|
||||
*
|
||||
* - read-only voter returned `writeIntent: true` →
|
||||
* `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`.
|
||||
* - prepare callback threw OR `PREPARE TRANSACTION` SQL threw →
|
||||
* `TWOPC_PREPARE_FAILED` (the cause is preserved).
|
||||
* - signal aborted during the SQL call → `TWOPC_TIMEOUT`.
|
||||
*/
|
||||
export async function preparePhaseOne(
|
||||
input: PreparePhaseInput
|
||||
): Promise<ParticipantPrepareOutcome> {
|
||||
const { hostInstanceId, txId, participant, payload, instruments, signal } =
|
||||
input;
|
||||
const span = instruments.tracer.startSpan("2pc.prepare", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.id": participant.id,
|
||||
"voter.mode": participant.voterMode,
|
||||
},
|
||||
});
|
||||
const t0 = Date.now();
|
||||
let outcome: TwoPcPrepareResult;
|
||||
try {
|
||||
try {
|
||||
outcome = await participant.prepare(txId, payload);
|
||||
} catch (cause) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PREPARE_FAILED,
|
||||
`Participant ${participant.id} prepare callback raised`,
|
||||
{
|
||||
cause,
|
||||
context: { participantId: participant.id, txId },
|
||||
}
|
||||
);
|
||||
}
|
||||
if (outcome.vote === "abort") {
|
||||
instruments.prepareDurationMs.record(Date.now() - t0, {
|
||||
outcome: "aborted",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
return {
|
||||
participantId: participant.id,
|
||||
vote: "abort",
|
||||
prepared: false,
|
||||
gid: "",
|
||||
};
|
||||
}
|
||||
if (
|
||||
participant.voterMode === "read-only" &&
|
||||
outcome.writeIntent === true
|
||||
) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.READ_ONLY_PARTICIPANT_CANNOT_WRITE,
|
||||
`Read-only voter ${participant.id} returned writeIntent=true`,
|
||||
{ context: { participantId: participant.id, txId } }
|
||||
);
|
||||
}
|
||||
if (participant.voterMode === "read-only") {
|
||||
instruments.prepareDurationMs.record(Date.now() - t0, {
|
||||
outcome: "committed",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
return {
|
||||
participantId: participant.id,
|
||||
vote: "commit",
|
||||
prepared: false,
|
||||
gid: "",
|
||||
};
|
||||
}
|
||||
if (signal.aborted) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT,
|
||||
`Phase-1 aborted before PREPARE TRANSACTION for ${participant.id}`,
|
||||
{ context: { participantId: participant.id, txId } }
|
||||
);
|
||||
}
|
||||
const gid = buildGid(hostInstanceId, txId, participant.id);
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (cause) {
|
||||
if (signal.aborted) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.TIMEOUT,
|
||||
`Phase-1 deadline expired while preparing ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: { participantId: participant.id, txId },
|
||||
}
|
||||
);
|
||||
}
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PREPARE_FAILED,
|
||||
`PREPARE TRANSACTION failed for ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: { participantId: participant.id, txId, gid },
|
||||
}
|
||||
);
|
||||
}
|
||||
instruments.prepareDurationMs.record(Date.now() - t0, {
|
||||
outcome: "committed",
|
||||
"participant.id": participant.id,
|
||||
});
|
||||
return {
|
||||
participantId: participant.id,
|
||||
vote: "commit",
|
||||
prepared: true,
|
||||
gid,
|
||||
};
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
92
packages/api-stack/api/api-2pc/src/phases/rollback.md
Normal file
92
packages/api-stack/api/api-2pc/src/phases/rollback.md
Normal file
@@ -0,0 +1,92 @@
|
||||
# `phases/rollback`
|
||||
|
||||
## Purpose
|
||||
|
||||
Phase-2 rollback for a single participant. Used both by the
|
||||
coordinator's abort path (vote_abort or prepare failure) and by the
|
||||
recovery runner (orphan with no `all-voted-commit` ledger row).
|
||||
|
||||
The function is idempotent at the protocol level: Postgres returns
|
||||
`42704` (`undefined_object`, "prepared transaction does not exist")
|
||||
for a gid that was already rolled back, and this module catches that
|
||||
specific error and treats it as a no-op. That property is what makes
|
||||
recovery safe to re-run on every boot.
|
||||
|
||||
## Features
|
||||
|
||||
- Emits `ROLLBACK PREPARED '<gid>'` against the participant's
|
||||
connection.
|
||||
- Idempotent against Postgres `42704` — a second rollback against the
|
||||
same gid is a no-op.
|
||||
- Skips participants that did not actually prepare (`outcome.prepared
|
||||
=== false`).
|
||||
- Emits a `2pc.rollback` span with `tx.id`, `participant.id`, and
|
||||
`gid` attributes.
|
||||
- Failure preserves the underlying cause on a thrown
|
||||
`TwoPcError(TWOPC_ROLLBACK_FAILED)`.
|
||||
- Exports `isUnknownPreparedTransaction(cause)` so the recovery
|
||||
runner can apply the same idempotency check without duplicating the
|
||||
SQLSTATE sniff.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { rollbackPhaseTwo } from "@suaveplan/api-2pc/phases";
|
||||
|
||||
await rollbackPhaseTwo({
|
||||
txId,
|
||||
participant,
|
||||
outcome: preparedOutcome,
|
||||
instruments,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Recovery uses the same idempotency check directly to short-circuit
|
||||
when the row was concurrently resolved:
|
||||
|
||||
```ts
|
||||
import {
|
||||
isUnknownPreparedTransaction,
|
||||
rollbackPhaseTwo,
|
||||
} from "@suaveplan/api-2pc/phases";
|
||||
|
||||
try {
|
||||
await rollbackPhaseTwo({ txId, participant, outcome, instruments });
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof TwoPcError &&
|
||||
err.twoPcCode === TWOPC_CODES.ROLLBACK_FAILED &&
|
||||
isUnknownPreparedTransaction(err.cause)
|
||||
) {
|
||||
// race: another process already rolled this back — treat as success
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface RollbackPhaseInput` — same fields as `CommitPhaseInput`.
|
||||
- `function rollbackPhaseTwo(input): Promise<void>`
|
||||
- no-op for `outcome.prepared === false`
|
||||
- throws `TwoPcError(TWOPC_ROLLBACK_FAILED)` on non-idempotency SQL
|
||||
failure.
|
||||
- `function isUnknownPreparedTransaction(cause: unknown): boolean`
|
||||
- matches SQLSTATE `42704` AND the substring `"does not exist"` as
|
||||
a fallback for drivers that omit the SQLSTATE.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The idempotency check sniffs both `cause.code` and `cause.message`
|
||||
because the two postgres clients in use (`postgres`/porsager and
|
||||
`pg.Client`) surface the SQLSTATE differently. Using both lets the
|
||||
module work against either driver without a wrapper layer.
|
||||
|
||||
The function is split into a small public `rollbackPhaseTwo` and an
|
||||
exported `isUnknownPreparedTransaction` helper so recovery can re-use
|
||||
the idempotency predicate without re-implementing it. Recovery sees
|
||||
this exact race regularly: the coordinator and the recovery runner
|
||||
both target the same prepared gid when a host restarts mid-rollback.
|
||||
145
packages/api-stack/api/api-2pc/src/phases/rollback.test.ts
Normal file
145
packages/api-stack/api/api-2pc/src/phases/rollback.test.ts
Normal file
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import { createTwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
import { isUnknownPreparedTransaction, rollbackPhaseTwo } from "./rollback.js";
|
||||
|
||||
const INSTRUMENTS = createTwoPcInstruments({
|
||||
outstandingPreparedSnapshot: () => 0,
|
||||
});
|
||||
|
||||
const OUTCOME: ParticipantPrepareOutcome = {
|
||||
participantId: "billing",
|
||||
vote: "commit",
|
||||
prepared: true,
|
||||
gid: "2pc-h-t-billing",
|
||||
};
|
||||
|
||||
function makeParticipant(opts: { fail?: unknown } = {}) {
|
||||
const calls: string[] = [];
|
||||
const p: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (opts.fail !== undefined) throw opts.fail;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
return { participant: p, calls };
|
||||
}
|
||||
|
||||
describe("rollbackPhaseTwo", () => {
|
||||
it("issues ROLLBACK PREPARED against a prepared writer", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls[0]).toBe("ROLLBACK PREPARED '2pc-h-t-billing'");
|
||||
});
|
||||
|
||||
it("is a no-op when outcome.prepared === false", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { participant, calls } = makeParticipant();
|
||||
await rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: { ...OUTCOME, prepared: false, gid: "" },
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("treats 42704 (undefined_object) as a no-op (idempotency)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = Object.assign(new Error("does not exist"), {
|
||||
code: "42704",
|
||||
});
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
await expect(
|
||||
rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("treats 'does not exist' message-only errors as a no-op", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error(
|
||||
"prepared transaction with identifier 'x' does not exist"
|
||||
);
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
await expect(
|
||||
rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
})
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("wraps a non-idempotency SQL throw into TWOPC.ROLLBACK_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cause = new Error("connection broken");
|
||||
const { participant } = makeParticipant({ fail: cause });
|
||||
try {
|
||||
await rollbackPhaseTwo({
|
||||
txId: "t",
|
||||
participant,
|
||||
outcome: OUTCOME,
|
||||
instruments: INSTRUMENTS,
|
||||
});
|
||||
throw new Error("should throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.ROLLBACK_FAILED
|
||||
);
|
||||
expect((err as TwoPcError).cause).toBe(cause);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("isUnknownPreparedTransaction", () => {
|
||||
it("returns true for SQLSTATE 42704", { timeout: 5000 }, () => {
|
||||
expect(isUnknownPreparedTransaction({ code: "42704" })).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for messages containing 'does not exist'", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
isUnknownPreparedTransaction({ message: "tx X does not exist" })
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for null / undefined / primitives", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(isUnknownPreparedTransaction(null)).toBe(false);
|
||||
expect(isUnknownPreparedTransaction(undefined)).toBe(false);
|
||||
expect(isUnknownPreparedTransaction(42)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for unrelated errors", { timeout: 5000 }, () => {
|
||||
expect(isUnknownPreparedTransaction(new Error("other"))).toBe(false);
|
||||
});
|
||||
});
|
||||
85
packages/api-stack/api/api-2pc/src/phases/rollback.ts
Normal file
85
packages/api-stack/api/api-2pc/src/phases/rollback.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { TwoPcInstruments } from "../instruments/instruments.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import type { ParticipantPrepareOutcome } from "./prepare.js";
|
||||
|
||||
/**
|
||||
* Input to {@link rollbackPhaseTwo}. Mirrors the commit-phase input
|
||||
* shape so the coordinator can swap between them without remapping.
|
||||
*/
|
||||
export interface RollbackPhaseInput {
|
||||
readonly txId: string;
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
readonly outcome: ParticipantPrepareOutcome;
|
||||
readonly instruments: TwoPcInstruments;
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase-2 rollback for a single participant. Issues
|
||||
* `ROLLBACK PREPARED '<gid>'`. Idempotent at the protocol level —
|
||||
* Postgres returns an error for a gid that does not exist in
|
||||
* `pg_prepared_xacts`, which the coordinator catches and treats as a
|
||||
* no-op (the row was already rolled back, e.g. by recovery).
|
||||
*
|
||||
* Failures other than "no such prepared transaction" throw
|
||||
* `TWOPC_ROLLBACK_FAILED` with the cause preserved.
|
||||
*/
|
||||
export async function rollbackPhaseTwo(
|
||||
input: RollbackPhaseInput
|
||||
): Promise<void> {
|
||||
const { txId, participant, outcome, instruments } = input;
|
||||
if (!outcome.prepared) return;
|
||||
const span = instruments.tracer.startSpan("2pc.rollback", {
|
||||
attributes: {
|
||||
"tx.id": txId,
|
||||
"participant.id": participant.id,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
});
|
||||
try {
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${outcome.gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (cause) {
|
||||
if (isUnknownPreparedTransaction(cause)) {
|
||||
return; // idempotent — already rolled back
|
||||
}
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.ROLLBACK_FAILED,
|
||||
`ROLLBACK PREPARED failed for ${participant.id}`,
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
participantId: participant.id,
|
||||
txId,
|
||||
gid: outcome.gid,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect Postgres' "prepared transaction with identifier ... does not
|
||||
* exist" error. The driver-level cause includes the SQLSTATE `42704`
|
||||
* (undefined_object) — we sniff the message and the code defensively
|
||||
* so the check works against both `postgres` (porsager/postgres) and
|
||||
* `pg.Client`. The fallback to message-substring lookup catches the
|
||||
* uncommon case where a driver does not surface the SQLSTATE.
|
||||
*/
|
||||
export function isUnknownPreparedTransaction(cause: unknown): boolean {
|
||||
if (cause === null || typeof cause !== "object") return false;
|
||||
const errObj = cause as { code?: unknown; message?: unknown };
|
||||
if (errObj.code === "42704") return true;
|
||||
if (
|
||||
typeof errObj.message === "string" &&
|
||||
errObj.message.includes("does not exist")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
7
packages/api-stack/api/api-2pc/src/recovery/index.ts
Normal file
7
packages/api-stack/api/api-2pc/src/recovery/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
buildProbeFromRegistry,
|
||||
type RecoveryInput,
|
||||
type RecoveryProbe,
|
||||
type RecoveryReport,
|
||||
resolveRecovery,
|
||||
} from "./recovery.js";
|
||||
133
packages/api-stack/api/api-2pc/src/recovery/recovery.md
Normal file
133
packages/api-stack/api/api-2pc/src/recovery/recovery.md
Normal file
@@ -0,0 +1,133 @@
|
||||
# `recovery/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Boot-time orphan resolution for the 2PC coordinator. When a host
|
||||
crashes between phase 1 and phase 2, prepared transactions remain
|
||||
held in `pg_prepared_xacts` on every writer participant. The recovery
|
||||
runner sweeps those rows on the next boot, consults the host-owned
|
||||
ledger, and either commits or rolls back each one — guaranteeing the
|
||||
"once prepared, eventually resolved" half of the 2PC durability
|
||||
property.
|
||||
|
||||
The runner is the only path that should ever resolve an in-doubt
|
||||
transaction. The coordinator's hot path NEVER auto-recovers; it
|
||||
escalates to `TWOPC_IN_DOUBT` and lets the next-boot recovery sweep
|
||||
handle it. That separation keeps the hot path's failure handling
|
||||
small and the recovery path testable in isolation.
|
||||
|
||||
## Features
|
||||
|
||||
- `resolveRecovery({ hostInstanceId, ledger, probes, onResolved })` —
|
||||
the entry point. Returns a `RecoveryReport` enumerating committed,
|
||||
rolled-back, skipped, and errored gids.
|
||||
- Idempotent — re-running the function on a host with no outstanding
|
||||
prepared transactions is a no-op (the inner `queryOrphans` returns
|
||||
an empty list).
|
||||
- Treats Postgres `42704` ("does not exist") as success during
|
||||
rollback — covers the race where another process already resolved
|
||||
the orphan.
|
||||
- Marks rows as `in_doubt` on unresolvable errors so the next sweep
|
||||
can retry; errored gids are surfaced on the report rather than
|
||||
thrown.
|
||||
- `buildProbeFromRegistry(registry)` — convenience adapter that
|
||||
spins up one probe per writer participant.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import {
|
||||
buildProbeFromRegistry,
|
||||
resolveRecovery,
|
||||
} from "@suaveplan/api-2pc/recovery";
|
||||
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "host-01",
|
||||
ledger,
|
||||
probes: buildProbeFromRegistry(registry),
|
||||
});
|
||||
|
||||
console.log(
|
||||
`committed=${report.committed.length} ` +
|
||||
`rolledBack=${report.rolledBack.length} ` +
|
||||
`errors=${report.errors.length}`
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Wire a custom probe to feed an alternate identifier convention or a
|
||||
replicated-read connection — useful when the recovery sweep should
|
||||
hit a read replica to avoid contending with the live writer:
|
||||
|
||||
```ts
|
||||
import { gidPrefixFor } from "@suaveplan/api-2pc/phases";
|
||||
import { resolveRecovery, type RecoveryProbe } from "@suaveplan/api-2pc/recovery";
|
||||
|
||||
const probes: RecoveryProbe[] = [
|
||||
{
|
||||
participantId: "billing",
|
||||
async queryOrphans(prefix) {
|
||||
const rows = await readReplica.unsafe<{ gid: string }[]>(
|
||||
`SELECT gid FROM pg_prepared_xacts WHERE gid LIKE $1`,
|
||||
[`${prefix}%`]
|
||||
);
|
||||
return rows.map((r) => r.gid);
|
||||
},
|
||||
async commit(gid) {
|
||||
await primary.unsafe(`COMMIT PREPARED $1`, [gid]);
|
||||
},
|
||||
async rollback(gid) {
|
||||
await primary.unsafe(`ROLLBACK PREPARED $1`, [gid]);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "host-01",
|
||||
ledger,
|
||||
probes,
|
||||
onResolved: (e) => log.info(e),
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface RecoveryProbe`
|
||||
- `participantId: string`
|
||||
- `queryOrphans(prefix: string): Promise<readonly string[]>`
|
||||
- `commit(gid: string): Promise<void>`
|
||||
- `rollback(gid: string): Promise<void>`
|
||||
- `interface RecoveryReport`
|
||||
- `committed: readonly string[]`
|
||||
- `rolledBack: readonly string[]`
|
||||
- `skipped: readonly string[]`
|
||||
- `errors: readonly { gid; error: TwoPcError }[]`
|
||||
- `interface RecoveryInput`
|
||||
- `hostInstanceId, ledger, probes, onResolved?`
|
||||
- `function resolveRecovery(input): Promise<RecoveryReport>`
|
||||
- `function buildProbeFromRegistry(registry): readonly RecoveryProbe[]`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The ledger lookup uses the gid's parsed `txId` as the primary key —
|
||||
the gid format from `phases/gid.ts` is what makes that mapping
|
||||
unambiguous. Rows whose ledger state is `committing` OR `committed`
|
||||
trigger `COMMIT PREPARED`; everything else (including missing rows)
|
||||
triggers `ROLLBACK PREPARED`. That asymmetric default ("rollback by
|
||||
default") is the safe choice — committing a row whose ledger does
|
||||
not explicitly say "all voted commit" would violate the design
|
||||
guarantee.
|
||||
|
||||
The `42704` swallow during rollback is what makes the runner safe to
|
||||
re-execute: if the previous sweep crashed AFTER rolling back the
|
||||
prepared transaction but BEFORE updating the ledger, the second
|
||||
sweep will see the orphan listed in `pg_prepared_xacts` (or not, if
|
||||
truly gone) and will not throw on the "does not exist" follow-up.
|
||||
Errors that cannot be classified as idempotency are captured per-gid
|
||||
and surfaced on the report — operators page-on `errors.length > 0`,
|
||||
not on a generic throw.
|
||||
308
packages/api-stack/api/api-2pc/src/recovery/recovery.test.ts
Normal file
308
packages/api-stack/api/api-2pc/src/recovery/recovery.test.ts
Normal file
@@ -0,0 +1,308 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES } from "../error-codes/error-codes.js";
|
||||
import {
|
||||
type LedgerDb,
|
||||
type LedgerParticipantEntry,
|
||||
TwoPcLedgerStore,
|
||||
} from "../persistence/persistence.js";
|
||||
import { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import {
|
||||
buildProbeFromRegistry,
|
||||
type RecoveryProbe,
|
||||
resolveRecovery,
|
||||
} from "./recovery.js";
|
||||
|
||||
function makeLedger(): TwoPcLedgerStore {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
return new TwoPcLedgerStore(db);
|
||||
}
|
||||
|
||||
const SAMPLE: ReadonlyArray<LedgerParticipantEntry> = [
|
||||
{ participantId: "billing", voterMode: "writer", vote: "commit" },
|
||||
];
|
||||
|
||||
function makeProbe(opts: {
|
||||
orphans?: readonly string[];
|
||||
failCommit?: Error;
|
||||
failRollback?: Error;
|
||||
}): RecoveryProbe & { committed: string[]; rolledBack: string[] } {
|
||||
const committed: string[] = [];
|
||||
const rolledBack: string[] = [];
|
||||
return {
|
||||
participantId: "billing",
|
||||
committed,
|
||||
rolledBack,
|
||||
async queryOrphans(_prefix) {
|
||||
return opts.orphans ?? [];
|
||||
},
|
||||
async commit(gid) {
|
||||
if (opts.failCommit) throw opts.failCommit;
|
||||
committed.push(gid);
|
||||
},
|
||||
async rollback(gid) {
|
||||
if (opts.failRollback) throw opts.failRollback;
|
||||
rolledBack.push(gid);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("resolveRecovery", () => {
|
||||
it("commits orphans whose ledger state is committing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-1",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
ledger.markAllVotedCommit({
|
||||
id: "tx-1",
|
||||
participants: SAMPLE,
|
||||
preparedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
const probe = makeProbe({ orphans: ["2pc-h-tx-1-billing"] });
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.committed).toEqual(["2pc-h-tx-1-billing"]);
|
||||
expect(report.rolledBack).toEqual([]);
|
||||
expect(probe.committed).toEqual(["2pc-h-tx-1-billing"]);
|
||||
expect(ledger.findById("tx-1")?.state).toBe("committed");
|
||||
});
|
||||
|
||||
it("rolls back orphans whose ledger row is missing (safe default)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const probe = makeProbe({ orphans: ["2pc-h-tx-2-billing"] });
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.rolledBack).toEqual(["2pc-h-tx-2-billing"]);
|
||||
expect(report.committed).toEqual([]);
|
||||
expect(probe.rolledBack).toEqual(["2pc-h-tx-2-billing"]);
|
||||
});
|
||||
|
||||
it("rolls back orphans whose ledger state is voting (not yet all-voted-commit)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-3",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
const probe = makeProbe({ orphans: ["2pc-h-tx-3-billing"] });
|
||||
await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(ledger.findById("tx-3")?.state).toBe("aborted");
|
||||
});
|
||||
|
||||
it("treats Postgres 42704 during rollback as success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const cause = Object.assign(new Error("does not exist"), {
|
||||
code: "42704",
|
||||
});
|
||||
const probe = makeProbe({
|
||||
orphans: ["2pc-h-tx-4-billing"],
|
||||
failRollback: cause,
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.errors).toEqual([]);
|
||||
expect(report.rolledBack).toEqual(["2pc-h-tx-4-billing"]);
|
||||
});
|
||||
|
||||
it("skips foreign gids that don't match the host prefix", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const probe = makeProbe({
|
||||
orphans: ["2pc-other-host-tx-5-billing", "foreign-gid"],
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.skipped).toContain("2pc-other-host-tx-5-billing");
|
||||
expect(report.skipped).toContain("foreign-gid");
|
||||
expect(report.committed).toEqual([]);
|
||||
expect(report.rolledBack).toEqual([]);
|
||||
});
|
||||
|
||||
it("marks irrecoverable orphans in_doubt and surfaces them on the report", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-6",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
ledger.markAllVotedCommit({
|
||||
id: "tx-6",
|
||||
participants: SAMPLE,
|
||||
preparedAt: "2026-05-26T00:00:00.000Z",
|
||||
});
|
||||
const cause = new Error("connection broken");
|
||||
const probe = makeProbe({
|
||||
orphans: ["2pc-h-tx-6-billing"],
|
||||
failCommit: cause,
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(report.errors).toHaveLength(1);
|
||||
expect(report.errors[0]?.error.twoPcCode).toBe(TWOPC_CODES.IN_DOUBT);
|
||||
expect(ledger.findById("tx-6")?.state).toBe("in_doubt");
|
||||
});
|
||||
|
||||
it("invokes onResolved for every outcome", { timeout: 5000 }, async () => {
|
||||
const ledger = makeLedger();
|
||||
ledger.insert({
|
||||
id: "tx-7",
|
||||
coordinator: "h",
|
||||
participants: SAMPLE,
|
||||
});
|
||||
const events: Array<{ gid: string; outcome: string }> = [];
|
||||
await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [makeProbe({ orphans: ["2pc-h-tx-7-billing", "foreign"] })],
|
||||
onResolved: (e) => events.push({ ...e }),
|
||||
});
|
||||
expect(events.length).toBe(2);
|
||||
expect(events.some((e) => e.outcome === "rolledBack")).toBe(true);
|
||||
expect(events.some((e) => e.outcome === "skipped")).toBe(true);
|
||||
});
|
||||
|
||||
it("re-running recovery on a clean state is a no-op", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const ledger = makeLedger();
|
||||
const probe = makeProbe({ orphans: [] });
|
||||
const r1 = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
const r2 = await resolveRecovery({
|
||||
hostInstanceId: "h",
|
||||
ledger,
|
||||
probes: [probe],
|
||||
});
|
||||
expect(r1.committed.length).toBe(0);
|
||||
expect(r1.rolledBack.length).toBe(0);
|
||||
expect(r2.committed.length).toBe(0);
|
||||
expect(r2.rolledBack.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildProbeFromRegistry", () => {
|
||||
it("emits one probe per writer participant; skips read-only voters", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
const writer: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
const reader: RegisteredTwoPcParticipant = {
|
||||
id: "audit",
|
||||
voterMode: "read-only",
|
||||
connection: {
|
||||
engine: "surrealdb",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: false }),
|
||||
};
|
||||
registry.register(writer);
|
||||
registry.register(reader);
|
||||
const probes = buildProbeFromRegistry(registry);
|
||||
expect(probes).toHaveLength(1);
|
||||
expect(probes[0]?.participantId).toBe("billing");
|
||||
});
|
||||
|
||||
it("emitted probe runs the documented pg_prepared_xacts query", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const calls: string[] = [];
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
calls.push(sql);
|
||||
if (sql.includes("SELECT gid"))
|
||||
return [{ gid: "2pc-h-tx-1-billing" }];
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
});
|
||||
const probes = buildProbeFromRegistry(registry);
|
||||
const orphans = await probes[0]?.queryOrphans("2pc-h-");
|
||||
expect(orphans).toEqual(["2pc-h-tx-1-billing"]);
|
||||
expect(calls[0]).toContain("SELECT gid FROM pg_prepared_xacts");
|
||||
expect(calls[0]).toContain("LIKE '2pc-h-%'");
|
||||
await probes[0]?.commit("2pc-h-tx-1-billing");
|
||||
await probes[0]?.rollback("2pc-h-tx-1-billing");
|
||||
expect(calls).toEqual([
|
||||
"SELECT gid FROM pg_prepared_xacts WHERE gid LIKE '2pc-h-%'",
|
||||
"COMMIT PREPARED '2pc-h-tx-1-billing'",
|
||||
"ROLLBACK PREPARED '2pc-h-tx-1-billing'",
|
||||
]);
|
||||
});
|
||||
|
||||
it("queryOrphans handles a probe that returns undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
});
|
||||
const probes = buildProbeFromRegistry(registry);
|
||||
const orphans = await probes[0]?.queryOrphans("2pc-h-");
|
||||
expect(orphans).toEqual([]);
|
||||
});
|
||||
});
|
||||
196
packages/api-stack/api/api-2pc/src/recovery/recovery.ts
Normal file
196
packages/api-stack/api/api-2pc/src/recovery/recovery.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type {
|
||||
TwoPcLedgerRow,
|
||||
TwoPcLedgerStore,
|
||||
} from "../persistence/persistence.js";
|
||||
import { gidPrefixFor, parseGid } from "../phases/gid.js";
|
||||
import { isUnknownPreparedTransaction } from "../phases/rollback.js";
|
||||
import type { TwoPcParticipantRegistry } from "../registry/participant-registry.js";
|
||||
|
||||
/**
|
||||
* Per-participant probe surface — pings a single Postgres connection
|
||||
* for prepared transactions matching the host's gid prefix. The
|
||||
* coordinator iterates registered writers and asks each one for its
|
||||
* orphans on boot.
|
||||
*/
|
||||
export interface RecoveryProbe {
|
||||
readonly participantId: string;
|
||||
queryOrphans(prefix: string): Promise<readonly string[]>;
|
||||
commit(gid: string): Promise<void>;
|
||||
rollback(gid: string): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of running recovery. `committed` and `rolledBack` carry the
|
||||
* exact gids resolved on this boot; `skipped` carries gids the
|
||||
* coordinator could not parse (foreign prepared transactions sharing
|
||||
* the host).
|
||||
*/
|
||||
export interface RecoveryReport {
|
||||
readonly committed: readonly string[];
|
||||
readonly rolledBack: readonly string[];
|
||||
readonly skipped: readonly string[];
|
||||
readonly errors: ReadonlyArray<{
|
||||
readonly gid: string;
|
||||
readonly error: TwoPcError;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Input to {@link resolveRecovery}.
|
||||
*/
|
||||
export interface RecoveryInput {
|
||||
readonly hostInstanceId: string;
|
||||
readonly ledger: TwoPcLedgerStore;
|
||||
/**
|
||||
* Per-participant Postgres probes. The recovery runner does not
|
||||
* care which participants are registered — it sweeps every probe.
|
||||
*/
|
||||
readonly probes: ReadonlyArray<RecoveryProbe>;
|
||||
/**
|
||||
* Optional callback fired for every resolved gid; useful for
|
||||
* progress logging in tests and ops dashboards.
|
||||
*/
|
||||
readonly onResolved?: (event: {
|
||||
readonly gid: string;
|
||||
readonly outcome: "committed" | "rolledBack" | "skipped";
|
||||
}) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Boot-time orphan resolution. Per design doc §9.2 paragraph 3:
|
||||
*
|
||||
* 1. Query every participant's `pg_prepared_xacts` for gids matching
|
||||
* this host's prefix.
|
||||
* 2. For each orphan: if the ledger row records `all-voted-commit`
|
||||
* (state `committing` or `committed`) issue `COMMIT PREPARED`;
|
||||
* otherwise issue `ROLLBACK PREPARED`. Missing ledger row → safe
|
||||
* default `ROLLBACK PREPARED` (no commit without explicit log).
|
||||
* 3. Update the ledger row to a terminal state. Idempotent — running
|
||||
* recovery twice in a row is a no-op once orphans are resolved.
|
||||
*
|
||||
* Errors are collected per-gid rather than thrown — the runner makes
|
||||
* progress on every recoverable orphan and surfaces unresolved ones
|
||||
* to the caller for follow-up.
|
||||
*/
|
||||
export async function resolveRecovery(
|
||||
input: RecoveryInput
|
||||
): Promise<RecoveryReport> {
|
||||
const { hostInstanceId, ledger, probes, onResolved } = input;
|
||||
const prefix = gidPrefixFor(hostInstanceId);
|
||||
const committed: string[] = [];
|
||||
const rolledBack: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
const errors: Array<{ gid: string; error: TwoPcError }> = [];
|
||||
const nowIso = () => new Date().toISOString();
|
||||
|
||||
for (const probe of probes) {
|
||||
const orphans = await probe.queryOrphans(prefix);
|
||||
for (const gid of orphans) {
|
||||
const parts = parseGid(gid, hostInstanceId);
|
||||
if (parts === null) {
|
||||
skipped.push(gid);
|
||||
onResolved?.({ gid, outcome: "skipped" });
|
||||
continue;
|
||||
}
|
||||
const row = ledger.findById(parts.txId);
|
||||
const shouldCommit =
|
||||
row !== null &&
|
||||
(row.state === "committing" || row.state === "committed");
|
||||
try {
|
||||
if (shouldCommit) {
|
||||
await probe.commit(gid);
|
||||
finaliseLedgerForCommit(ledger, row, nowIso);
|
||||
committed.push(gid);
|
||||
onResolved?.({ gid, outcome: "committed" });
|
||||
} else {
|
||||
try {
|
||||
await probe.rollback(gid);
|
||||
} catch (cause) {
|
||||
if (!isUnknownPreparedTransaction(cause)) throw cause;
|
||||
}
|
||||
finaliseLedgerForAbort(ledger, row, nowIso);
|
||||
rolledBack.push(gid);
|
||||
onResolved?.({ gid, outcome: "rolledBack" });
|
||||
}
|
||||
} catch (cause) {
|
||||
if (row !== null) ledger.markInDoubt(row.id);
|
||||
errors.push({
|
||||
gid,
|
||||
error: new TwoPcError(
|
||||
TWOPC_CODES.IN_DOUBT,
|
||||
`Recovery could not resolve orphan ${gid}`,
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
gid,
|
||||
participantId: probe.participantId,
|
||||
},
|
||||
}
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { committed, rolledBack, skipped, errors };
|
||||
}
|
||||
|
||||
function finaliseLedgerForCommit(
|
||||
ledger: TwoPcLedgerStore,
|
||||
row: TwoPcLedgerRow | null,
|
||||
nowIso: () => string
|
||||
): void {
|
||||
if (row === null) return;
|
||||
if (row.state === "committed") return;
|
||||
ledger.markCommitted(row.id, nowIso());
|
||||
}
|
||||
|
||||
function finaliseLedgerForAbort(
|
||||
ledger: TwoPcLedgerStore,
|
||||
row: TwoPcLedgerRow | null,
|
||||
nowIso: () => string
|
||||
): void {
|
||||
if (row === null) return;
|
||||
if (row.state === "aborted") return;
|
||||
ledger.markAborted(row.id, nowIso());
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience adapter: builds a {@link RecoveryProbe} backed by a
|
||||
* registered participant. The participant's connection is used for
|
||||
* the `pg_prepared_xacts` scan AND the COMMIT/ROLLBACK PREPARED SQL;
|
||||
* the coordinator never asks a participant to manipulate another
|
||||
* participant's prepared transactions.
|
||||
*/
|
||||
export function buildProbeFromRegistry(
|
||||
registry: TwoPcParticipantRegistry
|
||||
): readonly RecoveryProbe[] {
|
||||
const probes: RecoveryProbe[] = [];
|
||||
for (const p of registry.list()) {
|
||||
if (p.voterMode === "read-only") continue;
|
||||
const conn = p.connection;
|
||||
probes.push({
|
||||
participantId: p.id,
|
||||
async queryOrphans(prefix) {
|
||||
const rows = (await conn.run<ReadonlyArray<{ gid: string }>>(
|
||||
`SELECT gid FROM pg_prepared_xacts WHERE gid LIKE ${escapeLiteral(`${prefix}%`)}`
|
||||
)) as ReadonlyArray<{ gid: string }> | undefined;
|
||||
return (rows ?? []).map((r) => r.gid);
|
||||
},
|
||||
async commit(gid) {
|
||||
await conn.run(`COMMIT PREPARED '${gid.replace(/'/g, "''")}'`);
|
||||
},
|
||||
async rollback(gid) {
|
||||
await conn.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
return probes;
|
||||
}
|
||||
|
||||
function escapeLiteral(s: string): string {
|
||||
return `'${s.replace(/'/g, "''")}'`;
|
||||
}
|
||||
4
packages/api-stack/api/api-2pc/src/registry/index.ts
Normal file
4
packages/api-stack/api/api-2pc/src/registry/index.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
type PreparableCapabilityCheck,
|
||||
TwoPcParticipantRegistry,
|
||||
} from "./participant-registry.js";
|
||||
@@ -0,0 +1,111 @@
|
||||
# `registry/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
The participant registry is the install-time admission control between
|
||||
plugins and the 2PC coordinator. Plugins declare
|
||||
`twoPhaseCommit.participates: true` in their manifest; the host
|
||||
registers them with this registry; the registry refuses any writer
|
||||
whose bound database engine cannot honour `PREPARE TRANSACTION`.
|
||||
|
||||
This is where the design doc §9.2 rule "SQLite and SurrealDB MUST NOT
|
||||
participate as writers" is enforced. The capability check is injected
|
||||
as a callback so the registry stays decoupled from
|
||||
`@suaveplan/api-host`'s concrete `CapabilityRegistry`.
|
||||
|
||||
## Features
|
||||
|
||||
- `TwoPcParticipantRegistry.register(participant)` — refuses writers
|
||||
without `db.preparable`; accepts read-only voters regardless of
|
||||
engine.
|
||||
- `get(id)` / `list()` / `size()` / `unregister(id)` — standard
|
||||
registry surface.
|
||||
- Injection seam: `PreparableCapabilityCheck` is a plain
|
||||
`(id: string) => boolean` callback — zero coupling to api-host.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { TwoPcParticipantRegistry } from "@suaveplan/api-2pc/registry";
|
||||
|
||||
const isPreparable = (pluginId: string) =>
|
||||
pluginId === "billing" || pluginId === "inventory";
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(isPreparable);
|
||||
registry.register({
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: { engine: "postgres", run: pg.unsafe.bind(pg) },
|
||||
prepare: billingPrepareCallback,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Refusing a SQLite-backed plugin as a writer happens at register-time
|
||||
— wrap the registration in a try/catch to surface the failure to the
|
||||
plugin loader without crashing the host boot:
|
||||
|
||||
```ts
|
||||
import { TWOPC_CODES, TwoPcError } from "@suaveplan/api-2pc/error-codes";
|
||||
import { TwoPcParticipantRegistry } from "@suaveplan/api-2pc/registry";
|
||||
|
||||
function safeRegisterParticipants(
|
||||
registry: TwoPcParticipantRegistry,
|
||||
participants: ReadonlyArray<RegisteredTwoPcParticipant>
|
||||
) {
|
||||
const failures: { id: string; reason: string }[] = [];
|
||||
for (const p of participants) {
|
||||
try {
|
||||
registry.register(p);
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof TwoPcError &&
|
||||
err.twoPcCode === TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE
|
||||
) {
|
||||
failures.push({ id: p.id, reason: err.message });
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
```
|
||||
|
||||
The same plugin can then re-register as a `read-only` voter — the
|
||||
registry accepts read-only voters regardless of engine, because the
|
||||
coordinator never sends them prepare/commit SQL.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type PreparableCapabilityCheck = (participantId: string) => boolean`
|
||||
— capability resolver injected at construction.
|
||||
- `class TwoPcParticipantRegistry`
|
||||
- `constructor(isPreparable: PreparableCapabilityCheck)`
|
||||
- `register(participant): void` — throws
|
||||
`TWOPC_PARTICIPANT_NOT_PREPARABLE` on writer refusal.
|
||||
- `get(id): RegisteredTwoPcParticipant | undefined`
|
||||
- `list(): readonly RegisteredTwoPcParticipant[]`
|
||||
- `size(): number`
|
||||
- `unregister(id): boolean`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The registry refuses a writer at `register`-time rather than at
|
||||
`run2pc`-time. This is deliberate — admission failures should surface
|
||||
during plugin install, not when the first 2PC transaction is attempted
|
||||
in production. The capability check fires only for writers; read-only
|
||||
voters are accepted unconditionally because the coordinator drops their
|
||||
connection from phase-2 SQL traffic. The second-half guard against
|
||||
read-only voters that try to write a payload lives in
|
||||
`phases/prepare.ts` (raising `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`
|
||||
when the participant returns `writeIntent: true`).
|
||||
|
||||
`unregister` exists for hot-reload paths: when a plugin is removed
|
||||
from the running host, the registry drops its participant entry so
|
||||
subsequent `run2pc` calls cannot accidentally route to it.
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { TwoPcParticipantRegistry } from "./participant-registry.js";
|
||||
|
||||
const NOOP_PREPARE = async () => ({
|
||||
vote: "commit" as const,
|
||||
writeIntent: false,
|
||||
});
|
||||
|
||||
function makeParticipant(
|
||||
id: string,
|
||||
overrides: Partial<RegisteredTwoPcParticipant> = {}
|
||||
): RegisteredTwoPcParticipant {
|
||||
return {
|
||||
id,
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async () => undefined,
|
||||
},
|
||||
prepare: NOOP_PREPARE,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("TwoPcParticipantRegistry", () => {
|
||||
it("registers preparable writers", { timeout: 5000 }, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => true);
|
||||
r.register(makeParticipant("billing"));
|
||||
expect(r.size()).toBe(1);
|
||||
expect(r.get("billing")?.id).toBe("billing");
|
||||
});
|
||||
|
||||
it("rejects writer whose engine is not preparable with TWOPC.PARTICIPANT_NOT_PREPARABLE", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => false);
|
||||
try {
|
||||
r.register(
|
||||
makeParticipant("sqlite-plugin", {
|
||||
connection: {
|
||||
engine: "sqlite",
|
||||
run: async () => undefined,
|
||||
},
|
||||
})
|
||||
);
|
||||
throw new Error("should have thrown");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(TwoPcError);
|
||||
expect((err as TwoPcError).twoPcCode).toBe(
|
||||
TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts read-only voter regardless of engine", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => false);
|
||||
r.register(
|
||||
makeParticipant("audit", {
|
||||
voterMode: "read-only",
|
||||
connection: {
|
||||
engine: "surrealdb",
|
||||
run: async () => undefined,
|
||||
},
|
||||
})
|
||||
);
|
||||
expect(r.size()).toBe(1);
|
||||
expect(r.get("audit")?.voterMode).toBe("read-only");
|
||||
});
|
||||
|
||||
it("lists every registered participant", { timeout: 5000 }, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => true);
|
||||
r.register(makeParticipant("a"));
|
||||
r.register(makeParticipant("b"));
|
||||
expect(
|
||||
r
|
||||
.list()
|
||||
.map((p) => p.id)
|
||||
.sort()
|
||||
).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("unregister removes the entry and reports outcome", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const r = new TwoPcParticipantRegistry(() => true);
|
||||
r.register(makeParticipant("c"));
|
||||
expect(r.unregister("c")).toBe(true);
|
||||
expect(r.unregister("c")).toBe(false);
|
||||
expect(r.size()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { TWOPC_CODES, TwoPcError } from "../error-codes/error-codes.js";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Capability assertion handed to the registry. The host's capability
|
||||
* registry implements this — the registry only needs to know whether
|
||||
* a specific participant's engine is `db.preparable`. Modelling this
|
||||
* as a structural callback keeps the registry free of any direct
|
||||
* dependency on `@suaveplan/api-host`.
|
||||
*/
|
||||
export type PreparableCapabilityCheck = (participantId: string) => boolean;
|
||||
|
||||
/**
|
||||
* Registry of participants the coordinator can hand a transaction to.
|
||||
* Refuses writers whose bound engine is not `db.preparable` (SQLite,
|
||||
* SurrealDB). Read-only voters are allowed regardless of engine but
|
||||
* are refused later if their prepare callback returns
|
||||
* `writeIntent: true` — that refusal lives in `phases/prepare.ts`.
|
||||
*/
|
||||
export class TwoPcParticipantRegistry {
|
||||
private readonly entries = new Map<string, RegisteredTwoPcParticipant>();
|
||||
|
||||
public constructor(
|
||||
private readonly isPreparable: PreparableCapabilityCheck
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Add a participant. Throws
|
||||
* `TWOPC_PARTICIPANT_NOT_PREPARABLE` when a writer's engine does
|
||||
* not advertise the `db.preparable` capability.
|
||||
*/
|
||||
public register(participant: RegisteredTwoPcParticipant): void {
|
||||
if (
|
||||
participant.voterMode === "writer" &&
|
||||
!this.isPreparable(participant.id)
|
||||
) {
|
||||
throw new TwoPcError(
|
||||
TWOPC_CODES.PARTICIPANT_NOT_PREPARABLE,
|
||||
`Participant ${participant.id} is not preparable (engine: ${participant.connection.engine})`,
|
||||
{
|
||||
context: {
|
||||
participantId: participant.id,
|
||||
engine: participant.connection.engine,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
this.entries.set(participant.id, participant);
|
||||
}
|
||||
|
||||
/** Lookup by id; returns `undefined` when none registered. */
|
||||
public get(id: string): RegisteredTwoPcParticipant | undefined {
|
||||
return this.entries.get(id);
|
||||
}
|
||||
|
||||
/** Snapshot of every registered participant. */
|
||||
public list(): readonly RegisteredTwoPcParticipant[] {
|
||||
return Array.from(this.entries.values());
|
||||
}
|
||||
|
||||
/** Total registered participants — useful for diagnostic assertions. */
|
||||
public size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
/** Remove a participant; returns `true` when one was removed. */
|
||||
public unregister(id: string): boolean {
|
||||
return this.entries.delete(id);
|
||||
}
|
||||
}
|
||||
100
packages/api-stack/api/api-2pc/src/testing/conformance.md
Normal file
100
packages/api-stack/api/api-2pc/src/testing/conformance.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# `testing/`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides `run2pcParticipantConformance(factory)` — a runnable test
|
||||
suite that pins the contract every 2PC writer participant MUST honour.
|
||||
Adapter authors drop a single import + invocation into a
|
||||
`tests/conformance/run-2pc-participant.test.ts` file in their package;
|
||||
the harness handles the rest.
|
||||
|
||||
Shipping the harness on this package's `./testing` subpath keeps
|
||||
participants accountable to the same prepare/commit/rollback semantics
|
||||
the coordinator depends on at runtime. Without this harness, drift
|
||||
between participant implementations would only surface during
|
||||
integration testing — too late.
|
||||
|
||||
## Features
|
||||
|
||||
- Six pinned invariants (engine identifier, voter mode, prepare-then-
|
||||
prepare-SQL, commit-prepared, rollback-prepared, idempotency-on-
|
||||
second-rollback).
|
||||
- Per-scenario factory: every test starts from a fresh participant +
|
||||
teardown so the suite is isolation-safe under parallel runs.
|
||||
- Uses `@suaveplan/testing/runner` — works under both Vitest and
|
||||
Bun:test without re-importing.
|
||||
- Generates unique gids per scenario so re-runs do not collide.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
// In an adapter package's tests/conformance/run-2pc-participant.test.ts
|
||||
import { run2pcParticipantConformance } from "@suaveplan/api-2pc/testing";
|
||||
import { createMyParticipant, teardownMyParticipant } from "../helpers";
|
||||
|
||||
run2pcParticipantConformance({
|
||||
async create() {
|
||||
const participant = await createMyParticipant();
|
||||
return {
|
||||
participant,
|
||||
teardown: () => teardownMyParticipant(participant),
|
||||
};
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
The harness exposes no surface beyond `run2pcParticipantConformance` —
|
||||
deliberately. Adapter-specific assertions belong in adapter test
|
||||
files, not in the conformance harness. If your adapter needs to
|
||||
exercise behaviour the suite does not cover (e.g. cancellation,
|
||||
custom serialization), add a sibling test file:
|
||||
|
||||
```ts
|
||||
import { describe, it } from "@suaveplan/testing/runner";
|
||||
import { run2pcParticipantConformance } from "@suaveplan/api-2pc/testing";
|
||||
|
||||
describe("my-participant", () => {
|
||||
run2pcParticipantConformance({ create: createMyParticipant });
|
||||
|
||||
it("cancels a stuck prepare on signal abort", { timeout: 30_000 }, async () => {
|
||||
// adapter-specific assertion
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface TwoPcParticipantFactory`
|
||||
- `create(): Promise<{ participant: RegisteredTwoPcParticipant; teardown: () => Promise<void> }>`
|
||||
- `function run2pcParticipantConformance(factory): void`
|
||||
- Registers six `describe`/`it` blocks against the surrounding
|
||||
runner. Returns nothing.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The harness deliberately does NOT use a shared Postgres connection
|
||||
across scenarios. Each scenario calls `factory.create()` to get a
|
||||
fresh participant; the factory is expected to wire its own connection
|
||||
(directly to a Postgres container, against a shared pool, or against
|
||||
an `env-bridge` passthrough URL). This isolation prevents one
|
||||
scenario's leftover prepared transactions from contaminating the
|
||||
next.
|
||||
|
||||
Gids embed `Date.now()` + a random suffix so re-runs within the same
|
||||
millisecond never collide on the underlying `pg_prepared_xacts`
|
||||
table. The cleanup `ROLLBACK PREPARED` at the end of the
|
||||
prepare-only scenario is best-effort: failure to roll back leaves
|
||||
the gid prepared until the next teardown sweep, but the harness's
|
||||
own teardown handle is the primary cleanup path.
|
||||
|
||||
The harness is intentionally NOT exported from the package root (`./`
|
||||
subpath) — only `./testing`. This mirrors the convention used by
|
||||
sibling packages (`@suaveplan/i18n-shared/testing`,
|
||||
`@suaveplan/api-host/testing`) so production code can not accidentally
|
||||
depend on test-only symbols.
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
import { run2pcParticipantConformance } from "./conformance.js";
|
||||
|
||||
/**
|
||||
* Build a fake Postgres-shaped participant that maintains an internal
|
||||
* set of "prepared" gids. Used purely to exercise the conformance
|
||||
* harness against deterministic semantics — the actual integration
|
||||
* test runs the same harness against a real Postgres container.
|
||||
*/
|
||||
function fakeParticipantFactory(): {
|
||||
create(): Promise<{
|
||||
participant: RegisteredTwoPcParticipant;
|
||||
teardown(): Promise<void>;
|
||||
}>;
|
||||
} {
|
||||
return {
|
||||
async create() {
|
||||
const prepared = new Set<string>();
|
||||
const participant: RegisteredTwoPcParticipant = {
|
||||
id: "fake",
|
||||
voterMode: "writer",
|
||||
connection: {
|
||||
engine: "postgres",
|
||||
run: async (sql: string) => {
|
||||
const trimmed = sql.trim();
|
||||
const match = trimmed.match(
|
||||
/^(PREPARE TRANSACTION|COMMIT PREPARED|ROLLBACK PREPARED) '(.+)'$/
|
||||
);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
const verb = match[1] as string;
|
||||
const gid = (match[2] as string).replace(/''/g, "'");
|
||||
if (verb === "PREPARE TRANSACTION") {
|
||||
prepared.add(gid);
|
||||
return undefined;
|
||||
}
|
||||
if (verb === "COMMIT PREPARED") {
|
||||
if (!prepared.has(gid)) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`prepared transaction with identifier '${gid}' does not exist`
|
||||
),
|
||||
{ code: "42704" }
|
||||
);
|
||||
}
|
||||
prepared.delete(gid);
|
||||
return undefined;
|
||||
}
|
||||
if (verb === "ROLLBACK PREPARED") {
|
||||
if (!prepared.has(gid)) {
|
||||
throw Object.assign(
|
||||
new Error(
|
||||
`prepared transaction with identifier '${gid}' does not exist`
|
||||
),
|
||||
{ code: "42704" }
|
||||
);
|
||||
}
|
||||
prepared.delete(gid);
|
||||
return undefined;
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
prepare: async () => ({
|
||||
vote: "commit",
|
||||
writeIntent: true,
|
||||
}),
|
||||
};
|
||||
return {
|
||||
participant,
|
||||
async teardown() {
|
||||
prepared.clear();
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
run2pcParticipantConformance(fakeParticipantFactory());
|
||||
|
||||
describe("run2pcParticipantConformance smoke", () => {
|
||||
it("is callable without throwing at module load", { timeout: 5000 }, () => {
|
||||
expect(typeof run2pcParticipantConformance).toBe("function");
|
||||
});
|
||||
});
|
||||
155
packages/api-stack/api/api-2pc/src/testing/conformance.ts
Normal file
155
packages/api-stack/api/api-2pc/src/testing/conformance.ts
Normal file
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { RegisteredTwoPcParticipant } from "../types/types.types.js";
|
||||
|
||||
/**
|
||||
* Factory the conformance suite calls before every scenario. Returns
|
||||
* a fresh participant + a teardown handle. The harness drives
|
||||
* prepare/commit/rollback semantics against the participant; the
|
||||
* factory's only job is to wire a real (or sufficiently realistic)
|
||||
* Postgres connection in the returned `participant.connection`.
|
||||
*/
|
||||
export interface TwoPcParticipantFactory {
|
||||
create(): Promise<{
|
||||
readonly participant: RegisteredTwoPcParticipant;
|
||||
teardown(): Promise<void>;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Conformance suite. Re-exported on the `./testing` subpath so every
|
||||
* participant implementation can drop the following one-liner into
|
||||
* its `tests/conformance/run-2pc-participant.test.ts`:
|
||||
*
|
||||
* `run2pcParticipantConformance({ create: createMyParticipant })`
|
||||
*
|
||||
* The suite asserts six invariants:
|
||||
*
|
||||
* 1. A writer's prepare callback returning `commit` must be valid
|
||||
* for the coordinator to subsequently call `connection.run` with
|
||||
* `PREPARE TRANSACTION '<gid>'` and have the statement succeed
|
||||
* against the underlying engine.
|
||||
* 2. `COMMIT PREPARED '<gid>'` against a prepared gid must succeed.
|
||||
* 3. `ROLLBACK PREPARED '<gid>'` against a prepared gid must
|
||||
* succeed.
|
||||
* 4. A second `ROLLBACK PREPARED` against the same already-rolled-back
|
||||
* gid must raise SQLSTATE `42704` (or a "does not exist" message),
|
||||
* proving the idempotency anchor the coordinator depends on.
|
||||
* 5. The participant's `connection.engine` must be `"postgres"`.
|
||||
* 6. The participant's `voterMode` must be `"writer"` (read-only
|
||||
* voters are not subject to this suite — they do not run prepare
|
||||
* SQL).
|
||||
*/
|
||||
export function run2pcParticipantConformance(
|
||||
factory: TwoPcParticipantFactory
|
||||
): void {
|
||||
describe("2PC participant conformance", () => {
|
||||
it("declares engine 'postgres' and voterMode 'writer'", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
try {
|
||||
expect(participant.connection.engine).toBe("postgres");
|
||||
expect(participant.voterMode).toBe("writer");
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("PREPARE TRANSACTION succeeds after a commit-voted prepare callback", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
const vote = await participant.prepare(gid, {});
|
||||
expect(vote.vote).toBe("commit");
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
// Cleanup so the test doesn't leave row locks behind.
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("COMMIT PREPARED resolves a prepared gid", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-commit-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await participant.prepare(gid, {});
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
await participant.connection.run(
|
||||
`COMMIT PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("ROLLBACK PREPARED resolves a prepared gid", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-rollback-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await participant.prepare(gid, {});
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
|
||||
it("a second ROLLBACK PREPARED raises the documented 'does not exist' error", {
|
||||
timeout: 30_000,
|
||||
}, async () => {
|
||||
const { participant, teardown } = await factory.create();
|
||||
const gid = `2pc-conformance-idem-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await participant.prepare(gid, {});
|
||||
await participant.connection.run(
|
||||
`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
let caught: unknown;
|
||||
try {
|
||||
await participant.connection.run(
|
||||
`ROLLBACK PREPARED '${gid.replace(/'/g, "''")}'`
|
||||
);
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeDefined();
|
||||
const e = caught as { code?: string; message?: string };
|
||||
const matches =
|
||||
e.code === "42704" ||
|
||||
(typeof e.message === "string" &&
|
||||
e.message.includes("does not exist"));
|
||||
expect(matches).toBe(true);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
4
packages/api-stack/api/api-2pc/src/testing/index.ts
Normal file
4
packages/api-stack/api/api-2pc/src/testing/index.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
run2pcParticipantConformance,
|
||||
type TwoPcParticipantFactory,
|
||||
} from "./conformance.js";
|
||||
8
packages/api-stack/api/api-2pc/src/types/index.ts
Normal file
8
packages/api-stack/api/api-2pc/src/types/index.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
export type {
|
||||
PreparableConnection,
|
||||
RegisteredTwoPcParticipant,
|
||||
Run2PcInput,
|
||||
Run2PcOutcome,
|
||||
TwoPcCoordinatorState,
|
||||
TwoPcPrepareResult,
|
||||
} from "./types.types.js";
|
||||
108
packages/api-stack/api/api-2pc/src/types/types.types.md
Normal file
108
packages/api-stack/api/api-2pc/src/types/types.types.md
Normal file
@@ -0,0 +1,108 @@
|
||||
# `types/`
|
||||
|
||||
## Purpose
|
||||
|
||||
Shared cross-module type definitions for the 2PC coordinator. Holds
|
||||
the `TwoPcCoordinatorState` state-machine alphabet, the
|
||||
`PreparableConnection` and `RegisteredTwoPcParticipant` participant
|
||||
shapes, the `Run2PcInput` / `Run2PcOutcome` public input/output
|
||||
contracts, and the `TwoPcPrepareResult` per-participant prepare
|
||||
return type.
|
||||
|
||||
Centralising these types here prevents the cyclic-import smell that
|
||||
would otherwise emerge from `coordinator/`, `phases/`, `recovery/`,
|
||||
and `registry/` all needing to know the same wire shapes. The
|
||||
sibling modules import from this barrel and stay decoupled from each
|
||||
other at the type level.
|
||||
|
||||
## Features
|
||||
|
||||
- `TwoPcCoordinatorState` — string-literal union covering
|
||||
`voting`, `committing`, `aborting`, `committed`, `aborted`,
|
||||
`in_doubt`.
|
||||
- `PreparableConnection` — structural connection shape exposing
|
||||
`engine` (a `DatabaseEngine` from `@suaveplan/api-types/db`) and
|
||||
`run(sql, params?)` returning a `Promise<R>`.
|
||||
- `RegisteredTwoPcParticipant` — coordinator-side participant view
|
||||
carrying `id`, `voterMode`, `connection`, and an async
|
||||
`prepare(txId, payload) => TwoPcPrepareResult` callback.
|
||||
- `TwoPcPrepareResult` — `{ vote, writeIntent }` returned by a
|
||||
participant's `prepare` callback.
|
||||
- `Run2PcInput` / `Run2PcOutcome` — public input/output contracts
|
||||
accepted and returned by `run2pc`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type {
|
||||
Run2PcInput,
|
||||
RegisteredTwoPcParticipant,
|
||||
} from "@suaveplan/api-2pc";
|
||||
|
||||
const input: Run2PcInput = {
|
||||
participants: [{ id: "billing", input: { invoiceId: "inv-1" } }],
|
||||
timeoutMs: 5000,
|
||||
};
|
||||
|
||||
const billing: RegisteredTwoPcParticipant = {
|
||||
id: "billing",
|
||||
voterMode: "writer",
|
||||
connection: { engine: "postgres", run: pg.unsafe.bind(pg) },
|
||||
prepare: async (_txId) => ({ vote: "commit", writeIntent: true }),
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
Read-only voter typing — combine `voterMode: "read-only"` with a
|
||||
`writeIntent: false` invariant on the `prepare` return shape:
|
||||
|
||||
```ts
|
||||
import type {
|
||||
RegisteredTwoPcParticipant,
|
||||
TwoPcPrepareResult,
|
||||
} from "@suaveplan/api-2pc";
|
||||
|
||||
async function auditPrepare(): Promise<TwoPcPrepareResult> {
|
||||
const ok = await checkConsistency();
|
||||
return { vote: ok ? "commit" : "abort", writeIntent: false };
|
||||
}
|
||||
|
||||
const audit: RegisteredTwoPcParticipant = {
|
||||
id: "audit",
|
||||
voterMode: "read-only",
|
||||
connection: { engine: "surrealdb", run: surrealUnsafe },
|
||||
prepare: () => auditPrepare(),
|
||||
};
|
||||
```
|
||||
|
||||
The coordinator refuses a read-only voter whose `prepare` returns
|
||||
`writeIntent: true` with `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`
|
||||
— the static type doesn't enforce that pairing, but the runtime
|
||||
boundary does.
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type TwoPcCoordinatorState`
|
||||
- `interface PreparableConnection`
|
||||
- `interface RegisteredTwoPcParticipant`
|
||||
- `interface TwoPcPrepareResult`
|
||||
- `interface Run2PcInput`
|
||||
- `interface Run2PcOutcome`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Every interface is `readonly` on every field. This is deliberate —
|
||||
the coordinator caches inputs (participant lookups, ledger entries)
|
||||
and a mutation by accident would corrupt in-flight state. The
|
||||
strictness is enforced at the type level via `readonly` plus
|
||||
`ReadonlyArray<...>` and at the runtime level by the boundary Zod
|
||||
schemas in `options/` (for caller-facing inputs).
|
||||
|
||||
The `Run2PcOutcome.state` field uses `Extract<TwoPcCoordinatorState,
|
||||
"committed" | "aborted" | "in_doubt">` so callers do not have to
|
||||
exhaustively switch over the in-flight states (`voting`,
|
||||
`committing`, `aborting`) that never appear on a returned outcome.
|
||||
The structural typing of `PreparableConnection` keeps the package
|
||||
independent of `pg` and `postgres` (porsager) — both clients can be
|
||||
adapted at the seam with a one-line wrapper.
|
||||
117
packages/api-stack/api/api-2pc/src/types/types.types.ts
Normal file
117
packages/api-stack/api/api-2pc/src/types/types.types.ts
Normal file
@@ -0,0 +1,117 @@
|
||||
import type {
|
||||
DatabaseEngine,
|
||||
PreparedStatement,
|
||||
} from "@suaveplan/api-types/db";
|
||||
import type { TwoPcVote } from "@suaveplan/api-types/twopc";
|
||||
|
||||
/**
|
||||
* Coordinator-side lifecycle state machine for a single 2PC run. Mirrors
|
||||
* the design doc §9.2 vocabulary (`voting → committing | aborting →
|
||||
* committed | aborted | in_doubt`) — `voting` is the prepare phase,
|
||||
* the terminal states are persisted to `suaveplan_2pc_transactions` for
|
||||
* recovery.
|
||||
*/
|
||||
export type TwoPcCoordinatorState =
|
||||
| "voting"
|
||||
| "committing"
|
||||
| "aborting"
|
||||
| "committed"
|
||||
| "aborted"
|
||||
| "in_doubt";
|
||||
|
||||
/**
|
||||
* Participant write surface used by the coordinator during phase 1 and
|
||||
* phase 2. Modelled as a single structural method `run(sql, params)`
|
||||
* to match `postgres` (porsager/postgres)'s `sql.unsafe(...)` and
|
||||
* `pg.Client.query(text, values)` interchangeably. Returning a row
|
||||
* shape is irrelevant for 2PC SQL — the coordinator only cares that
|
||||
* the statement either succeeds or throws.
|
||||
*/
|
||||
export interface PreparableConnection {
|
||||
/** Engine identifier — coordinator gates on `db.preparable` capability. */
|
||||
readonly engine: DatabaseEngine;
|
||||
/**
|
||||
* Execute an arbitrary SQL statement. Used for
|
||||
* `PREPARE TRANSACTION '<gid>'`, `COMMIT PREPARED '<gid>'`,
|
||||
* `ROLLBACK PREPARED '<gid>'`, and the recovery's
|
||||
* `SELECT gid FROM pg_prepared_xacts ...` probe.
|
||||
*/
|
||||
run<R = unknown>(sql: string, params?: ReadonlyArray<unknown>): Promise<R>;
|
||||
/**
|
||||
* Optional prepared-statement bridge. The 2PC coordinator does not
|
||||
* use prepared statements itself (every call is one-shot SQL), but
|
||||
* the participant SHOULD expose its production `prepare` hook so
|
||||
* the in-package conformance suite can verify the engine actually
|
||||
* supports prepared transactions.
|
||||
*/
|
||||
prepare?<R = unknown>(sql: string): Promise<PreparedStatement<R>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coordinator-side view of a participant. The `id` is unique per
|
||||
* coordinator and is folded into the global transaction id passed to
|
||||
* `PREPARE TRANSACTION '<gid>'`. `voterMode` distinguishes write
|
||||
* participants (full prepare/commit/rollback SQL) from read-only
|
||||
* voters (vote-only — no SQL crosses the wire in phase 2).
|
||||
*/
|
||||
export interface RegisteredTwoPcParticipant {
|
||||
readonly id: string;
|
||||
readonly voterMode: "writer" | "read-only";
|
||||
readonly connection: PreparableConnection;
|
||||
/**
|
||||
* Prepare callback — runs the participant's business work inside
|
||||
* a transaction, returns `commit` if the work passed local
|
||||
* invariants. The coordinator turns a `commit` vote into a
|
||||
* `PREPARE TRANSACTION` SQL call. Write-intent payloads from a
|
||||
* read-only voter are rejected with
|
||||
* `TWOPC_READ_ONLY_PARTICIPANT_CANNOT_WRITE`.
|
||||
*/
|
||||
prepare(
|
||||
txId: string,
|
||||
input: Readonly<Record<string, unknown>>
|
||||
): Promise<TwoPcPrepareResult>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return value from a participant's prepare callback. `writeIntent`
|
||||
* carries whether the work would mutate state — read-only voters MUST
|
||||
* return `writeIntent: false` (the coordinator refuses otherwise).
|
||||
*/
|
||||
export interface TwoPcPrepareResult {
|
||||
readonly vote: TwoPcVote;
|
||||
readonly writeIntent: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Caller-supplied input to `run2pc`. `participants` is an ordered list
|
||||
* — phase 1 collects votes left-to-right, phase 2 commits/rolls back
|
||||
* left-to-right. `timeoutMs` is the phase-1 deadline; phase 2 is
|
||||
* unbounded by design (the protocol cannot tolerate giving up on
|
||||
* commit-prepared mid-flight).
|
||||
*/
|
||||
export interface Run2PcInput {
|
||||
readonly participants: ReadonlyArray<{
|
||||
readonly id: string;
|
||||
readonly input: Readonly<Record<string, unknown>>;
|
||||
}>;
|
||||
readonly timeoutMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only outcome handed back from `run2pc`. Includes the final
|
||||
* coordinator state and the participant votes — useful for callers
|
||||
* that want to surface per-participant outcomes in audit logs.
|
||||
*/
|
||||
export interface Run2PcOutcome {
|
||||
readonly txId: string;
|
||||
readonly state: Extract<
|
||||
TwoPcCoordinatorState,
|
||||
"committed" | "aborted" | "in_doubt"
|
||||
>;
|
||||
readonly votes: ReadonlyArray<{
|
||||
readonly participantId: string;
|
||||
readonly vote: TwoPcVote;
|
||||
}>;
|
||||
readonly prepareDurationMs: number;
|
||||
readonly commitDurationMs: number;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* Vitest globalSetup — registers the Postgres fixture in the
|
||||
* globalSetup process's module scope.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
import { createIntegrationSetup } from "@suaveplan/testing-integration";
|
||||
import { PostgresFixture } from "@suaveplan/testing-integration-postgres";
|
||||
|
||||
createIntegrationSetup({
|
||||
postgres: new PostgresFixture(),
|
||||
});
|
||||
|
||||
export {
|
||||
setup,
|
||||
teardown,
|
||||
} from "@suaveplan/testing-integration/setup/global-setup";
|
||||
@@ -0,0 +1,33 @@
|
||||
import { inject as vitestInject } from "@suaveplan/testing/runner";
|
||||
import type { PostgresConfig } from "@suaveplan/testing-integration-postgres";
|
||||
|
||||
interface IntegrationConfigs {
|
||||
postgres: PostgresConfig;
|
||||
}
|
||||
|
||||
export function inject<K extends keyof IntegrationConfigs>(
|
||||
key: K
|
||||
): IntegrationConfigs[K] {
|
||||
const all = (vitestInject as (k: string) => IntegrationConfigs)(
|
||||
"__integration_configs__"
|
||||
);
|
||||
return all[key];
|
||||
}
|
||||
|
||||
export const postgresUnavailable: boolean = (() => {
|
||||
try {
|
||||
const failed = (vitestInject as (k: string) => string[] | undefined)(
|
||||
"__integration_failed__"
|
||||
);
|
||||
if (failed === undefined) return true;
|
||||
if (failed.includes("postgres")) return true;
|
||||
const configs = (
|
||||
vitestInject as (
|
||||
k: string
|
||||
) => Record<string, Record<string, unknown>> | undefined
|
||||
)("__integration_configs__");
|
||||
return configs === undefined || configs.postgres === undefined;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,268 @@
|
||||
/**
|
||||
* Real-Postgres integration test for the 2PC coordinator. Drives a
|
||||
* single coordinator against TWO Postgres connections (acting as two
|
||||
* participants on the same physical instance) and asserts the full
|
||||
* commit + abort + recovery scenarios end-to-end.
|
||||
*
|
||||
* NOTE: 2PC requires `max_prepared_transactions > 0` in the Postgres
|
||||
* config. The default-image fixture starts with `max_prepared_transactions=0`,
|
||||
* so this test bootstraps a fresh config and reloads it via SQL before
|
||||
* running. We can't ALTER SYSTEM SET inside an in-memory fixture without
|
||||
* restarting, so we run the prepare against an in-test-cluster ALTER
|
||||
* SYSTEM + SELECT pg_reload_conf() flow that postgres supports without
|
||||
* a full restart.
|
||||
*
|
||||
* @module
|
||||
*/
|
||||
import { Database } from "bun:sqlite";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import postgres from "postgres";
|
||||
import {
|
||||
createTwoPcCoordinator,
|
||||
type LedgerDb,
|
||||
type RegisteredTwoPcParticipant,
|
||||
TwoPcParticipantRegistry,
|
||||
} from "../../src/index.js";
|
||||
import { TwoPcLedgerStore } from "../../src/persistence/persistence.js";
|
||||
import {
|
||||
buildProbeFromRegistry,
|
||||
resolveRecovery,
|
||||
} from "../../src/recovery/index.js";
|
||||
import { inject, postgresUnavailable } from "./integration-setup.js";
|
||||
|
||||
interface PgClient {
|
||||
readonly engine: "postgres";
|
||||
run<R = unknown>(sql: string, params?: ReadonlyArray<unknown>): Promise<R>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
function makeClient(url: string): PgClient {
|
||||
const sql = postgres(url, { max: 4, prepare: false });
|
||||
return {
|
||||
engine: "postgres",
|
||||
async run<R>(text: string, params?: ReadonlyArray<unknown>) {
|
||||
const out = await sql.unsafe(text, params as unknown[]);
|
||||
return out as unknown as R;
|
||||
},
|
||||
async close() {
|
||||
await sql.end({ timeout: 1 });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureMaxPrepared(adminUrl: string): Promise<void> {
|
||||
const sql = postgres(adminUrl, { max: 1, prepare: false });
|
||||
try {
|
||||
const cur = (await sql.unsafe(
|
||||
"SHOW max_prepared_transactions"
|
||||
)) as unknown as { max_prepared_transactions: string }[];
|
||||
const current = Number.parseInt(
|
||||
cur[0]?.max_prepared_transactions ?? "0",
|
||||
10
|
||||
);
|
||||
if (Number.isFinite(current) && current > 0) return;
|
||||
// ALTER SYSTEM requires a restart for max_prepared_transactions.
|
||||
// For a fresh fixture this is the first run; we cannot reload
|
||||
// dynamically, so we surface that the test cannot proceed.
|
||||
throw new Error(
|
||||
"max_prepared_transactions=0 on the Postgres fixture; cannot run 2PC integration tests"
|
||||
);
|
||||
} finally {
|
||||
await sql.end({ timeout: 1 });
|
||||
}
|
||||
}
|
||||
|
||||
function makeLedger(): { ledgerDb: LedgerDb; ledger: TwoPcLedgerStore } {
|
||||
const db = new Database(":memory:") as unknown as LedgerDb;
|
||||
db.run(`CREATE TABLE IF NOT EXISTS suaveplan_2pc_transactions (
|
||||
id TEXT PRIMARY KEY,
|
||||
coordinator TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
participants_json TEXT NOT NULL,
|
||||
prepared_at TEXT,
|
||||
committed_at TEXT,
|
||||
aborted_at TEXT
|
||||
)`);
|
||||
return { ledgerDb: db, ledger: new TwoPcLedgerStore(db) };
|
||||
}
|
||||
|
||||
describe.skipIf(postgresUnavailable)("real Postgres 2PC coordinator", () => {
|
||||
it("commits successfully across two participants", {
|
||||
timeout: 60000,
|
||||
}, async () => {
|
||||
const { url } = inject("postgres");
|
||||
await ensureMaxPrepared(url);
|
||||
const c1 = makeClient(url);
|
||||
const c2 = makeClient(url);
|
||||
await c1.run("CREATE TABLE IF NOT EXISTS twopc_smoke_a (n int)");
|
||||
await c2.run("CREATE TABLE IF NOT EXISTS twopc_smoke_b (n int)");
|
||||
await c1.run("DELETE FROM twopc_smoke_a");
|
||||
await c2.run("DELETE FROM twopc_smoke_b");
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
const a: RegisteredTwoPcParticipant = {
|
||||
id: "a",
|
||||
voterMode: "writer",
|
||||
connection: c1,
|
||||
prepare: async (txId) => {
|
||||
await c1.run("BEGIN");
|
||||
await c1.run("INSERT INTO twopc_smoke_a (n) VALUES (1)");
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
const b: RegisteredTwoPcParticipant = {
|
||||
id: "b",
|
||||
voterMode: "writer",
|
||||
connection: c2,
|
||||
prepare: async (txId) => {
|
||||
await c2.run("BEGIN");
|
||||
await c2.run("INSERT INTO twopc_smoke_b (n) VALUES (2)");
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
};
|
||||
registry.register(a);
|
||||
registry.register(b);
|
||||
|
||||
const { ledgerDb } = makeLedger();
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-it" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () =>
|
||||
`tx-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
});
|
||||
|
||||
const outcome = await coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "a", input: {} },
|
||||
{ id: "b", input: {} },
|
||||
],
|
||||
});
|
||||
expect(outcome.state).toBe("committed");
|
||||
const rowsA = (await c1.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_a"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
const rowsB = (await c2.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_b"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
expect(rowsA[0]?.c).toBe(1);
|
||||
expect(rowsB[0]?.c).toBe(1);
|
||||
|
||||
await c1.close();
|
||||
await c2.close();
|
||||
});
|
||||
|
||||
it("rolls back everywhere when one participant votes abort", {
|
||||
timeout: 60000,
|
||||
}, async () => {
|
||||
const { url } = inject("postgres");
|
||||
const c1 = makeClient(url);
|
||||
const c2 = makeClient(url);
|
||||
await c1.run("CREATE TABLE IF NOT EXISTS twopc_smoke_c (n int)");
|
||||
await c2.run("CREATE TABLE IF NOT EXISTS twopc_smoke_d (n int)");
|
||||
await c1.run("DELETE FROM twopc_smoke_c");
|
||||
await c2.run("DELETE FROM twopc_smoke_d");
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "c",
|
||||
voterMode: "writer",
|
||||
connection: c1,
|
||||
prepare: async () => {
|
||||
await c1.run("BEGIN");
|
||||
await c1.run("INSERT INTO twopc_smoke_c (n) VALUES (3)");
|
||||
return { vote: "commit", writeIntent: true };
|
||||
},
|
||||
});
|
||||
registry.register({
|
||||
id: "d",
|
||||
voterMode: "writer",
|
||||
connection: c2,
|
||||
prepare: async () => {
|
||||
return { vote: "abort", writeIntent: false };
|
||||
},
|
||||
});
|
||||
|
||||
const { ledgerDb } = makeLedger();
|
||||
const coordinator = createTwoPcCoordinator({
|
||||
options: { hostInstanceId: "host-it" },
|
||||
registry,
|
||||
ledgerDb,
|
||||
clock: () => Date.now(),
|
||||
idGenerator: () =>
|
||||
`tx-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
});
|
||||
|
||||
await expect(
|
||||
coordinator.run2pc({
|
||||
participants: [
|
||||
{ id: "c", input: {} },
|
||||
{ id: "d", input: {} },
|
||||
],
|
||||
})
|
||||
).rejects.toMatchObject({ twoPcCode: "TWOPC.VOTE_ABORT" });
|
||||
|
||||
const rowsC = (await c1.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_c"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
expect(rowsC[0]?.c).toBe(0);
|
||||
|
||||
await c1.close();
|
||||
await c2.close();
|
||||
});
|
||||
|
||||
it("recovery commits an orphan whose ledger row says committing", {
|
||||
timeout: 60000,
|
||||
}, async () => {
|
||||
const { url } = inject("postgres");
|
||||
const c1 = makeClient(url);
|
||||
await c1.run("CREATE TABLE IF NOT EXISTS twopc_smoke_e (n int)");
|
||||
await c1.run("DELETE FROM twopc_smoke_e");
|
||||
|
||||
const { ledger } = makeLedger();
|
||||
// Simulate a crash: PREPARE the transaction, write the ledger row
|
||||
// as "committing", but never run COMMIT PREPARED.
|
||||
const txId = `recover-${Date.now()}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2, 8)}`;
|
||||
const gid = `2pc-host-it-${txId}-e`;
|
||||
await c1.run("BEGIN");
|
||||
await c1.run("INSERT INTO twopc_smoke_e (n) VALUES (42)");
|
||||
await c1.run(`PREPARE TRANSACTION '${gid.replace(/'/g, "''")}'`);
|
||||
ledger.insert({
|
||||
id: txId,
|
||||
coordinator: "host-it",
|
||||
participants: [
|
||||
{ participantId: "e", voterMode: "writer", vote: "commit" },
|
||||
],
|
||||
});
|
||||
ledger.markAllVotedCommit({
|
||||
id: txId,
|
||||
participants: [
|
||||
{ participantId: "e", voterMode: "writer", vote: "commit" },
|
||||
],
|
||||
preparedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
const registry = new TwoPcParticipantRegistry(() => true);
|
||||
registry.register({
|
||||
id: "e",
|
||||
voterMode: "writer",
|
||||
connection: c1,
|
||||
prepare: async () => ({ vote: "commit", writeIntent: true }),
|
||||
});
|
||||
const report = await resolveRecovery({
|
||||
hostInstanceId: "host-it",
|
||||
ledger,
|
||||
probes: buildProbeFromRegistry(registry),
|
||||
});
|
||||
expect(report.committed).toContain(gid);
|
||||
const rows = (await c1.run(
|
||||
"SELECT count(*)::int as c FROM twopc_smoke_e"
|
||||
)) as ReadonlyArray<{ c: number }>;
|
||||
expect(rows[0]?.c).toBe(1);
|
||||
|
||||
await c1.close();
|
||||
});
|
||||
});
|
||||
12
packages/api-stack/api/api-2pc/tsconfig.lib.json
Normal file
12
packages/api-stack/api/api-2pc/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
37
packages/api-stack/api/api-2pc/vite.config.ts
Normal file
37
packages/api-stack/api/api-2pc/vite.config.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/api-2pc",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
/^@suaveplan\//,
|
||||
/^node:/,
|
||||
/^@opentelemetry\//,
|
||||
"tslib",
|
||||
"zod",
|
||||
"postgres",
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
31
packages/api-stack/api/api-2pc/vitest.config.ts
Normal file
31
packages/api-stack/api/api-2pc/vitest.config.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
include: ["src/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**", "tests/integration/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/index.ts",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
19
packages/api-stack/api/api-2pc/vitest.integration.config.ts
Normal file
19
packages/api-stack/api/api-2pc/vitest.integration.config.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
const globalSetup = fileURLToPath(
|
||||
new URL("./tests/integration/global-setup.ts", import.meta.url)
|
||||
);
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
globalSetup: [globalSetup],
|
||||
include: ["tests/integration/**/*.integration.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**", "src/**"],
|
||||
testTimeout: 90000,
|
||||
hookTimeout: 180000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user