Initial Commit
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
# @suaveplan/network-plugin-circuit-breaker - Circuit-breaker plugin for mesh relay paths: per-path state machine, registry, half-open probes, and relay middleware integration
|
||||
|
||||
## Summary
|
||||
|
||||
Mesh-level circuit breakers: per-path state machine (`CLOSED → OPEN → HALF_OPEN`), registry, half-open probe loop, and relay middleware integration. Fast-fails dead relay paths so retries don't cascade.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [breaker](./src/breaker/mesh-circuit-breaker.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [mesh-circuit-breaker](./src/breaker/mesh-circuit-breaker.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [circuit-breaker-middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [half-open-probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [circuit-breaker-registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [error-codes](./src/error-codes.md) — Frozen error-code registry for the mesh circuit-breaker plugin. Two codes cover the full surface: `PATH_OPEN` when a re…
|
||||
<!-- end:auto-docs --> - [mesh-circuit-breaker](./src/breaker/mesh-circuit-breaker.md) — <!-- begin:auto-docs --> _(no submodules)_- [middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [circuit-breaker-middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [half-open-probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [circuit-breaker-registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen error-code registry for the mesh circuit-breaker plugin. Two codes cover the full surface: `PATH_OPEN` when a re…
|
||||
- [mesh-circuit-breaker](./src/breaker/mesh-circuit-breaker.md) — <!-- begin:auto-docs --> _(no submodules)_- [middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [circuit-breaker-middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [half-open-probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [circuit-breaker-registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen error-code registry for the mesh circuit-breaker plugin. Two codes cover the full surface: `PATH_OPEN` when a re…
|
||||
- [mesh-circuit-breaker](./src/breaker/mesh-circuit-breaker.md) — <!-- begin:auto-docs --> _(no submodules)_- [middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [circuit-breaker-middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [half-open-probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [circuit-breaker-registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen error-code registry for the mesh circuit-breaker plugin. Two codes cover the full surface: `PATH_OPEN` when a re…
|
||||
- [mesh-circuit-breaker](./src/breaker/mesh-circuit-breaker.md) — <!-- begin:auto-docs --> _(no submodules)_- [middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_ - [circuit-breaker-middleware](./src/middleware/circuit-breaker-middleware.md) — <!-- begin:auto-docs --> _(no submodules)_- [probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_ - [half-open-probe](./src/probe/half-open-probe.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_ - [circuit-breaker-registry](./src/registry/circuit-breaker-registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen error-code registry for the mesh circuit-breaker plugin. Two codes cover the full surface: `PATH_OPEN` when a re…
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"name": "@suaveplan/network-plugin-circuit-breaker",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Circuit-breaker plugin for mesh relay paths: per-path state machine, registry, half-open probes, and relay middleware integration",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"files": ["dist", "src", "README.md"],
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
|
||||
"require": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "bunx biome check src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/network": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { MeshCircuitBreaker } from "./mesh-circuit-breaker.js";
|
||||
|
||||
describe("MeshCircuitBreaker", () => {
|
||||
it("starts CLOSED and accepts work", () => {
|
||||
const cb = new MeshCircuitBreaker({ pathId: "p" });
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
expect(cb.canExecute()).toBe(true);
|
||||
});
|
||||
|
||||
it("opens after failureThreshold consecutive failures", () => {
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 2,
|
||||
});
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
expect(cb.canExecute()).toBe(false);
|
||||
});
|
||||
|
||||
it("clears the failure counter on success in CLOSED", () => {
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 2,
|
||||
});
|
||||
cb.recordFailure();
|
||||
cb.recordSuccess();
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("transitions OPEN → HALF_OPEN after openDurationMs", () => {
|
||||
let now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
now += 200;
|
||||
expect(cb.getState()).toBe("HALF_OPEN");
|
||||
expect(cb.canExecute()).toBe(true);
|
||||
});
|
||||
|
||||
it("HALF_OPEN closes after successThreshold successes", () => {
|
||||
let now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
successThreshold: 2,
|
||||
openDurationMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
cb.recordFailure();
|
||||
now += 200;
|
||||
cb.recordSuccess();
|
||||
cb.recordSuccess();
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("HALF_OPEN failure trips back to OPEN", () => {
|
||||
let now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
cb.recordFailure();
|
||||
now += 200;
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
});
|
||||
|
||||
it("HALF_OPEN gates execution at halfOpenMaxCalls", () => {
|
||||
let now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
successThreshold: 100,
|
||||
halfOpenMaxCalls: 2,
|
||||
openDurationMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
cb.recordFailure();
|
||||
now += 200;
|
||||
expect(cb.canExecute()).toBe(true);
|
||||
cb.recordSuccess();
|
||||
cb.recordSuccess();
|
||||
// halfOpenCalls === 2 — no more allowed until close
|
||||
expect(cb.canExecute()).toBe(false);
|
||||
});
|
||||
|
||||
it("recordSuccess from OPEN throws INVALID_TRANSITION", () => {
|
||||
const now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100_000,
|
||||
clock: () => now,
|
||||
});
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
expect(() => cb.recordSuccess()).toThrow("breaker.success-from-open");
|
||||
});
|
||||
|
||||
it("on(handler) fires on every state change", () => {
|
||||
const events: string[] = [];
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 0,
|
||||
});
|
||||
const off = cb.on((s) => events.push(s));
|
||||
cb.recordFailure();
|
||||
off();
|
||||
cb.recordSuccess();
|
||||
expect(events).toContain("OPEN");
|
||||
});
|
||||
|
||||
it("uses Date.now-based clock when none injected", () => {
|
||||
const cb = new MeshCircuitBreaker({ pathId: "p" });
|
||||
cb.recordFailure();
|
||||
cb.recordSuccess();
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("recordFailure from OPEN is a no-op", () => {
|
||||
const now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100_000,
|
||||
clock: () => now,
|
||||
});
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
# MeshCircuitBreaker
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Per-path circuit-breaker FSM that walks `CLOSED → OPEN → HALF_OPEN → CLOSED`. Stops repeated wire-level retries against a known-dead path so retries don't cascade into healthy paths. Each `(localHostId, nextHopHostId)` pair gets its own breaker instance, identified by `pathId`.
|
||||
|
||||
## Features
|
||||
|
||||
- Three-state FSM with deterministic transitions.
|
||||
- Configurable `failureThreshold` (default 5), `successThreshold` (default 3), `openDurationMs` (default 30 s), `halfOpenMaxCalls` (default 3).
|
||||
- Auto-transition `OPEN → HALF_OPEN` after the cooldown elapses (lazy-evaluated on `getState`/`canExecute`).
|
||||
- Throws `INVALID_TRANSITION` on `recordSuccess()` from `OPEN` to surface programmer errors loudly.
|
||||
- Bus-free: state-change listeners installed via `on(handler)` for downstream observability without coupling to a specific bus.
|
||||
- Injectable `clock` for deterministic tests.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { MeshCircuitBreaker } from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const cb = new MeshCircuitBreaker({ pathId: "meshA:host1→meshB:host2" });
|
||||
|
||||
if (cb.canExecute()) {
|
||||
try {
|
||||
await sendThroughPath();
|
||||
cb.recordSuccess();
|
||||
} catch (err) {
|
||||
cb.recordFailure();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { MeshCircuitBreaker } from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
let now = 0;
|
||||
const cb = new MeshCircuitBreaker({
|
||||
pathId: "p",
|
||||
failureThreshold: 2,
|
||||
successThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
halfOpenMaxCalls: 1,
|
||||
clock: () => now,
|
||||
});
|
||||
|
||||
cb.on((state) => {
|
||||
console.log("breaker", cb.pathId, "→", state);
|
||||
});
|
||||
|
||||
cb.recordFailure();
|
||||
cb.recordFailure(); // OPEN
|
||||
now += 200;
|
||||
// canExecute lazily transitions to HALF_OPEN
|
||||
if (cb.canExecute()) {
|
||||
cb.recordSuccess(); // CLOSED
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type CircuitBreakerState = "CLOSED" | "OPEN" | "HALF_OPEN"`.
|
||||
- `interface MeshCircuitBreakerOptions { pathId, failureThreshold?, successThreshold?, openDurationMs?, halfOpenMaxCalls?, clock? }`.
|
||||
- `getState(): CircuitBreakerState` — lazy `OPEN → HALF_OPEN` if cooled down.
|
||||
- `canExecute(): boolean` — gating check.
|
||||
- `recordSuccess()` / `recordFailure()` — feedback hooks.
|
||||
- `on(handler) → () => void` — state-change subscription.
|
||||
- `pathId: string` — read-only public path identifier.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The auto-transition from `OPEN → HALF_OPEN` is lazy because we don't run a clock-driven timer inside the breaker — every `getState`/`canExecute` call is the moment we recompute the state if cooldown has elapsed. This avoids leaking timers and keeps the breaker fully synchronous from the caller's perspective. `recordSuccess()` from `OPEN` is a programmer error (the FSM should be `HALF_OPEN` first), so the breaker throws `INVALID_TRANSITION` to surface the bug rather than silently accepting it. In `HALF_OPEN`, `halfOpenMaxCalls` is the cap on trial executions; once exceeded, `canExecute` returns false until the next successful close.
|
||||
@@ -0,0 +1,136 @@
|
||||
import { CB_CODES, CircuitBreakerError } from "../error-codes.js";
|
||||
|
||||
/**
|
||||
* Three-state breaker FSM.
|
||||
*
|
||||
* `CLOSED` → normal operation, failures counted.
|
||||
* `OPEN` → fast-fail, no executions allowed.
|
||||
* `HALF_OPEN` → trial executions allowed; success → `CLOSED`, failure → `OPEN`.
|
||||
*/
|
||||
export type CircuitBreakerState = "CLOSED" | "OPEN" | "HALF_OPEN";
|
||||
|
||||
/**
|
||||
* Configuration for `MeshCircuitBreaker`.
|
||||
*/
|
||||
export interface MeshCircuitBreakerOptions {
|
||||
readonly pathId: string;
|
||||
readonly failureThreshold?: number;
|
||||
readonly successThreshold?: number;
|
||||
readonly openDurationMs?: number;
|
||||
readonly halfOpenMaxCalls?: number;
|
||||
readonly clock?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* State-change listener.
|
||||
*/
|
||||
export type StateChangeListener = (state: CircuitBreakerState) => void;
|
||||
|
||||
/**
|
||||
* Per-path circuit breaker: tracks failures, opens on a configurable
|
||||
* threshold, allows a limited number of trial executions in `HALF_OPEN`,
|
||||
* and closes once enough successes accumulate.
|
||||
*/
|
||||
export class MeshCircuitBreaker {
|
||||
public readonly pathId: string;
|
||||
private readonly failureThreshold: number;
|
||||
private readonly successThreshold: number;
|
||||
private readonly openDurationMs: number;
|
||||
private readonly halfOpenMaxCalls: number;
|
||||
private readonly clock: () => number;
|
||||
private state: CircuitBreakerState = "CLOSED";
|
||||
private failures = 0;
|
||||
private successes = 0;
|
||||
private halfOpenCalls = 0;
|
||||
private openedAt = 0;
|
||||
private readonly listeners = new Set<StateChangeListener>();
|
||||
|
||||
public constructor(options: MeshCircuitBreakerOptions) {
|
||||
this.pathId = options.pathId;
|
||||
this.failureThreshold = options.failureThreshold ?? 5;
|
||||
this.successThreshold = options.successThreshold ?? 3;
|
||||
this.openDurationMs = options.openDurationMs ?? 30_000;
|
||||
this.halfOpenMaxCalls = options.halfOpenMaxCalls ?? 3;
|
||||
this.clock = options.clock ?? (() => Date.now());
|
||||
}
|
||||
|
||||
public getState(): CircuitBreakerState {
|
||||
if (this.state === "OPEN" && this.cooledDown()) {
|
||||
this.transition("HALF_OPEN");
|
||||
}
|
||||
return this.state;
|
||||
}
|
||||
|
||||
public canExecute(): boolean {
|
||||
const current = this.getState();
|
||||
if (current === "CLOSED") return true;
|
||||
if (current === "OPEN") return false;
|
||||
return this.halfOpenCalls < this.halfOpenMaxCalls;
|
||||
}
|
||||
|
||||
public recordSuccess(): void {
|
||||
const current = this.getState();
|
||||
if (current === "OPEN") {
|
||||
throw new CircuitBreakerError(
|
||||
CB_CODES.INVALID_TRANSITION,
|
||||
"breaker.success-from-open",
|
||||
{ context: { pathId: this.pathId } }
|
||||
);
|
||||
}
|
||||
if (current === "HALF_OPEN") {
|
||||
this.halfOpenCalls += 1;
|
||||
this.successes += 1;
|
||||
if (this.successes >= this.successThreshold) {
|
||||
this.transition("CLOSED");
|
||||
}
|
||||
return;
|
||||
}
|
||||
// CLOSED: clear failure counter on success
|
||||
this.failures = 0;
|
||||
}
|
||||
|
||||
public recordFailure(): void {
|
||||
const current = this.getState();
|
||||
if (current === "HALF_OPEN") {
|
||||
this.halfOpenCalls += 1;
|
||||
this.transition("OPEN");
|
||||
return;
|
||||
}
|
||||
if (current === "CLOSED") {
|
||||
this.failures += 1;
|
||||
if (this.failures >= this.failureThreshold) {
|
||||
this.transition("OPEN");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public on(handler: StateChangeListener): () => void {
|
||||
this.listeners.add(handler);
|
||||
return () => {
|
||||
this.listeners.delete(handler);
|
||||
};
|
||||
}
|
||||
|
||||
private cooledDown(): boolean {
|
||||
return this.clock() - this.openedAt >= this.openDurationMs;
|
||||
}
|
||||
|
||||
private transition(next: CircuitBreakerState): void {
|
||||
this.state = next;
|
||||
if (next === "OPEN") {
|
||||
this.openedAt = this.clock();
|
||||
this.successes = 0;
|
||||
} else if (next === "CLOSED") {
|
||||
this.failures = 0;
|
||||
this.successes = 0;
|
||||
this.halfOpenCalls = 0;
|
||||
} else {
|
||||
// entering HALF_OPEN
|
||||
this.successes = 0;
|
||||
this.halfOpenCalls = 0;
|
||||
}
|
||||
for (const listener of this.listeners) {
|
||||
listener(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
# CB_CODES
|
||||
|
||||
## Purpose
|
||||
|
||||
Frozen error-code registry for the mesh circuit-breaker plugin. Two codes cover the full surface: `PATH_OPEN` when a request is rejected by an open breaker, and `INVALID_TRANSITION` when the FSM is asked to do something unrepresentable (e.g. `recordSuccess()` while still in `OPEN`). The `CircuitBreakerError` class extends the cross-package `SuaveplanError` so consumers can pattern-match on `code` without parsing message strings.
|
||||
|
||||
## Features
|
||||
|
||||
- Two frozen codes registered under the `CB` namespace.
|
||||
- `CircuitBreakerError extends SuaveplanError<CbErrorCode>`.
|
||||
- `name: "CircuitBreakerError"` on every instance for log filtering and dashboard alerting.
|
||||
- `cause` and `context` propagation via `SuaveplanErrorOptions`.
|
||||
- Codes are stable across versions — once registered they never disappear, only new codes are appended.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
CB_CODES,
|
||||
CircuitBreakerError,
|
||||
} from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
throw new CircuitBreakerError(CB_CODES.PATH_OPEN, "breaker.path-open", {
|
||||
context: { pathId: "meshA:host1→meshB:host2" },
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
CB_CODES,
|
||||
CircuitBreakerError,
|
||||
} from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
try {
|
||||
sendThroughRelay(envelope);
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof CircuitBreakerError &&
|
||||
err.code === CB_CODES.PATH_OPEN
|
||||
) {
|
||||
return sendThroughBackup(envelope);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `CB_CODES` — frozen `{ PATH_OPEN, INVALID_TRANSITION }`.
|
||||
- `type CbErrorCode` — union of the registered code values.
|
||||
- `class CircuitBreakerError extends SuaveplanError<CbErrorCode>` — typed throwable for every failure mode the plugin surfaces.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The registry is intentionally tight — two codes are enough because every other failure mode bubbles up as the underlying transport error (the breaker just wraps the wire-level error in `recordFailure`). `INVALID_TRANSITION` is the only "programmer-error" code; it should never appear in production unless someone calls `recordSuccess()` from `OPEN` state, which the FSM forbids. The package barrel side-effect-imports `./error-codes.js` so subpath consumers register the codes too — relying on a tree-shaken root import is unsafe when consumers may import deeper paths only.
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { CB_CODES, CircuitBreakerError } from "./error-codes.js";
|
||||
|
||||
describe("CB_CODES", () => {
|
||||
it("registers under CB", () => {
|
||||
for (const value of Object.values(CB_CODES)) {
|
||||
expect(value).toMatch(/^CB\./);
|
||||
}
|
||||
});
|
||||
|
||||
it("is frozen", () => {
|
||||
expect(Object.isFrozen(CB_CODES)).toBe(true);
|
||||
});
|
||||
|
||||
it("contains the documented codes", () => {
|
||||
for (const key of ["PATH_OPEN", "INVALID_TRANSITION"] as const) {
|
||||
expect(CB_CODES).toHaveProperty(key);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("CircuitBreakerError", () => {
|
||||
it("extends SuaveplanError", () => {
|
||||
const err = new CircuitBreakerError(
|
||||
CB_CODES.PATH_OPEN,
|
||||
"test.path-open",
|
||||
{ context: { pathId: "x" } }
|
||||
);
|
||||
expect(err.name).toBe("CircuitBreakerError");
|
||||
expect(err.code).toBe(CB_CODES.PATH_OPEN);
|
||||
});
|
||||
|
||||
it("preserves cause", () => {
|
||||
const cause = new Error("inner");
|
||||
const err = new CircuitBreakerError(
|
||||
CB_CODES.INVALID_TRANSITION,
|
||||
"with-cause",
|
||||
{ cause }
|
||||
);
|
||||
expect(err.cause).toBe(cause);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { defineCodes, SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
export const CB_CODES = defineCodes("CB", [
|
||||
"PATH_OPEN",
|
||||
"INVALID_TRANSITION",
|
||||
] as const);
|
||||
|
||||
export type CbErrorCode = (typeof CB_CODES)[keyof typeof CB_CODES];
|
||||
|
||||
export class CircuitBreakerError extends SuaveplanError<CbErrorCode> {
|
||||
public constructor(
|
||||
code: CbErrorCode,
|
||||
messageKey: string,
|
||||
options?: import("@suaveplan/error").SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, messageKey, options);
|
||||
this.name = "CircuitBreakerError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Error codes (side-effect registration)
|
||||
import "./error-codes.js";
|
||||
|
||||
// Breaker
|
||||
export {
|
||||
type CircuitBreakerState,
|
||||
MeshCircuitBreaker,
|
||||
type MeshCircuitBreakerOptions,
|
||||
type StateChangeListener,
|
||||
} from "./breaker/mesh-circuit-breaker.js";
|
||||
// Errors
|
||||
export {
|
||||
CB_CODES,
|
||||
type CbErrorCode,
|
||||
CircuitBreakerError,
|
||||
} from "./error-codes.js";
|
||||
// Middleware
|
||||
export {
|
||||
type CircuitBreakerMiddlewareOptions,
|
||||
createCircuitBreakerMiddleware,
|
||||
type PathIdBuilder,
|
||||
type RelayNext,
|
||||
} from "./middleware/circuit-breaker-middleware.js";
|
||||
// Probe
|
||||
export {
|
||||
HalfOpenProbe,
|
||||
type HalfOpenProbeOptions,
|
||||
type ProbeSender,
|
||||
} from "./probe/half-open-probe.js";
|
||||
// Registry
|
||||
export {
|
||||
CircuitBreakerRegistry,
|
||||
type CircuitBreakerRegistryOptions,
|
||||
} from "./registry/circuit-breaker-registry.js";
|
||||
@@ -0,0 +1,75 @@
|
||||
# createCircuitBreakerMiddleware
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Factory that produces the relay-middleware glue between a `CircuitBreakerRegistry` and any next-hop send pipeline. The middleware fast-fails with `PATH_OPEN` when the relevant breaker is open and otherwise records the outcome of `next()` so the breaker accumulates the right success/failure counts.
|
||||
|
||||
## Features
|
||||
|
||||
- `buildPathId` is injectable; default is `${local}→${next}`.
|
||||
- Throws `CircuitBreakerError(CB.PATH_OPEN)` on the open path, with `context: { pathId }`.
|
||||
- Records success on `next()` resolve, failure on `next()` reject (and re-throws so callers see the original error).
|
||||
- Pure function: returns a middleware closure that holds no mutable state of its own — all state lives in the `registry`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
CircuitBreakerRegistry,
|
||||
createCircuitBreakerMiddleware,
|
||||
} from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "host1",
|
||||
});
|
||||
|
||||
await middleware(envelope, "host2", async () => {
|
||||
await rawSend(envelope);
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
CB_CODES,
|
||||
CircuitBreakerError,
|
||||
CircuitBreakerRegistry,
|
||||
createCircuitBreakerMiddleware,
|
||||
} from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "host1",
|
||||
buildPathId: (local, next) => `mesh:${local}::${next}`,
|
||||
});
|
||||
|
||||
try {
|
||||
await middleware(envelope, "host2", () => realSend(envelope));
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof CircuitBreakerError &&
|
||||
err.code === CB_CODES.PATH_OPEN
|
||||
) {
|
||||
await sendThroughBackup(envelope);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type PathIdBuilder = (localHostId, nextHopHostId) => string`.
|
||||
- `interface CircuitBreakerMiddlewareOptions { registry, localHostId, buildPathId? }`.
|
||||
- `type RelayNext = () => Promise<void>`.
|
||||
- `createCircuitBreakerMiddleware(options): (envelope, nextHopHostId, next) => Promise<void>`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The middleware is a free function (rather than a class) because it has no internal state — the registry owns all the breaker state and the closure only needs `localHostId` + `buildPathId`. This makes the middleware trivially safe to share across many concurrent sends. The thrown `CircuitBreakerError` always carries `pathId` in its context so log filters and dashboards can group breaker rejections by path. Callers MUST treat the thrown error as a hard fast-fail signal — retrying immediately defeats the entire point of the breaker.
|
||||
@@ -0,0 +1,65 @@
|
||||
import type { MessageEnvelope } from "@suaveplan/network";
|
||||
import { CB_CODES, CircuitBreakerError } from "../error-codes.js";
|
||||
import type { CircuitBreakerRegistry } from "../registry/circuit-breaker-registry.js";
|
||||
|
||||
/**
|
||||
* Builder for the canonical `pathId` from a hop pair.
|
||||
*/
|
||||
export type PathIdBuilder = (
|
||||
localHostId: string,
|
||||
nextHopHostId: string
|
||||
) => string;
|
||||
|
||||
/**
|
||||
* Configuration for the middleware.
|
||||
*/
|
||||
export interface CircuitBreakerMiddlewareOptions {
|
||||
readonly registry: CircuitBreakerRegistry;
|
||||
readonly localHostId: string;
|
||||
readonly buildPathId?: PathIdBuilder;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps any next-hop send. If the breaker is open, fast-fails with
|
||||
* `PATH_OPEN`. Otherwise records success or failure based on the
|
||||
* outcome of `next()`.
|
||||
*/
|
||||
export type RelayNext = () => Promise<void>;
|
||||
|
||||
const defaultPathId: PathIdBuilder = (local, next) => `${local}→${next}`;
|
||||
|
||||
/**
|
||||
* Circuit-breaker middleware for relay sends. Returns a function that
|
||||
* wraps any `next()` step in the relay pipeline with breaker gating.
|
||||
*/
|
||||
export function createCircuitBreakerMiddleware(
|
||||
options: CircuitBreakerMiddlewareOptions
|
||||
): (
|
||||
envelope: MessageEnvelope,
|
||||
nextHopHostId: string,
|
||||
next: RelayNext
|
||||
) => Promise<void> {
|
||||
const buildPathId = options.buildPathId ?? defaultPathId;
|
||||
return async (
|
||||
_envelope: MessageEnvelope,
|
||||
nextHopHostId: string,
|
||||
next: RelayNext
|
||||
): Promise<void> => {
|
||||
const pathId = buildPathId(options.localHostId, nextHopHostId);
|
||||
const breaker = options.registry.getOrCreate(pathId);
|
||||
if (!breaker.canExecute()) {
|
||||
throw new CircuitBreakerError(
|
||||
CB_CODES.PATH_OPEN,
|
||||
"middleware.path-open",
|
||||
{ context: { pathId } }
|
||||
);
|
||||
}
|
||||
try {
|
||||
await next();
|
||||
breaker.recordSuccess();
|
||||
} catch (err) {
|
||||
breaker.recordFailure();
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { MessageEnvelope } from "@suaveplan/network";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { CB_CODES, CircuitBreakerError } from "../error-codes.js";
|
||||
import { CircuitBreakerRegistry } from "../registry/circuit-breaker-registry.js";
|
||||
import { createCircuitBreakerMiddleware } from "./circuit-breaker-middleware.js";
|
||||
|
||||
const envelope = {
|
||||
id: "x",
|
||||
type: "y",
|
||||
senderId: "self",
|
||||
payload: null,
|
||||
timestamp: 0,
|
||||
headers: {},
|
||||
ttl: 8,
|
||||
visitedNodes: [],
|
||||
} as MessageEnvelope;
|
||||
|
||||
describe("createCircuitBreakerMiddleware", () => {
|
||||
it("forwards through next on a healthy path", async () => {
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "h1",
|
||||
});
|
||||
let called = false;
|
||||
await middleware(envelope, "h2", async () => {
|
||||
called = true;
|
||||
});
|
||||
expect(called).toBe(true);
|
||||
});
|
||||
|
||||
it("records success after next resolves", async () => {
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 2 },
|
||||
});
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "h1",
|
||||
});
|
||||
const cb = registry.getOrCreate("h1→h2");
|
||||
cb.recordFailure(); // failure counter = 1
|
||||
await middleware(envelope, "h2", async () => {}); // resets to 0
|
||||
cb.recordFailure(); // back to 1, still CLOSED
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("records failure and re-throws when next rejects", async () => {
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 1 },
|
||||
});
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "h1",
|
||||
});
|
||||
const inner = new Error("transport-bang");
|
||||
await expect(
|
||||
middleware(envelope, "h2", async () => {
|
||||
throw inner;
|
||||
})
|
||||
).rejects.toBe(inner);
|
||||
expect(registry.getState("h1→h2")).toBe("OPEN");
|
||||
});
|
||||
|
||||
it("fast-fails with PATH_OPEN when the breaker is open", async () => {
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 1, openDurationMs: 100_000 },
|
||||
});
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "h1",
|
||||
});
|
||||
registry.getOrCreate("h1→h2").recordFailure();
|
||||
try {
|
||||
await middleware(envelope, "h2", async () => {
|
||||
expect.fail("next should not be called");
|
||||
});
|
||||
expect.fail("middleware should have thrown");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(CircuitBreakerError);
|
||||
expect((err as CircuitBreakerError).code).toBe(CB_CODES.PATH_OPEN);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses the default pathId builder", async () => {
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "h1",
|
||||
});
|
||||
await middleware(envelope, "h2", async () => {});
|
||||
expect(registry.getState("h1→h2")).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("respects a custom pathId builder", async () => {
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const middleware = createCircuitBreakerMiddleware({
|
||||
registry,
|
||||
localHostId: "h1",
|
||||
buildPathId: (a, b) => `mesh::${a}::${b}`,
|
||||
});
|
||||
await middleware(envelope, "h2", async () => {});
|
||||
expect(registry.getState("mesh::h1::h2")).toBe("CLOSED");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
# HalfOpenProbe
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Periodically issues lightweight probe envelopes against every relay path the registry reports as `HALF_OPEN`, then records success/failure into the corresponding breaker. Decoupling the probe from real traffic means a flaky path can be re-tested without risking application payload — if the probe fails the path stays open and real callers continue to fast-fail.
|
||||
|
||||
## Features
|
||||
|
||||
- Configurable `probeIntervalMs` (default 5 s).
|
||||
- Pluggable `probeMessageFactory` so callers can attach signing/auth metadata.
|
||||
- Injectable `send`, `setIntervalFn`, `clearIntervalFn` for deterministic tests.
|
||||
- Idempotent `start()` and `stop()`; no leaked timers.
|
||||
- `tick()` is exposed publicly so test harnesses can step the loop manually.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
CircuitBreakerRegistry,
|
||||
HalfOpenProbe,
|
||||
} from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const probe = new HalfOpenProbe({
|
||||
registry,
|
||||
probeMessageFactory: (pathId) => ({
|
||||
id: crypto.randomUUID(),
|
||||
type: "relay:probe",
|
||||
senderId: "self",
|
||||
payload: null,
|
||||
timestamp: Date.now(),
|
||||
headers: { pathId },
|
||||
ttl: 16,
|
||||
visitedNodes: [],
|
||||
}),
|
||||
send: async (pathId, envelope) => {
|
||||
await sendThroughRelay(pathId, envelope);
|
||||
},
|
||||
});
|
||||
probe.start();
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
CircuitBreakerRegistry,
|
||||
HalfOpenProbe,
|
||||
} from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const probe = new HalfOpenProbe({
|
||||
registry,
|
||||
probeIntervalMs: 1_000,
|
||||
probeMessageFactory: () => makeProbeEnvelope(),
|
||||
send: realSend,
|
||||
setIntervalFn: testHarness.setInterval,
|
||||
clearIntervalFn: testHarness.clearInterval,
|
||||
});
|
||||
|
||||
probe.start();
|
||||
testHarness.advance(1_000);
|
||||
await probe.tick(); // explicit tick if the harness doesn't auto-fire
|
||||
probe.stop();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type ProbeSender = (pathId, envelope) => Promise<void>`.
|
||||
- `interface HalfOpenProbeOptions { registry, probeIntervalMs?, probeMessageFactory, send, setIntervalFn?, clearIntervalFn? }`.
|
||||
- `start()` — idempotent.
|
||||
- `stop()` — idempotent; cancels the timer.
|
||||
- `tick()` — runs one probe pass; resolves once every half-open path has a result.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`tick()` lists `OPEN` paths from the registry and then re-queries `getState(pathId)` to filter for `HALF_OPEN` — this catches the lazy `OPEN → HALF_OPEN` transition without forcing the registry to expose a separate "ready to probe" query. A failed probe trips the breaker back to `OPEN` via `recordFailure`; a successful probe accumulates toward `successThreshold` and eventually closes the breaker. The probe never throws — every send error is swallowed into `recordFailure` so a transient probe rejection doesn't crash the periodic loop.
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { MessageEnvelope } from "@suaveplan/network";
|
||||
import type { CircuitBreakerRegistry } from "../registry/circuit-breaker-registry.js";
|
||||
|
||||
/**
|
||||
* Function that delivers a probe envelope to a remote path. Resolves
|
||||
* iff the path acknowledged the probe; rejects on transport error or
|
||||
* timeout.
|
||||
*/
|
||||
export type ProbeSender = (
|
||||
pathId: string,
|
||||
envelope: MessageEnvelope
|
||||
) => Promise<void>;
|
||||
|
||||
/**
|
||||
* Configuration for `HalfOpenProbe`.
|
||||
*/
|
||||
export interface HalfOpenProbeOptions {
|
||||
readonly registry: CircuitBreakerRegistry;
|
||||
readonly probeIntervalMs?: number;
|
||||
readonly probeMessageFactory: (pathId: string) => MessageEnvelope;
|
||||
readonly send: ProbeSender;
|
||||
readonly setIntervalFn?: typeof setInterval;
|
||||
readonly clearIntervalFn?: typeof clearInterval;
|
||||
}
|
||||
|
||||
/**
|
||||
* Periodically issues lightweight probe messages against every path
|
||||
* the registry reports as in HALF_OPEN, recording the result through
|
||||
* the relevant breaker. Stops cleanly via `stop()` so callers don't
|
||||
* leak timers in test harnesses.
|
||||
*/
|
||||
export class HalfOpenProbe {
|
||||
private readonly registry: CircuitBreakerRegistry;
|
||||
private readonly probeIntervalMs: number;
|
||||
private readonly probeMessageFactory: (pathId: string) => MessageEnvelope;
|
||||
private readonly send: ProbeSender;
|
||||
private readonly setIntervalFn: typeof setInterval;
|
||||
private readonly clearIntervalFn: typeof clearInterval;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
public constructor(options: HalfOpenProbeOptions) {
|
||||
this.registry = options.registry;
|
||||
this.probeIntervalMs = options.probeIntervalMs ?? 5_000;
|
||||
this.probeMessageFactory = options.probeMessageFactory;
|
||||
this.send = options.send;
|
||||
this.setIntervalFn = options.setIntervalFn ?? setInterval;
|
||||
this.clearIntervalFn = options.clearIntervalFn ?? clearInterval;
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
if (this.timer !== null) return;
|
||||
this.timer = this.setIntervalFn(() => {
|
||||
void this.tick();
|
||||
}, this.probeIntervalMs);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
if (this.timer !== null) {
|
||||
this.clearIntervalFn(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
}
|
||||
|
||||
public async tick(): Promise<void> {
|
||||
const targets = this.findHalfOpenPaths();
|
||||
for (const pathId of targets) {
|
||||
const breaker = this.registry.getOrCreate(pathId);
|
||||
if (!breaker.canExecute()) continue;
|
||||
const envelope = this.probeMessageFactory(pathId);
|
||||
try {
|
||||
await this.send(pathId, envelope);
|
||||
breaker.recordSuccess();
|
||||
} catch {
|
||||
breaker.recordFailure();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private findHalfOpenPaths(): readonly string[] {
|
||||
const out: string[] = [];
|
||||
for (const pathId of this.registry.paths()) {
|
||||
if (this.registry.getState(pathId) === "HALF_OPEN") {
|
||||
out.push(pathId);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import type { MessageEnvelope } from "@suaveplan/network";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { CircuitBreakerRegistry } from "../registry/circuit-breaker-registry.js";
|
||||
import { HalfOpenProbe } from "./half-open-probe.js";
|
||||
|
||||
function makeEnvelope(pathId: string): MessageEnvelope {
|
||||
return {
|
||||
id: pathId,
|
||||
type: "relay:probe",
|
||||
senderId: "self",
|
||||
payload: null,
|
||||
timestamp: 0,
|
||||
headers: {},
|
||||
ttl: 8,
|
||||
visitedNodes: [],
|
||||
} as MessageEnvelope;
|
||||
}
|
||||
|
||||
describe("HalfOpenProbe", () => {
|
||||
it("probes only HALF_OPEN paths and records success", async () => {
|
||||
let now = 0;
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: {
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
successThreshold: 1,
|
||||
clock: () => now,
|
||||
},
|
||||
});
|
||||
registry.getOrCreate("alive");
|
||||
const dead = registry.getOrCreate("dead");
|
||||
dead.recordFailure();
|
||||
now += 200;
|
||||
// dead is now lazily HALF_OPEN
|
||||
const sent: string[] = [];
|
||||
const probe = new HalfOpenProbe({
|
||||
registry,
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async (pathId) => {
|
||||
sent.push(pathId);
|
||||
},
|
||||
});
|
||||
await probe.tick();
|
||||
expect(sent).toEqual(["dead"]);
|
||||
expect(registry.getState("dead")).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("records failure when send rejects", async () => {
|
||||
let now = 0;
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: {
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
clock: () => now,
|
||||
},
|
||||
});
|
||||
const dead = registry.getOrCreate("dead");
|
||||
dead.recordFailure();
|
||||
now += 200;
|
||||
const probe = new HalfOpenProbe({
|
||||
registry,
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async () => {
|
||||
throw new Error("probe-rejected");
|
||||
},
|
||||
});
|
||||
await probe.tick();
|
||||
expect(registry.getState("dead")).toBe("OPEN");
|
||||
});
|
||||
|
||||
it("skips paths where canExecute is false", async () => {
|
||||
let now = 0;
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: {
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
halfOpenMaxCalls: 1,
|
||||
clock: () => now,
|
||||
},
|
||||
});
|
||||
const dead = registry.getOrCreate("dead");
|
||||
dead.recordFailure();
|
||||
now += 200;
|
||||
// exhaust the half-open call budget
|
||||
dead.recordSuccess();
|
||||
const sent: string[] = [];
|
||||
const probe = new HalfOpenProbe({
|
||||
registry,
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async (pathId) => {
|
||||
sent.push(pathId);
|
||||
},
|
||||
});
|
||||
await probe.tick();
|
||||
// dead is now CLOSED (success closed it) so listOpenPaths returns []
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("start/stop are idempotent and use injected timer fns", () => {
|
||||
const setIntervalFn = ((cb: () => void) => {
|
||||
// capture to ensure it's installed
|
||||
return cb as never;
|
||||
}) as never;
|
||||
const clearIntervalFn = (() => {}) as never;
|
||||
const probe = new HalfOpenProbe({
|
||||
registry: new CircuitBreakerRegistry(),
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async () => {},
|
||||
setIntervalFn,
|
||||
clearIntervalFn,
|
||||
});
|
||||
probe.start();
|
||||
probe.start();
|
||||
probe.stop();
|
||||
probe.stop();
|
||||
});
|
||||
|
||||
it("ignores OPEN paths that have not cooled down yet", async () => {
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 1, openDurationMs: 100_000 },
|
||||
});
|
||||
registry.getOrCreate("dead").recordFailure();
|
||||
const sent: string[] = [];
|
||||
const probe = new HalfOpenProbe({
|
||||
registry,
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async (pathId) => {
|
||||
sent.push(pathId);
|
||||
},
|
||||
});
|
||||
await probe.tick();
|
||||
expect(sent).toEqual([]);
|
||||
});
|
||||
|
||||
it("uses default setInterval/clearInterval when not injected", () => {
|
||||
const probe = new HalfOpenProbe({
|
||||
registry: new CircuitBreakerRegistry(),
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async () => {},
|
||||
probeIntervalMs: 1_000_000,
|
||||
});
|
||||
probe.start();
|
||||
probe.stop();
|
||||
});
|
||||
|
||||
it("invokes the registered timer callback to drive ticks", async () => {
|
||||
let interval: (() => void) | null = null;
|
||||
const setIntervalFn = ((cb: () => void) => {
|
||||
interval = cb;
|
||||
return 1 as never;
|
||||
}) as never;
|
||||
const clearIntervalFn = (() => {}) as never;
|
||||
const probe = new HalfOpenProbe({
|
||||
registry: new CircuitBreakerRegistry(),
|
||||
probeMessageFactory: makeEnvelope,
|
||||
send: async () => {},
|
||||
setIntervalFn,
|
||||
clearIntervalFn,
|
||||
});
|
||||
probe.start();
|
||||
interval?.();
|
||||
await Promise.resolve();
|
||||
probe.stop();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
# CircuitBreakerRegistry
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Owns one `MeshCircuitBreaker` per relay path and exposes a thin facade for cross-path queries (`listOpenPaths`, `getState(pathId)`). Centralising the breakers means there's exactly one place that decides "is this path healthy?" and middleware/probes can both consult it without state-sharing.
|
||||
|
||||
## Features
|
||||
|
||||
- Lazy construction: `getOrCreate` makes a breaker on first reference, returns the existing instance on subsequent calls.
|
||||
- Per-call config overrides on `getOrCreate` so different paths can have different thresholds.
|
||||
- Shared `defaults` config applied to every breaker that doesn't override.
|
||||
- `listOpenPaths()` for observability dashboards / health endpoints.
|
||||
- `remove(pathId)` for explicit teardown when a peer leaves the mesh.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { CircuitBreakerRegistry } from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const registry = new CircuitBreakerRegistry();
|
||||
const cb = registry.getOrCreate("meshA:host1→meshB:host2");
|
||||
if (cb.canExecute()) {
|
||||
/* ... */
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { CircuitBreakerRegistry } from "@suaveplan/network-plugin-circuit-breaker";
|
||||
|
||||
const registry = new CircuitBreakerRegistry({
|
||||
defaults: {
|
||||
failureThreshold: 3,
|
||||
successThreshold: 2,
|
||||
openDurationMs: 10_000,
|
||||
halfOpenMaxCalls: 2,
|
||||
},
|
||||
});
|
||||
|
||||
// per-path override for a known-flaky link
|
||||
registry.getOrCreate("meshA:host1→meshC:host5", {
|
||||
failureThreshold: 10,
|
||||
});
|
||||
|
||||
const open = registry.listOpenPaths();
|
||||
if (open.length > 0) {
|
||||
metrics.gauge("mesh.open_breakers", open.length);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface CircuitBreakerRegistryOptions { defaults? }`.
|
||||
- `getOrCreate(pathId, options?): MeshCircuitBreaker`.
|
||||
- `getState(pathId): CircuitBreakerState | undefined`.
|
||||
- `remove(pathId): boolean`.
|
||||
- `listOpenPaths(): readonly string[]`.
|
||||
- `paths(): readonly string[]` — every known path id, regardless of state.
|
||||
- `size(): number`.
|
||||
- `clear()`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`getOrCreate` is the canonical constructor — the registry never returns a raw `new` breaker, so observers don't have to wonder whether they're looking at a registry-owned or one-off instance. Per-call options shallow-merge over `defaults`, with caller options winning. `listOpenPaths` consults `getState()` (not the cached state) so paths that have lazy-transitioned to `HALF_OPEN` are excluded from the open list — matching what callers expect from "which paths are dead right now?". The companion `paths()` accessor returns every known path so probe loops can re-query each one's live state without missing entries that have already moved to `HALF_OPEN`. This split keeps health-dashboard semantics ("just the dead ones") separate from probe-driver semantics ("everything we might want to retry").
|
||||
@@ -0,0 +1,68 @@
|
||||
import {
|
||||
type CircuitBreakerState,
|
||||
MeshCircuitBreaker,
|
||||
type MeshCircuitBreakerOptions,
|
||||
} from "../breaker/mesh-circuit-breaker.js";
|
||||
|
||||
/**
|
||||
* Configuration shared across every breaker created by the registry.
|
||||
*/
|
||||
export interface CircuitBreakerRegistryOptions {
|
||||
readonly defaults?: Omit<MeshCircuitBreakerOptions, "pathId">;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns one `MeshCircuitBreaker` per `(pathId)` and lets callers ask
|
||||
* questions across the full set without iterating manually.
|
||||
*/
|
||||
export class CircuitBreakerRegistry {
|
||||
private readonly breakers = new Map<string, MeshCircuitBreaker>();
|
||||
private readonly defaults: Omit<MeshCircuitBreakerOptions, "pathId">;
|
||||
|
||||
public constructor(options: CircuitBreakerRegistryOptions = {}) {
|
||||
this.defaults = options.defaults ?? {};
|
||||
}
|
||||
|
||||
public getOrCreate(
|
||||
pathId: string,
|
||||
options?: Omit<MeshCircuitBreakerOptions, "pathId">
|
||||
): MeshCircuitBreaker {
|
||||
const existing = this.breakers.get(pathId);
|
||||
if (existing !== undefined) return existing;
|
||||
const breaker = new MeshCircuitBreaker({
|
||||
pathId,
|
||||
...this.defaults,
|
||||
...options,
|
||||
});
|
||||
this.breakers.set(pathId, breaker);
|
||||
return breaker;
|
||||
}
|
||||
|
||||
public getState(pathId: string): CircuitBreakerState | undefined {
|
||||
return this.breakers.get(pathId)?.getState();
|
||||
}
|
||||
|
||||
public remove(pathId: string): boolean {
|
||||
return this.breakers.delete(pathId);
|
||||
}
|
||||
|
||||
public listOpenPaths(): readonly string[] {
|
||||
const out: string[] = [];
|
||||
for (const [pathId, breaker] of this.breakers) {
|
||||
if (breaker.getState() === "OPEN") out.push(pathId);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
public size(): number {
|
||||
return this.breakers.size;
|
||||
}
|
||||
|
||||
public paths(): readonly string[] {
|
||||
return [...this.breakers.keys()];
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
this.breakers.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { CircuitBreakerRegistry } from "./circuit-breaker-registry.js";
|
||||
|
||||
describe("CircuitBreakerRegistry", () => {
|
||||
it("creates and reuses breakers per pathId", () => {
|
||||
const r = new CircuitBreakerRegistry();
|
||||
const a = r.getOrCreate("p");
|
||||
const b = r.getOrCreate("p");
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("applies defaults to new breakers", () => {
|
||||
const r = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 1, openDurationMs: 100_000 },
|
||||
});
|
||||
const cb = r.getOrCreate("p");
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
});
|
||||
|
||||
it("per-path overrides win over defaults", () => {
|
||||
const r = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 1 },
|
||||
});
|
||||
const cb = r.getOrCreate("p", { failureThreshold: 2 });
|
||||
cb.recordFailure();
|
||||
expect(cb.getState()).toBe("CLOSED");
|
||||
});
|
||||
|
||||
it("getState returns undefined for unknown paths", () => {
|
||||
const r = new CircuitBreakerRegistry();
|
||||
expect(r.getState("ghost")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("listOpenPaths returns only OPEN paths", () => {
|
||||
const r = new CircuitBreakerRegistry({
|
||||
defaults: { failureThreshold: 1, openDurationMs: 100_000 },
|
||||
});
|
||||
r.getOrCreate("p1").recordFailure();
|
||||
r.getOrCreate("p2");
|
||||
expect(r.listOpenPaths()).toEqual(["p1"]);
|
||||
});
|
||||
|
||||
it("excludes lazy HALF_OPEN paths from listOpenPaths", () => {
|
||||
let now = 0;
|
||||
const r = new CircuitBreakerRegistry({
|
||||
defaults: {
|
||||
failureThreshold: 1,
|
||||
openDurationMs: 100,
|
||||
clock: () => now,
|
||||
},
|
||||
});
|
||||
r.getOrCreate("p1").recordFailure();
|
||||
now += 200;
|
||||
expect(r.listOpenPaths()).toEqual([]);
|
||||
});
|
||||
|
||||
it("remove deletes a breaker", () => {
|
||||
const r = new CircuitBreakerRegistry();
|
||||
r.getOrCreate("p");
|
||||
expect(r.remove("p")).toBe(true);
|
||||
expect(r.size()).toBe(0);
|
||||
expect(r.remove("p")).toBe(false);
|
||||
});
|
||||
|
||||
it("clear empties the registry", () => {
|
||||
const r = new CircuitBreakerRegistry();
|
||||
r.getOrCreate("p1");
|
||||
r.getOrCreate("p2");
|
||||
r.clear();
|
||||
expect(r.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("default registry uses empty defaults", () => {
|
||||
const r = new CircuitBreakerRegistry();
|
||||
const cb = r.getOrCreate("p");
|
||||
// default failureThreshold = 5
|
||||
for (let i = 0; i < 5; i++) cb.recordFailure();
|
||||
expect(cb.getState()).toBe("OPEN");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("network-plugin-circuit-breaker dep-graph", () => {
|
||||
it("loads index.js (acyclic static imports)", async () => {
|
||||
const mod = await import("../src/index.js");
|
||||
expect(mod.MeshCircuitBreaker).toBeDefined();
|
||||
expect(mod.CircuitBreakerRegistry).toBeDefined();
|
||||
expect(mod.HalfOpenProbe).toBeDefined();
|
||||
expect(mod.createCircuitBreakerMiddleware).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
// Ensure `globalThis.crypto` is available in bare-Node vitest runs.
|
||||
import { webcrypto } from "node:crypto";
|
||||
|
||||
type WritableGlobal = { crypto?: Crypto };
|
||||
const g = globalThis as WritableGlobal;
|
||||
if (g.crypto === undefined) {
|
||||
g.crypto = webcrypto as unknown as Crypto;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"references": [{ "path": "./tsconfig.lib.json" }]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir:
|
||||
"../../../node_modules/.vite/packages/network-plugin-circuit-breaker",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/network",
|
||||
"@suaveplan/types",
|
||||
"tslib",
|
||||
"zod",
|
||||
"@opentelemetry/api",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: [
|
||||
"src/__benchmarks__/**",
|
||||
"benchmarks/**",
|
||||
"e2e/**",
|
||||
"dist/**",
|
||||
"node_modules/**",
|
||||
],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/**/*.schemas.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 5000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user