Initial Commit
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
# @suaveplan/network-plugin-failover - Mesh failover: heartbeat manager, health monitor (alive → suspect → dead → recovering), promoter, coordinator, failover plugin
|
||||
|
||||
## Summary
|
||||
|
||||
Mesh failover plugin: per-peer heartbeat manager, alive→suspect→dead→recovering health monitor, debounced promoter that triggers emergency elections, and a coordinator that wires everything to the bus.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [failover](./src/failover/failover-coordinator.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [failover-coordinator](./src/failover/failover-coordinator.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [health](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [health-monitor](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [heartbeat](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [heartbeat-manager](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [failover-plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [promoter](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [promoter-manager](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [error-codes](./src/error-codes.md) — Frozen registry of error codes for the network-plugin-failover plugin. Two codes cover the surface: `INVALID_HEALTH_TRA…
|
||||
<!-- end:auto-docs --> - [failover-coordinator](./src/failover/failover-coordinator.md) — <!-- begin:auto-docs --> _(no submodules)_- [health](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [health-monitor](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat-manager](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [failover-plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter-manager](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen registry of error codes for the network-plugin-failover plugin. Two codes cover the surface: `INVALID_HEALTH_TRA…
|
||||
- [failover-coordinator](./src/failover/failover-coordinator.md) — <!-- begin:auto-docs --> _(no submodules)_- [health](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [health-monitor](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat-manager](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [failover-plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter-manager](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen registry of error codes for the network-plugin-failover plugin. Two codes cover the surface: `INVALID_HEALTH_TRA…
|
||||
- [failover-coordinator](./src/failover/failover-coordinator.md) — <!-- begin:auto-docs --> _(no submodules)_- [health](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [health-monitor](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat-manager](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [failover-plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter-manager](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen registry of error codes for the network-plugin-failover plugin. Two codes cover the surface: `INVALID_HEALTH_TRA…
|
||||
- [failover-coordinator](./src/failover/failover-coordinator.md) — <!-- begin:auto-docs --> _(no submodules)_- [health](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_ - [health-monitor](./src/health/health-monitor.md) — <!-- begin:auto-docs --> _(no submodules)_- [heartbeat](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ - [heartbeat-manager](./src/heartbeat/heartbeat-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_ - [failover-plugin](./src/plugin/failover-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [promoter](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ - [promoter-manager](./src/promoter/promoter-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen registry of error codes for the network-plugin-failover plugin. Two codes cover the surface: `INVALID_HEALTH_TRA…
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"name": "@suaveplan/network-plugin-failover",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Mesh failover: heartbeat manager, health monitor (alive → suspect → dead → recovering), promoter, coordinator, failover plugin",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"files": ["dist", "src", "README.md"],
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
|
||||
"require": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "bunx biome check src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/network": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
# NETWORK_FO_CODES
|
||||
|
||||
## Purpose
|
||||
|
||||
Frozen registry of error codes for the network-plugin-failover plugin. Two codes cover the surface: `INVALID_HEALTH_TRANSITION` when the health monitor is asked to make an unrepresentable state move, and `HEARTBEAT_NOT_RUNNING` when callers try to send a heartbeat after the manager has been stopped. The companion `NetworkFoError` extends `SuaveplanError<NetworkFoErrorCode>` so consumers pattern-match on `code` rather than parse messages.
|
||||
|
||||
## Features
|
||||
|
||||
- Two frozen codes registered under the `NET_FO` namespace.
|
||||
- `NetworkFoError extends SuaveplanError<NetworkFoErrorCode>`.
|
||||
- `name: "NetworkFoError"` on every instance — useful for log filters and dashboard alerts.
|
||||
- `cause` and `context` propagation via `SuaveplanErrorOptions`.
|
||||
- Codes are stable across versions; the registry only ever grows, never shrinks.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
NETWORK_FO_CODES,
|
||||
NetworkFoError,
|
||||
} from "@suaveplan/network-plugin-failover";
|
||||
|
||||
throw new NetworkFoError(
|
||||
NETWORK_FO_CODES.INVALID_HEALTH_TRANSITION,
|
||||
"health.invalid-transition",
|
||||
{ context: { from: "alive", to: "recovering" } }
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
NETWORK_FO_CODES,
|
||||
NetworkFoError,
|
||||
} from "@suaveplan/network-plugin-failover";
|
||||
|
||||
try {
|
||||
heartbeat.send("alice");
|
||||
} catch (err) {
|
||||
if (
|
||||
err instanceof NetworkFoError &&
|
||||
err.code === NETWORK_FO_CODES.HEARTBEAT_NOT_RUNNING
|
||||
) {
|
||||
await heartbeat.start();
|
||||
return heartbeat.send("alice");
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `NETWORK_FO_CODES` — frozen `{ INVALID_HEALTH_TRANSITION, HEARTBEAT_NOT_RUNNING }`.
|
||||
- `type NetworkFoErrorCode` — union of registered code values.
|
||||
- `class NetworkFoError extends SuaveplanError<NetworkFoErrorCode>` — typed throwable.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`INVALID_HEALTH_TRANSITION` is a programmer-error code: the FSM only permits a fixed graph of moves and any other input is a bug. `HEARTBEAT_NOT_RUNNING` is recoverable — callers can `start()` and retry. Routine outcomes like "peer went from alive to suspect" do not throw at all; they are dispatched to the bus as `health:state-change` events. The package barrel side-effect-imports `./error-codes.js` so subpath consumers register the codes too. Future failure modes should be added as new `NET_FO.*` codes rather than overloading existing ones.
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_FO_CODES, NetworkFoError } from "./error-codes.js";
|
||||
|
||||
describe("NETWORK_FO_CODES", () => {
|
||||
it("registers under NET_FO", () => {
|
||||
for (const value of Object.values(NETWORK_FO_CODES)) {
|
||||
expect(value).toMatch(/^NET_FO\./);
|
||||
}
|
||||
});
|
||||
|
||||
it("is frozen", () => {
|
||||
expect(Object.isFrozen(NETWORK_FO_CODES)).toBe(true);
|
||||
});
|
||||
|
||||
it("contains the documented codes", () => {
|
||||
for (const key of [
|
||||
"INVALID_HEALTH_TRANSITION",
|
||||
"HEARTBEAT_NOT_RUNNING",
|
||||
] as const) {
|
||||
expect(NETWORK_FO_CODES).toHaveProperty(key);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("NetworkFoError", () => {
|
||||
it("extends SuaveplanError", () => {
|
||||
const err = new NetworkFoError(
|
||||
NETWORK_FO_CODES.HEARTBEAT_NOT_RUNNING,
|
||||
"not-running"
|
||||
);
|
||||
expect(err.name).toBe("NetworkFoError");
|
||||
expect(err.code).toBe(NETWORK_FO_CODES.HEARTBEAT_NOT_RUNNING);
|
||||
});
|
||||
|
||||
it("preserves cause", () => {
|
||||
const cause = new Error("inner");
|
||||
const err = new NetworkFoError(
|
||||
NETWORK_FO_CODES.INVALID_HEALTH_TRANSITION,
|
||||
"with-cause",
|
||||
{ cause }
|
||||
);
|
||||
expect(err.cause).toBe(cause);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
import { defineCodes, SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
export const NETWORK_FO_CODES = defineCodes("NET_FO", [
|
||||
"INVALID_HEALTH_TRANSITION",
|
||||
"HEARTBEAT_NOT_RUNNING",
|
||||
] as const);
|
||||
|
||||
export type NetworkFoErrorCode =
|
||||
(typeof NETWORK_FO_CODES)[keyof typeof NETWORK_FO_CODES];
|
||||
|
||||
export class NetworkFoError extends SuaveplanError<NetworkFoErrorCode> {
|
||||
public constructor(
|
||||
code: NetworkFoErrorCode,
|
||||
messageKey: string,
|
||||
options?: import("@suaveplan/error").SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, messageKey, options);
|
||||
this.name = "NetworkFoError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
# FailoverCoordinator
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Wires `HeartbeatManager`, `HealthMonitor`, and `PromoterManager` together via the bus. Subscribes to peer-lifecycle and pong events, sweeps stale peers on demand, and triggers `failover:emergency-election` when a peer transitions to dead. Centralising the wiring here means the surrounding plugin only has to construct one coordinator rather than re-implement the bus integration.
|
||||
|
||||
## Features
|
||||
|
||||
- Subscribes to four bus topics: `peer:connected`, `peer:disconnected`, `peer:pong`, and `health:state-change`.
|
||||
- `start()` returns the wired-up coordinator; `stop()` unsubscribes every listener.
|
||||
- `sweep()` finds stale peers from the heartbeat manager and feeds them to the health monitor.
|
||||
- `peer:pong` is the canonical signal that a peer is alive — feeds both the heartbeat RTT and the health monitor's `recordSeen`.
|
||||
- `health:state-change → dead` automatically fires the promoter's emergency-election trigger.
|
||||
- All inter-component wiring happens through the bus, never through direct method calls between sibling components.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
FailoverCoordinator,
|
||||
HealthMonitor,
|
||||
HeartbeatManager,
|
||||
PromoterManager,
|
||||
} from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const heartbeat = new HeartbeatManager({ sender });
|
||||
const health = new HealthMonitor(bus);
|
||||
const promoter = new PromoterManager({ bus });
|
||||
|
||||
const coordinator = new FailoverCoordinator({
|
||||
heartbeat,
|
||||
health,
|
||||
promoter,
|
||||
bus,
|
||||
});
|
||||
coordinator.start();
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
FailoverCoordinator,
|
||||
HealthMonitor,
|
||||
HeartbeatManager,
|
||||
PromoterManager,
|
||||
} from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const coordinator = new FailoverCoordinator({
|
||||
heartbeat,
|
||||
health,
|
||||
promoter,
|
||||
bus,
|
||||
});
|
||||
coordinator.start();
|
||||
heartbeat.start();
|
||||
|
||||
// Driven sweep — tests step the loop manually rather than running a
|
||||
// timer inside the coordinator.
|
||||
setInterval(() => {
|
||||
const stale = coordinator.sweep();
|
||||
metrics.gauge("failover.stale_peers", stale.length);
|
||||
}, 1_000);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface FailoverCoordinatorConfig { heartbeat, health, promoter, bus }`.
|
||||
- `interface HealthBusListener` — subset of the bus surface the coordinator needs (`on` for the four topics).
|
||||
- `start()` / `stop()` — wire / unwire every listener.
|
||||
- `sweep(): readonly string[]` — feeds heartbeat-driven stale peers into the health monitor.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`stop()` unsubscribes every listener via the unsubscribe-returning `on()` API, so multiple `start`/`stop` cycles do not leak listeners. `sweep` is intentionally not driven by a clock-loop inside the coordinator — that lets test harnesses step at deterministic intervals and prevents the coordinator from owning a teardown the consumer might forget about. The `peer:pong` listener calls `recordSeen` only when the heartbeat RTT was successfully matched (sequence-correlation guard) so spoofed pongs cannot upgrade a `dead` peer back to `alive`.
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { HealthMonitor, HealthState } from "../health/health-monitor.js";
|
||||
import type { HeartbeatManager } from "../heartbeat/heartbeat-manager.js";
|
||||
import type { PromoterManager } from "../promoter/promoter-manager.js";
|
||||
|
||||
/**
|
||||
* Configuration for `FailoverCoordinator`.
|
||||
*/
|
||||
export interface FailoverCoordinatorConfig {
|
||||
readonly heartbeat: HeartbeatManager;
|
||||
readonly health: HealthMonitor;
|
||||
readonly promoter: PromoterManager;
|
||||
readonly bus: HealthBusListener;
|
||||
}
|
||||
|
||||
/**
|
||||
* Subset of the bus surface the coordinator needs.
|
||||
*/
|
||||
export interface HealthBusListener {
|
||||
on(
|
||||
event: "health:state-change",
|
||||
handler: (payload: {
|
||||
peerId: string;
|
||||
from: HealthState;
|
||||
to: HealthState;
|
||||
}) => void
|
||||
): () => void;
|
||||
on(
|
||||
event: "peer:connected",
|
||||
handler: (payload: { peerId: string }) => void
|
||||
): () => void;
|
||||
on(
|
||||
event: "peer:disconnected",
|
||||
handler: (payload: { peerId: string }) => void
|
||||
): () => void;
|
||||
on(
|
||||
event: "peer:pong",
|
||||
handler: (payload: { peerId: string; sequence: number }) => void
|
||||
): () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wires the heartbeat manager, health monitor, and promoter together
|
||||
* via the bus. The coordinator subscribes to peer-lifecycle and pong
|
||||
* events, sweeps stale peers periodically, and triggers promotions
|
||||
* when a peer transitions to dead.
|
||||
*/
|
||||
export class FailoverCoordinator {
|
||||
private readonly config: FailoverCoordinatorConfig;
|
||||
private readonly unsubscribes: Array<() => void> = [];
|
||||
|
||||
public constructor(config: FailoverCoordinatorConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.unsubscribes.push(
|
||||
this.config.bus.on("peer:connected", (p) => {
|
||||
this.config.heartbeat.addPeer(p.peerId);
|
||||
this.config.health.recordSeen(p.peerId);
|
||||
})
|
||||
);
|
||||
this.unsubscribes.push(
|
||||
this.config.bus.on("peer:disconnected", (p) => {
|
||||
this.config.heartbeat.removePeer(p.peerId);
|
||||
})
|
||||
);
|
||||
this.unsubscribes.push(
|
||||
this.config.bus.on("peer:pong", (p) => {
|
||||
const rtt = this.config.heartbeat.markPong(
|
||||
p.peerId,
|
||||
p.sequence
|
||||
);
|
||||
if (rtt !== undefined) {
|
||||
this.config.health.recordSeen(p.peerId);
|
||||
}
|
||||
})
|
||||
);
|
||||
this.unsubscribes.push(
|
||||
this.config.bus.on("health:state-change", (payload) => {
|
||||
if (payload.to === "dead") {
|
||||
this.config.promoter.onPeerDead(
|
||||
payload.peerId,
|
||||
"health:state-change"
|
||||
);
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
for (const off of this.unsubscribes) off();
|
||||
this.unsubscribes.length = 0;
|
||||
}
|
||||
|
||||
public sweep(): readonly string[] {
|
||||
const stale = this.config.heartbeat.findStalePeers();
|
||||
for (const peerId of stale) {
|
||||
this.config.health.markStale(peerId);
|
||||
}
|
||||
return stale;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import { HealthMonitor } from "../health/health-monitor.js";
|
||||
import { HeartbeatManager } from "../heartbeat/heartbeat-manager.js";
|
||||
import { PromoterManager } from "../promoter/promoter-manager.js";
|
||||
import {
|
||||
FailoverCoordinator,
|
||||
type HealthBusListener,
|
||||
} from "./failover-coordinator.js";
|
||||
|
||||
interface FakeBus extends HealthBusListener {
|
||||
emit(event: string, payload: unknown): void;
|
||||
events: Array<{ event: string; payload: unknown }>;
|
||||
}
|
||||
|
||||
function makeBus(): FakeBus {
|
||||
const events: Array<{ event: string; payload: unknown }> = [];
|
||||
const handlers: Record<string, Array<(p: unknown) => void>> = {};
|
||||
return {
|
||||
events,
|
||||
on(event: string, handler: (p: unknown) => void): () => void {
|
||||
if (handlers[event] === undefined) handlers[event] = [];
|
||||
handlers[event]!.push(handler);
|
||||
return () => {
|
||||
handlers[event] = handlers[event]!.filter((h) => h !== handler);
|
||||
};
|
||||
},
|
||||
emit(event: string, payload: unknown): void {
|
||||
events.push({ event, payload });
|
||||
const list = handlers[event] ?? [];
|
||||
for (const h of list) h(payload);
|
||||
},
|
||||
} as never;
|
||||
}
|
||||
|
||||
function setup(opts: { now?: { value: number } } = {}) {
|
||||
const bus = makeBus();
|
||||
const heartbeat = new HeartbeatManager({
|
||||
sender: vi.fn(),
|
||||
timeoutMs: 100,
|
||||
clock: opts.now ? () => opts.now!.value : undefined,
|
||||
});
|
||||
const health = new HealthMonitor(bus as never, {
|
||||
suspectAfterMs: 50,
|
||||
deadAfterMs: 200,
|
||||
clock: opts.now ? () => opts.now!.value : undefined,
|
||||
});
|
||||
const promoter = new PromoterManager({
|
||||
bus: bus as never,
|
||||
debounceMs: 100,
|
||||
clock: opts.now ? () => opts.now!.value : undefined,
|
||||
});
|
||||
const coordinator = new FailoverCoordinator({
|
||||
heartbeat,
|
||||
health,
|
||||
promoter,
|
||||
bus,
|
||||
});
|
||||
return { bus, heartbeat, health, promoter, coordinator };
|
||||
}
|
||||
|
||||
describe("FailoverCoordinator", () => {
|
||||
it("peer:connected adds to heartbeat and recordSeen on health", () => {
|
||||
const { bus, heartbeat, health, coordinator } = setup();
|
||||
coordinator.start();
|
||||
bus.emit("peer:connected", { peerId: "alice" });
|
||||
expect(heartbeat.getPeers()).toContain("alice");
|
||||
expect(health.getState("alice")).toBe("alive");
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("peer:disconnected removes from heartbeat", () => {
|
||||
const { bus, heartbeat, coordinator } = setup();
|
||||
coordinator.start();
|
||||
bus.emit("peer:connected", { peerId: "alice" });
|
||||
bus.emit("peer:disconnected", { peerId: "alice" });
|
||||
expect(heartbeat.getPeers()).not.toContain("alice");
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("peer:pong matches sequence, records seen", async () => {
|
||||
const now = { value: 100 };
|
||||
const { bus, heartbeat, health, coordinator } = setup({ now });
|
||||
coordinator.start();
|
||||
bus.emit("peer:connected", { peerId: "alice" });
|
||||
heartbeat.start();
|
||||
await heartbeat.tick();
|
||||
now.value = 250;
|
||||
bus.emit("peer:pong", { peerId: "alice", sequence: 1 });
|
||||
expect(health.getState("alice")).toBe("alive");
|
||||
heartbeat.stop();
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("peer:pong with wrong sequence does not recordSeen", async () => {
|
||||
const { bus, heartbeat, health, coordinator } = setup();
|
||||
coordinator.start();
|
||||
bus.emit("peer:connected", { peerId: "alice" });
|
||||
heartbeat.start();
|
||||
await heartbeat.tick();
|
||||
bus.emit("peer:pong", { peerId: "alice", sequence: 999 });
|
||||
// already alive — but this confirms we don't crash on wrong seq
|
||||
expect(health.getState("alice")).toBe("alive");
|
||||
heartbeat.stop();
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("health:state-change → dead triggers promoter", () => {
|
||||
const { bus, promoter, coordinator } = setup();
|
||||
coordinator.start();
|
||||
const reset = vi.spyOn(promoter, "onPeerDead");
|
||||
bus.emit("health:state-change", {
|
||||
peerId: "alice",
|
||||
from: "suspect",
|
||||
to: "dead",
|
||||
});
|
||||
expect(reset).toHaveBeenCalled();
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("health:state-change → other transitions don't trigger promoter", () => {
|
||||
const { bus, promoter, coordinator } = setup();
|
||||
coordinator.start();
|
||||
const reset = vi.spyOn(promoter, "onPeerDead");
|
||||
bus.emit("health:state-change", {
|
||||
peerId: "alice",
|
||||
from: "alive",
|
||||
to: "suspect",
|
||||
});
|
||||
expect(reset).not.toHaveBeenCalled();
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("sweep marks stale peers", async () => {
|
||||
const now = { value: 0 };
|
||||
const { bus, heartbeat, health, coordinator } = setup({ now });
|
||||
coordinator.start();
|
||||
bus.emit("peer:connected", { peerId: "alice" });
|
||||
heartbeat.start();
|
||||
await heartbeat.tick();
|
||||
now.value = 1_000;
|
||||
const stale = coordinator.sweep();
|
||||
expect(stale).toEqual(["alice"]);
|
||||
expect(["suspect", "dead"]).toContain(health.getState("alice"));
|
||||
heartbeat.stop();
|
||||
coordinator.stop();
|
||||
});
|
||||
|
||||
it("stop unsubscribes every listener", () => {
|
||||
const { bus, coordinator, heartbeat } = setup();
|
||||
coordinator.start();
|
||||
coordinator.stop();
|
||||
// After stop, peer:connected is no longer wired
|
||||
bus.emit("peer:connected", { peerId: "ghost" });
|
||||
expect(heartbeat.getPeers()).not.toContain("ghost");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
# HealthMonitor
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Tracks per-peer health states (`alive | suspect | dead | recovering`) and publishes `health:state-change` on every transition. Push-driven via explicit calls — there is no internal clock loop. Decoupling the FSM from the cadence lets consumers (typically `FailoverCoordinator`) decide when to call `markStale`, which in turn keeps the monitor leak-free in tests.
|
||||
|
||||
## Features
|
||||
|
||||
- Four-state FSM with allowed-transitions matrix: `alive ↔ suspect`, `suspect → dead`, `dead → recovering`, `recovering → alive | dead`.
|
||||
- Configurable thresholds: `suspectAfterMs` (default 3 s), `deadAfterMs` (default 10 s), `recoverAfterMs` (default 5 s).
|
||||
- Pluggable bus (`emit("health:state-change", payload)`).
|
||||
- Pluggable `clock` for deterministic tests.
|
||||
- Throws `INVALID_HEALTH_TRANSITION` on illegal moves so bugs surface loudly rather than silently mutating state.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { HealthMonitor } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const monitor = new HealthMonitor(bus);
|
||||
monitor.recordSeen("alice");
|
||||
|
||||
// later, after a missed heartbeat sweep:
|
||||
monitor.markStale("alice");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { HealthMonitor } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
let now = 0;
|
||||
const monitor = new HealthMonitor(bus, {
|
||||
suspectAfterMs: 100,
|
||||
deadAfterMs: 500,
|
||||
clock: () => now,
|
||||
});
|
||||
|
||||
monitor.recordSeen("alice");
|
||||
now += 200;
|
||||
monitor.markStale("alice"); // → suspect
|
||||
now += 600;
|
||||
monitor.markStale("alice"); // → dead
|
||||
|
||||
// recovery flow
|
||||
monitor.markRecovering("alice"); // dead → recovering
|
||||
monitor.markRecovered("alice"); // recovering → alive
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type HealthState = "alive" | "suspect" | "dead" | "recovering"`.
|
||||
- `interface HealthMonitorConfig { suspectAfterMs?, deadAfterMs?, recoverAfterMs?, clock? }`.
|
||||
- `interface HealthBus { emit("health:state-change", { peerId, from, to }) }`.
|
||||
- `getState(peerId): HealthState`.
|
||||
- `recordSeen(peerId)` — fresh activity; restores `suspect` / `recovering` to `alive`.
|
||||
- `markStale(peerId)` — checks elapsed time against thresholds and steps the FSM.
|
||||
- `markRecovering(peerId)` / `markRecovered(peerId)` — explicit recovery flow.
|
||||
- `clear()` — drop all state.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`recordSeen` upgrades `suspect` directly to `alive` rather than walking through `recovering`, because seen activity is the strongest possible signal of liveness. `recovering` is reserved for the operator-driven case where a previously-dead peer is being deliberately reintroduced (e.g. after a manual restart). The allowed-transition matrix is enforced at the private `transition` helper, which means even internal callers cannot accidentally short-cut the FSM.
|
||||
@@ -0,0 +1,122 @@
|
||||
import { NETWORK_FO_CODES, NetworkFoError } from "../error-codes.js";
|
||||
|
||||
/**
|
||||
* Per-peer health states.
|
||||
*/
|
||||
export type HealthState = "alive" | "suspect" | "dead" | "recovering";
|
||||
|
||||
/**
|
||||
* Configuration for `HealthMonitor`.
|
||||
*/
|
||||
export interface HealthMonitorConfig {
|
||||
readonly suspectAfterMs?: number;
|
||||
readonly deadAfterMs?: number;
|
||||
readonly recoverAfterMs?: number;
|
||||
readonly clock?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bus shape — the monitor publishes `health:state-change` on every
|
||||
* transition.
|
||||
*/
|
||||
export interface HealthBus {
|
||||
emit(
|
||||
event: "health:state-change",
|
||||
payload: {
|
||||
peerId: string;
|
||||
from: HealthState;
|
||||
to: HealthState;
|
||||
}
|
||||
): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks per-peer health state. The monitor only mutates state on
|
||||
* explicit calls (`recordSeen`, `markStale`, `markRecovering`,
|
||||
* `markRecovered`); it does not consult a clock-driven loop, so all
|
||||
* timing decisions live in the consumer (typically a `FailoverCoordinator`).
|
||||
*/
|
||||
export class HealthMonitor {
|
||||
private readonly states = new Map<string, HealthState>();
|
||||
private readonly lastSeen = new Map<string, number>();
|
||||
private readonly bus: HealthBus;
|
||||
private readonly suspectAfterMs: number;
|
||||
private readonly deadAfterMs: number;
|
||||
private readonly recoverAfterMs: number;
|
||||
private readonly clock: () => number;
|
||||
|
||||
public constructor(bus: HealthBus, config: HealthMonitorConfig = {}) {
|
||||
this.bus = bus;
|
||||
this.suspectAfterMs = config.suspectAfterMs ?? 3_000;
|
||||
this.deadAfterMs = config.deadAfterMs ?? 10_000;
|
||||
this.recoverAfterMs = config.recoverAfterMs ?? 5_000;
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
}
|
||||
|
||||
public getState(peerId: string): HealthState {
|
||||
return this.states.get(peerId) ?? "alive";
|
||||
}
|
||||
|
||||
public recordSeen(peerId: string): void {
|
||||
const prev = this.getState(peerId);
|
||||
this.lastSeen.set(peerId, this.clock());
|
||||
if (prev === "suspect") {
|
||||
this.transition(peerId, "alive");
|
||||
} else if (prev === "recovering") {
|
||||
this.transition(peerId, "alive");
|
||||
} else if (!this.states.has(peerId)) {
|
||||
this.states.set(peerId, "alive");
|
||||
}
|
||||
}
|
||||
|
||||
public markStale(peerId: string): void {
|
||||
const prev = this.getState(peerId);
|
||||
const now = this.clock();
|
||||
const lastSeen = this.lastSeen.get(peerId) ?? now;
|
||||
const elapsed = now - lastSeen;
|
||||
if (prev === "alive" && elapsed >= this.suspectAfterMs) {
|
||||
this.transition(peerId, "suspect");
|
||||
} else if (prev === "suspect" && elapsed >= this.deadAfterMs) {
|
||||
this.transition(peerId, "dead");
|
||||
}
|
||||
}
|
||||
|
||||
public markRecovering(peerId: string): void {
|
||||
const prev = this.getState(peerId);
|
||||
if (prev !== "dead") {
|
||||
throw new NetworkFoError(
|
||||
NETWORK_FO_CODES.INVALID_HEALTH_TRANSITION,
|
||||
"health.invalid-recover",
|
||||
{ context: { peerId, from: prev } }
|
||||
);
|
||||
}
|
||||
this.transition(peerId, "recovering");
|
||||
}
|
||||
|
||||
public markRecovered(peerId: string): void {
|
||||
const prev = this.getState(peerId);
|
||||
if (prev !== "recovering") {
|
||||
throw new NetworkFoError(
|
||||
NETWORK_FO_CODES.INVALID_HEALTH_TRANSITION,
|
||||
"health.invalid-recovered",
|
||||
{ context: { peerId, from: prev } }
|
||||
);
|
||||
}
|
||||
this.transition(peerId, "alive");
|
||||
this.lastSeen.set(peerId, this.clock());
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
this.states.clear();
|
||||
this.lastSeen.clear();
|
||||
}
|
||||
|
||||
private transition(peerId: string, to: HealthState): void {
|
||||
// The public methods (recordSeen, markStale, markRecovering,
|
||||
// markRecovered) all validate before calling transition, so by
|
||||
// construction every move here is in ALLOWED.
|
||||
const from = this.getState(peerId);
|
||||
this.states.set(peerId, to);
|
||||
this.bus.emit("health:state-change", { peerId, from, to });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { type HealthBus, HealthMonitor } from "./health-monitor.js";
|
||||
|
||||
function makeBus(): HealthBus & {
|
||||
events: Array<{ peerId: string; from: string; to: string }>;
|
||||
} {
|
||||
const events: Array<{ peerId: string; from: string; to: string }> = [];
|
||||
return {
|
||||
events,
|
||||
emit(_e, payload): void {
|
||||
events.push(payload);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("HealthMonitor", () => {
|
||||
it("starts unknown peers in alive state", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus);
|
||||
expect(m.getState("ghost")).toBe("alive");
|
||||
});
|
||||
|
||||
it("recordSeen marks new peers alive", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus);
|
||||
m.recordSeen("alice");
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("markStale steps alive → suspect → dead with elapsed time", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus, {
|
||||
suspectAfterMs: 100,
|
||||
deadAfterMs: 500,
|
||||
clock: () => now,
|
||||
});
|
||||
m.recordSeen("alice");
|
||||
now = 200;
|
||||
m.markStale("alice");
|
||||
expect(m.getState("alice")).toBe("suspect");
|
||||
now = 1_000;
|
||||
m.markStale("alice");
|
||||
expect(m.getState("alice")).toBe("dead");
|
||||
expect(bus.events.map((e) => e.to)).toEqual(["suspect", "dead"]);
|
||||
});
|
||||
|
||||
it("markStale is a no-op before threshold elapsed", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus, {
|
||||
suspectAfterMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
m.recordSeen("alice");
|
||||
now = 50;
|
||||
m.markStale("alice");
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("recordSeen restores suspect to alive", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus, {
|
||||
suspectAfterMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
m.recordSeen("alice");
|
||||
now = 200;
|
||||
m.markStale("alice");
|
||||
m.recordSeen("alice");
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("markRecovering only works from dead", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus);
|
||||
expect(() => m.markRecovering("alice")).toThrow(
|
||||
"health.invalid-recover"
|
||||
);
|
||||
});
|
||||
|
||||
it("markRecovered only works from recovering", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus);
|
||||
expect(() => m.markRecovered("alice")).toThrow(
|
||||
"health.invalid-recovered"
|
||||
);
|
||||
});
|
||||
|
||||
it("recovery flow walks dead → recovering → alive", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus, {
|
||||
suspectAfterMs: 50,
|
||||
deadAfterMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
m.recordSeen("alice");
|
||||
now = 60;
|
||||
m.markStale("alice"); // suspect
|
||||
now = 200;
|
||||
m.markStale("alice"); // dead
|
||||
m.markRecovering("alice");
|
||||
expect(m.getState("alice")).toBe("recovering");
|
||||
m.markRecovered("alice");
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("recordSeen restores recovering to alive", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus, {
|
||||
suspectAfterMs: 50,
|
||||
deadAfterMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
m.recordSeen("alice");
|
||||
now = 60;
|
||||
m.markStale("alice");
|
||||
now = 200;
|
||||
m.markStale("alice");
|
||||
m.markRecovering("alice");
|
||||
m.recordSeen("alice");
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("clear drops all state", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus);
|
||||
m.recordSeen("alice");
|
||||
m.clear();
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("uses Date.now-based clock when none injected", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus);
|
||||
m.recordSeen("alice");
|
||||
// markStale shouldn't trigger immediately
|
||||
m.markStale("alice");
|
||||
expect(m.getState("alice")).toBe("alive");
|
||||
});
|
||||
|
||||
it("markStale on a never-seen peer treats now as last-seen", () => {
|
||||
const bus = makeBus();
|
||||
const m = new HealthMonitor(bus, { suspectAfterMs: 100 });
|
||||
m.markStale("ghost");
|
||||
expect(m.getState("ghost")).toBe("alive");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
# HeartbeatManager
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Sends periodic heartbeats to a set of peers, correlates pongs by sequence number, and surfaces missed beats via `findStalePeers`. The manager only handles the wire-level concern; the `HealthMonitor` decides what `alive`/`suspect`/`dead` mean. Decoupling them means the heartbeat cadence and the health policy can evolve independently.
|
||||
|
||||
## Features
|
||||
|
||||
- Configurable `intervalMs` (default 1 s) and `timeoutMs` (default 3 s).
|
||||
- Pluggable `HeartbeatSender` so consumers can route through any transport.
|
||||
- Injectable `setIntervalFn` / `clearIntervalFn` / `clock` for deterministic tests.
|
||||
- `addPeer` / `removePeer` for explicit lifecycle.
|
||||
- `tick()` is exposed publicly so test harnesses can step the loop manually.
|
||||
- Sequence-number correlation prevents stale pongs from masking missed heartbeats.
|
||||
- `markPong(peerId, sequence)` returns the RTT in ms or `undefined` if the sequence didn't match.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { HeartbeatManager } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const heartbeat = new HeartbeatManager({
|
||||
sender: async (peerId, seq) => {
|
||||
await transport.send(peerId, { type: "ping", seq });
|
||||
},
|
||||
});
|
||||
heartbeat.addPeer("alice");
|
||||
heartbeat.start();
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { HeartbeatManager } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
let now = 0;
|
||||
const heartbeat = new HeartbeatManager({
|
||||
sender: async () => {},
|
||||
intervalMs: 100,
|
||||
timeoutMs: 500,
|
||||
clock: () => now,
|
||||
});
|
||||
heartbeat.addPeer("alice");
|
||||
heartbeat.start();
|
||||
await heartbeat.tick();
|
||||
|
||||
now += 1_000;
|
||||
const stale = heartbeat.findStalePeers();
|
||||
// stale === ["alice"]
|
||||
|
||||
heartbeat.stop();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `type HeartbeatSender = (peerId, sequence) => void | Promise<void>`.
|
||||
- `interface HeartbeatManagerConfig { sender, intervalMs?, timeoutMs?, setIntervalFn?, clearIntervalFn?, clock? }`.
|
||||
- `addPeer(peerId)` / `removePeer(peerId)`.
|
||||
- `start()` / `stop()` — idempotent.
|
||||
- `tick(): Promise<void>` — throws `HEARTBEAT_NOT_RUNNING` when called before `start()`.
|
||||
- `markPong(peerId, sequence): number | undefined`.
|
||||
- `findStalePeers(): readonly string[]`.
|
||||
- `getPeers(): readonly string[]`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The pending map is keyed by `peerId` rather than `(peerId, sequence)` because we only ever care about the _latest_ outbound ping per peer — a pong for an older sequence is treated as stale and dropped via the sequence check inside `markPong`. This keeps memory bounded by peer count rather than tick count. `findStalePeers` is read-only and does not mutate the pending map; the consumer (typically `HealthMonitor`) is responsible for transitioning the peer's state and removing it from the heartbeat set.
|
||||
@@ -0,0 +1,118 @@
|
||||
import { NETWORK_FO_CODES, NetworkFoError } from "../error-codes.js";
|
||||
|
||||
/**
|
||||
* Outbound heartbeat send function.
|
||||
*/
|
||||
export type HeartbeatSender = (
|
||||
peerId: string,
|
||||
sequence: number
|
||||
) => void | Promise<void>;
|
||||
|
||||
/**
|
||||
* Configuration for `HeartbeatManager`.
|
||||
*/
|
||||
export interface HeartbeatManagerConfig {
|
||||
readonly sender: HeartbeatSender;
|
||||
readonly intervalMs?: number;
|
||||
readonly timeoutMs?: number;
|
||||
readonly setIntervalFn?: typeof setInterval;
|
||||
readonly clearIntervalFn?: typeof clearInterval;
|
||||
readonly clock?: () => number;
|
||||
}
|
||||
|
||||
interface PendingPing {
|
||||
readonly sentAt: number;
|
||||
readonly sequence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends periodic heartbeats to a peer set, correlates pongs by
|
||||
* sequence number, and surfaces missed beats via `markPong` /
|
||||
* `findStalePeers`. The manager itself does not know about health
|
||||
* states — that's the `HealthMonitor`'s job.
|
||||
*/
|
||||
export class HeartbeatManager {
|
||||
private readonly sender: HeartbeatSender;
|
||||
private readonly intervalMs: number;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly setIntervalFn: typeof setInterval;
|
||||
private readonly clearIntervalFn: typeof clearInterval;
|
||||
private readonly clock: () => number;
|
||||
private readonly peers = new Set<string>();
|
||||
private readonly pending = new Map<string, PendingPing>();
|
||||
private sequence = 0;
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
public constructor(config: HeartbeatManagerConfig) {
|
||||
this.sender = config.sender;
|
||||
this.intervalMs = config.intervalMs ?? 1_000;
|
||||
this.timeoutMs = config.timeoutMs ?? 3_000;
|
||||
this.setIntervalFn = config.setIntervalFn ?? setInterval;
|
||||
this.clearIntervalFn = config.clearIntervalFn ?? clearInterval;
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
}
|
||||
|
||||
public addPeer(peerId: string): void {
|
||||
this.peers.add(peerId);
|
||||
}
|
||||
|
||||
public removePeer(peerId: string): void {
|
||||
this.peers.delete(peerId);
|
||||
this.pending.delete(peerId);
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
if (this.timer !== null) return;
|
||||
this.timer = this.setIntervalFn(() => {
|
||||
void this.tick();
|
||||
}, this.intervalMs);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
if (this.timer === null) return;
|
||||
this.clearIntervalFn(this.timer);
|
||||
this.timer = null;
|
||||
this.pending.clear();
|
||||
}
|
||||
|
||||
public async tick(): Promise<void> {
|
||||
if (this.timer === null) {
|
||||
throw new NetworkFoError(
|
||||
NETWORK_FO_CODES.HEARTBEAT_NOT_RUNNING,
|
||||
"heartbeat.not-running"
|
||||
);
|
||||
}
|
||||
for (const peerId of this.peers) {
|
||||
this.sequence += 1;
|
||||
this.pending.set(peerId, {
|
||||
sentAt: this.clock(),
|
||||
sequence: this.sequence,
|
||||
});
|
||||
await this.sender(peerId, this.sequence);
|
||||
}
|
||||
}
|
||||
|
||||
public markPong(peerId: string, sequence: number): number | undefined {
|
||||
const pending = this.pending.get(peerId);
|
||||
if (pending === undefined || pending.sequence !== sequence) {
|
||||
return undefined;
|
||||
}
|
||||
this.pending.delete(peerId);
|
||||
return Math.max(0, this.clock() - pending.sentAt);
|
||||
}
|
||||
|
||||
public findStalePeers(): readonly string[] {
|
||||
const now = this.clock();
|
||||
const stale: string[] = [];
|
||||
for (const [peerId, pending] of this.pending) {
|
||||
if (now - pending.sentAt > this.timeoutMs) {
|
||||
stale.push(peerId);
|
||||
}
|
||||
}
|
||||
return stale;
|
||||
}
|
||||
|
||||
public getPeers(): readonly string[] {
|
||||
return [...this.peers];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import { HeartbeatManager } from "./heartbeat-manager.js";
|
||||
|
||||
describe("HeartbeatManager", () => {
|
||||
it("addPeer adds peers and removePeer removes them", () => {
|
||||
const m = new HeartbeatManager({ sender: vi.fn() });
|
||||
m.addPeer("alice");
|
||||
m.addPeer("bob");
|
||||
expect(m.getPeers().sort()).toEqual(["alice", "bob"]);
|
||||
m.removePeer("alice");
|
||||
expect(m.getPeers()).toEqual(["bob"]);
|
||||
});
|
||||
|
||||
it("tick sends a heartbeat to every peer", async () => {
|
||||
const send = vi.fn();
|
||||
const m = new HeartbeatManager({ sender: send });
|
||||
m.addPeer("alice");
|
||||
m.addPeer("bob");
|
||||
m.start();
|
||||
await m.tick();
|
||||
expect(send).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("tick before start throws HEARTBEAT_NOT_RUNNING", async () => {
|
||||
const m = new HeartbeatManager({ sender: vi.fn() });
|
||||
m.addPeer("alice");
|
||||
await expect(m.tick()).rejects.toThrow("heartbeat.not-running");
|
||||
});
|
||||
|
||||
it("markPong returns RTT for matching sequence and undefined otherwise", async () => {
|
||||
let now = 100;
|
||||
const m = new HeartbeatManager({
|
||||
sender: vi.fn(),
|
||||
clock: () => now,
|
||||
});
|
||||
m.addPeer("alice");
|
||||
m.start();
|
||||
await m.tick();
|
||||
now = 250;
|
||||
const rtt = m.markPong("alice", 1);
|
||||
expect(rtt).toBe(150);
|
||||
// stale sequence on a peer with no pending ping
|
||||
expect(m.markPong("alice", 999)).toBeUndefined();
|
||||
// unknown peer
|
||||
expect(m.markPong("ghost", 1)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("findStalePeers reports peers past timeoutMs", async () => {
|
||||
let now = 0;
|
||||
const m = new HeartbeatManager({
|
||||
sender: vi.fn(),
|
||||
timeoutMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
m.addPeer("alice");
|
||||
m.start();
|
||||
await m.tick();
|
||||
now = 50;
|
||||
expect(m.findStalePeers()).toEqual([]);
|
||||
now = 200;
|
||||
expect(m.findStalePeers()).toEqual(["alice"]);
|
||||
});
|
||||
|
||||
it("clamps negative RTT to 0 (clock skew)", async () => {
|
||||
let now = 100;
|
||||
const m = new HeartbeatManager({
|
||||
sender: vi.fn(),
|
||||
clock: () => now,
|
||||
});
|
||||
m.addPeer("alice");
|
||||
m.start();
|
||||
await m.tick();
|
||||
now = 50; // clock went backwards
|
||||
expect(m.markPong("alice", 1)).toBe(0);
|
||||
});
|
||||
|
||||
it("removePeer drops pending ping", async () => {
|
||||
const m = new HeartbeatManager({ sender: vi.fn() });
|
||||
m.addPeer("alice");
|
||||
m.start();
|
||||
await m.tick();
|
||||
m.removePeer("alice");
|
||||
expect(m.findStalePeers()).toEqual([]);
|
||||
});
|
||||
|
||||
it("start/stop are idempotent and use injected timer fns", () => {
|
||||
let cb: (() => void) | null = null;
|
||||
const setIntervalFn = ((fn: () => void) => {
|
||||
cb = fn;
|
||||
return 1 as never;
|
||||
}) as never;
|
||||
const clearIntervalFn = (() => {}) as never;
|
||||
const m = new HeartbeatManager({
|
||||
sender: vi.fn(),
|
||||
setIntervalFn,
|
||||
clearIntervalFn,
|
||||
});
|
||||
m.start();
|
||||
m.start();
|
||||
m.stop();
|
||||
m.stop();
|
||||
expect(cb).not.toBeNull();
|
||||
});
|
||||
|
||||
it("invokes the registered timer callback to drive tick", async () => {
|
||||
let cb: (() => void) | null = null;
|
||||
const setIntervalFn = ((fn: () => void) => {
|
||||
cb = fn;
|
||||
return 1 as never;
|
||||
}) as never;
|
||||
const clearIntervalFn = (() => {}) as never;
|
||||
const send = vi.fn();
|
||||
const m = new HeartbeatManager({
|
||||
sender: send,
|
||||
setIntervalFn,
|
||||
clearIntervalFn,
|
||||
});
|
||||
m.addPeer("alice");
|
||||
m.start();
|
||||
cb?.();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(send).toHaveBeenCalled();
|
||||
m.stop();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
// Error codes (side-effect registration)
|
||||
import "./error-codes.js";
|
||||
|
||||
// Errors
|
||||
export {
|
||||
NETWORK_FO_CODES,
|
||||
NetworkFoError,
|
||||
type NetworkFoErrorCode,
|
||||
} from "./error-codes.js";
|
||||
// Failover coordinator
|
||||
export {
|
||||
FailoverCoordinator,
|
||||
type FailoverCoordinatorConfig,
|
||||
type HealthBusListener,
|
||||
} from "./failover/failover-coordinator.js";
|
||||
// Health monitor
|
||||
export {
|
||||
type HealthBus,
|
||||
HealthMonitor,
|
||||
type HealthMonitorConfig,
|
||||
type HealthState,
|
||||
} from "./health/health-monitor.js";
|
||||
// Heartbeat manager
|
||||
export {
|
||||
HeartbeatManager,
|
||||
type HeartbeatManagerConfig,
|
||||
type HeartbeatSender,
|
||||
} from "./heartbeat/heartbeat-manager.js";
|
||||
// Plugin
|
||||
export {
|
||||
FailoverPlugin,
|
||||
type FailoverPluginConfig,
|
||||
} from "./plugin/failover-plugin.js";
|
||||
// Promoter
|
||||
export {
|
||||
type PromoterBus,
|
||||
PromoterManager,
|
||||
type PromoterManagerConfig,
|
||||
} from "./promoter/promoter-manager.js";
|
||||
@@ -0,0 +1,70 @@
|
||||
# FailoverPlugin
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`NetworkPlugin` that owns the failover stack: heartbeat manager, health monitor, promoter, and coordinator. Each sub-component is exposed as a public property so callers can inspect or extend behaviour without going through the plugin abstraction. The plugin only wires the components together; per-component behaviour lives in the dedicated module docs.
|
||||
|
||||
## Features
|
||||
|
||||
- Lifecycle: `install(node)` builds health + promoter + coordinator against the node's bus; `start()` starts heartbeat + coordinator subscriptions; `stop()` reverses both; `dispose()` clears state.
|
||||
- Heartbeat is constructed eagerly so callers can `addPeer` before `start()`.
|
||||
- Health and promoter are constructed lazily inside `install` because they need the node's bus.
|
||||
- Public `heartbeat` / `health` / `promoter` / `coordinator` properties for observability and test harnesses.
|
||||
- Configurable via `FailoverPluginConfig` — heartbeat is required, health and promoter inherit defaults.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { FailoverPlugin } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const plugin = new FailoverPlugin({
|
||||
heartbeat: {
|
||||
sender: async (peerId, seq) =>
|
||||
transport.send(peerId, { type: "ping", seq }),
|
||||
},
|
||||
});
|
||||
|
||||
node.use(plugin);
|
||||
await node.start();
|
||||
plugin.heartbeat.addPeer("alice");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { FailoverPlugin } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const plugin = new FailoverPlugin({
|
||||
heartbeat: {
|
||||
sender: async (peerId, seq) =>
|
||||
transport.send(peerId, { type: "ping", seq }),
|
||||
intervalMs: 500,
|
||||
timeoutMs: 2_000,
|
||||
},
|
||||
health: {
|
||||
suspectAfterMs: 1_000,
|
||||
deadAfterMs: 4_000,
|
||||
},
|
||||
promoter: {
|
||||
debounceMs: 500,
|
||||
},
|
||||
});
|
||||
|
||||
node.use(plugin);
|
||||
await node.start();
|
||||
setInterval(() => plugin.coordinator?.sweep(), 1_000);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface FailoverPluginConfig { heartbeat, health?, promoter? }`.
|
||||
- `name: "failover"`.
|
||||
- `install(node) / start() / stop() / dispose()` — standard `NetworkPlugin` lifecycle.
|
||||
- Public properties: `heartbeat: HeartbeatManager`, `health: HealthMonitor | null`, `promoter: PromoterManager | null`, `coordinator: FailoverCoordinator | null`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
`health`, `promoter`, and `coordinator` are typed nullable because they only exist after `install` runs — callers that need to access them before the plugin is installed must use the always-present `heartbeat` reference. `dispose` clears state but does not re-construct components, so a re-installed plugin instance must be a fresh `FailoverPlugin`. The bus type assertion (`as unknown as HealthBusListener & ...`) is necessary because `NetworkBus` is generically typed and the failover-specific topics are not in its compile-time map.
|
||||
@@ -0,0 +1,54 @@
|
||||
import { NetworkNode } from "@suaveplan/network";
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import { FailoverPlugin } from "./failover-plugin.js";
|
||||
|
||||
describe("FailoverPlugin", () => {
|
||||
it("exposes the canonical name", () => {
|
||||
const p = new FailoverPlugin({ heartbeat: { sender: vi.fn() } });
|
||||
expect(p.name).toBe("failover");
|
||||
});
|
||||
|
||||
it("install builds health, promoter, and coordinator", () => {
|
||||
const node = new NetworkNode();
|
||||
const plugin = new FailoverPlugin({
|
||||
heartbeat: { sender: vi.fn() },
|
||||
});
|
||||
node.use(plugin);
|
||||
expect(plugin.health).not.toBeNull();
|
||||
expect(plugin.promoter).not.toBeNull();
|
||||
expect(plugin.coordinator).not.toBeNull();
|
||||
});
|
||||
|
||||
it("start / stop runs through the node lifecycle without crashing", async () => {
|
||||
const node = new NetworkNode();
|
||||
const plugin = new FailoverPlugin({
|
||||
heartbeat: { sender: vi.fn() },
|
||||
});
|
||||
node.use(plugin);
|
||||
await node.start();
|
||||
await node.stop();
|
||||
});
|
||||
|
||||
it("dispose clears state and nulls out lazy components", async () => {
|
||||
const node = new NetworkNode();
|
||||
const plugin = new FailoverPlugin({
|
||||
heartbeat: { sender: vi.fn() },
|
||||
});
|
||||
node.use(plugin);
|
||||
plugin.dispose();
|
||||
expect(plugin.health).toBeNull();
|
||||
expect(plugin.promoter).toBeNull();
|
||||
expect(plugin.coordinator).toBeNull();
|
||||
});
|
||||
|
||||
it("supports custom health and promoter config", () => {
|
||||
const node = new NetworkNode();
|
||||
const plugin = new FailoverPlugin({
|
||||
heartbeat: { sender: vi.fn(), intervalMs: 100 },
|
||||
health: { suspectAfterMs: 100, deadAfterMs: 500 },
|
||||
promoter: { debounceMs: 250 },
|
||||
});
|
||||
node.use(plugin);
|
||||
expect(plugin.health).not.toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { NetworkNode, NetworkPlugin } from "@suaveplan/network";
|
||||
import {
|
||||
FailoverCoordinator,
|
||||
type HealthBusListener,
|
||||
} from "../failover/failover-coordinator.js";
|
||||
import {
|
||||
HealthMonitor,
|
||||
type HealthMonitorConfig,
|
||||
} from "../health/health-monitor.js";
|
||||
import {
|
||||
HeartbeatManager,
|
||||
type HeartbeatManagerConfig,
|
||||
} from "../heartbeat/heartbeat-manager.js";
|
||||
import {
|
||||
PromoterManager,
|
||||
type PromoterManagerConfig,
|
||||
} from "../promoter/promoter-manager.js";
|
||||
|
||||
/**
|
||||
* Configuration for `FailoverPlugin`.
|
||||
*/
|
||||
export interface FailoverPluginConfig {
|
||||
readonly heartbeat: HeartbeatManagerConfig;
|
||||
readonly health?: HealthMonitorConfig;
|
||||
readonly promoter?: Omit<PromoterManagerConfig, "bus">;
|
||||
}
|
||||
|
||||
/**
|
||||
* `NetworkPlugin` that owns the failover stack. Each sub-component is
|
||||
* exposed as a public property so callers can inspect or extend the
|
||||
* behaviour without going through the plugin abstraction.
|
||||
*/
|
||||
export class FailoverPlugin implements NetworkPlugin {
|
||||
public readonly name = "failover";
|
||||
public heartbeat: HeartbeatManager;
|
||||
public health: HealthMonitor | null = null;
|
||||
public promoter: PromoterManager | null = null;
|
||||
public coordinator: FailoverCoordinator | null = null;
|
||||
private readonly heartbeatConfig: HeartbeatManagerConfig;
|
||||
private readonly healthConfig: HealthMonitorConfig;
|
||||
private readonly promoterConfig: Omit<PromoterManagerConfig, "bus">;
|
||||
|
||||
public constructor(config: FailoverPluginConfig) {
|
||||
this.heartbeatConfig = config.heartbeat;
|
||||
this.healthConfig = config.health ?? {};
|
||||
this.promoterConfig = config.promoter ?? {};
|
||||
this.heartbeat = new HeartbeatManager(this.heartbeatConfig);
|
||||
}
|
||||
|
||||
public install(node: NetworkNode): void {
|
||||
const bus = node.networkBus as unknown as HealthBusListener &
|
||||
ConstructorParameters<typeof HealthMonitor>[0];
|
||||
this.health = new HealthMonitor(bus, this.healthConfig);
|
||||
this.promoter = new PromoterManager({
|
||||
...this.promoterConfig,
|
||||
bus: bus as never,
|
||||
});
|
||||
this.coordinator = new FailoverCoordinator({
|
||||
heartbeat: this.heartbeat,
|
||||
health: this.health,
|
||||
promoter: this.promoter,
|
||||
bus,
|
||||
});
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.coordinator?.start();
|
||||
this.heartbeat.start();
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this.heartbeat.stop();
|
||||
this.coordinator?.stop();
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this.health?.clear();
|
||||
this.promoter?.reset();
|
||||
this.health = null;
|
||||
this.promoter = null;
|
||||
this.coordinator = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
# PromoterManager
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Fires `failover:emergency-election` bus events when a peer transitions to dead, while debouncing duplicate triggers within a configurable window. Decouples failover from the governance plugin — communication is bus-only, never a direct import — so either side can be swapped without touching the other.
|
||||
|
||||
## Features
|
||||
|
||||
- `onPeerDead(peerId, reason): boolean` — returns `true` if the trigger fired, `false` if it was suppressed by debounce.
|
||||
- Configurable `debounceMs` (default 1 s).
|
||||
- Pluggable `clock` for deterministic tests.
|
||||
- `reset(peerId?)` clears one peer's debounce window or all of them.
|
||||
- Per-peer last-fired timestamps so peers can independently re-trigger after their windows elapse.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { PromoterManager } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
const promoter = new PromoterManager({ bus });
|
||||
const fired = promoter.onPeerDead("alice", "heartbeat-timeout");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { PromoterManager } from "@suaveplan/network-plugin-failover";
|
||||
|
||||
let now = 0;
|
||||
const promoter = new PromoterManager({
|
||||
bus,
|
||||
debounceMs: 500,
|
||||
clock: () => now,
|
||||
});
|
||||
|
||||
bus.on("health:state-change", ({ peerId, to }) => {
|
||||
if (to === "dead") {
|
||||
promoter.onPeerDead(peerId, "health:state-change");
|
||||
}
|
||||
});
|
||||
|
||||
// rapid duplicate transitions are de-duplicated
|
||||
promoter.onPeerDead("alice", "first");
|
||||
now += 100;
|
||||
promoter.onPeerDead("alice", "second"); // suppressed by debounce
|
||||
now += 600;
|
||||
promoter.onPeerDead("alice", "third"); // window elapsed → fires again
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `interface PromoterBus { emit("failover:emergency-election", { deadPeerId, reason }) }`.
|
||||
- `interface PromoterManagerConfig { bus, debounceMs?, clock? }`.
|
||||
- `onPeerDead(peerId, reason): boolean`.
|
||||
- `reset(peerId?)`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The debounce is per-peer rather than global because two peers dying at once is a normal cascading-failure scenario that operators want to see two events for, not one. Suppressing the second event by accident would mask compound outages. `reset` is exposed mostly for tests; in production, the lastFiredAt map is bounded by peer count and never grows without bound. The `onPeerDead` boolean return is intentional — callers logging promotion attempts can distinguish "fired" from "suppressed" without parsing event-bus state. The default debounce of 1 s is a reasonable middle-ground; aggressive meshes that need faster re-triggering can reduce it, while slow networks that want fewer false positives can stretch it.
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Bus shape used by `PromoterManager` to fire emergency elections.
|
||||
*/
|
||||
export interface PromoterBus {
|
||||
emit(
|
||||
event: "failover:emergency-election",
|
||||
payload: { deadPeerId: string; reason: string }
|
||||
): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for `PromoterManager`.
|
||||
*/
|
||||
export interface PromoterManagerConfig {
|
||||
readonly bus: PromoterBus;
|
||||
readonly debounceMs?: number;
|
||||
readonly clock?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Listens for "peer dead" signals and fires `failover:emergency-election`
|
||||
* events while debouncing duplicate triggers within the configured
|
||||
* window. Decouples the failover FSM from the governance plugin —
|
||||
* communication happens entirely on the bus.
|
||||
*/
|
||||
export class PromoterManager {
|
||||
private readonly bus: PromoterBus;
|
||||
private readonly debounceMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly lastFiredAt = new Map<string, number>();
|
||||
|
||||
public constructor(config: PromoterManagerConfig) {
|
||||
this.bus = config.bus;
|
||||
this.debounceMs = config.debounceMs ?? 1_000;
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
}
|
||||
|
||||
public onPeerDead(peerId: string, reason: string): boolean {
|
||||
const last = this.lastFiredAt.get(peerId);
|
||||
const now = this.clock();
|
||||
if (last !== undefined && now - last < this.debounceMs) {
|
||||
return false;
|
||||
}
|
||||
this.lastFiredAt.set(peerId, now);
|
||||
this.bus.emit("failover:emergency-election", {
|
||||
deadPeerId: peerId,
|
||||
reason,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
public reset(peerId?: string): void {
|
||||
if (peerId === undefined) {
|
||||
this.lastFiredAt.clear();
|
||||
return;
|
||||
}
|
||||
this.lastFiredAt.delete(peerId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { type PromoterBus, PromoterManager } from "./promoter-manager.js";
|
||||
|
||||
function makeBus(): PromoterBus & {
|
||||
events: Array<{ deadPeerId: string; reason: string }>;
|
||||
} {
|
||||
const events: Array<{ deadPeerId: string; reason: string }> = [];
|
||||
return {
|
||||
events,
|
||||
emit(_e, payload): void {
|
||||
events.push(payload);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("PromoterManager", () => {
|
||||
it("fires emergency election on first call", () => {
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({ bus });
|
||||
const fired = p.onPeerDead("alice", "test");
|
||||
expect(fired).toBe(true);
|
||||
expect(bus.events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("debounces duplicate triggers within the window", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({
|
||||
bus,
|
||||
debounceMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
p.onPeerDead("alice", "first");
|
||||
now = 50;
|
||||
const second = p.onPeerDead("alice", "second");
|
||||
expect(second).toBe(false);
|
||||
expect(bus.events).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("re-fires after the debounce window elapses", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({
|
||||
bus,
|
||||
debounceMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
p.onPeerDead("alice", "first");
|
||||
now = 200;
|
||||
const second = p.onPeerDead("alice", "second");
|
||||
expect(second).toBe(true);
|
||||
expect(bus.events).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("debounces per peer (not globally)", () => {
|
||||
let now = 0;
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({
|
||||
bus,
|
||||
debounceMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
p.onPeerDead("alice", "a");
|
||||
now = 50;
|
||||
const bobFired = p.onPeerDead("bob", "b");
|
||||
expect(bobFired).toBe(true);
|
||||
expect(bus.events).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("reset(peerId) drops one peer's debounce window", () => {
|
||||
const now = 0;
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({
|
||||
bus,
|
||||
debounceMs: 1_000_000,
|
||||
clock: () => now,
|
||||
});
|
||||
p.onPeerDead("alice", "first");
|
||||
p.reset("alice");
|
||||
const second = p.onPeerDead("alice", "second");
|
||||
expect(second).toBe(true);
|
||||
});
|
||||
|
||||
it("reset() drops every peer's window", () => {
|
||||
const now = 0;
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({
|
||||
bus,
|
||||
debounceMs: 1_000_000,
|
||||
clock: () => now,
|
||||
});
|
||||
p.onPeerDead("alice", "a");
|
||||
p.onPeerDead("bob", "b");
|
||||
p.reset();
|
||||
expect(p.onPeerDead("alice", "again")).toBe(true);
|
||||
expect(p.onPeerDead("bob", "again")).toBe(true);
|
||||
});
|
||||
|
||||
it("uses Date.now-based clock when none injected", () => {
|
||||
const bus = makeBus();
|
||||
const p = new PromoterManager({ bus });
|
||||
expect(p.onPeerDead("alice", "a")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("network-plugin-failover dep-graph", () => {
|
||||
it("loads index.js (acyclic static imports)", async () => {
|
||||
const mod = await import("../src/index.js");
|
||||
expect(mod.HeartbeatManager).toBeDefined();
|
||||
expect(mod.HealthMonitor).toBeDefined();
|
||||
expect(mod.PromoterManager).toBeDefined();
|
||||
expect(mod.FailoverCoordinator).toBeDefined();
|
||||
expect(mod.FailoverPlugin).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
// Ensure `globalThis.crypto` is available in bare-Node vitest runs.
|
||||
import { webcrypto } from "node:crypto";
|
||||
|
||||
type WritableGlobal = { crypto?: Crypto };
|
||||
const g = globalThis as WritableGlobal;
|
||||
if (g.crypto === undefined) {
|
||||
g.crypto = webcrypto as unknown as Crypto;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"references": [{ "path": "./tsconfig.lib.json" }]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir:
|
||||
"../../../node_modules/.vite/packages/network-plugin-failover",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/network",
|
||||
"@suaveplan/types",
|
||||
"tslib",
|
||||
"zod",
|
||||
"@opentelemetry/api",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: [
|
||||
"src/__benchmarks__/**",
|
||||
"benchmarks/**",
|
||||
"e2e/**",
|
||||
"dist/**",
|
||||
"node_modules/**",
|
||||
],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/**/*.schemas.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 5000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user