Initial Commit

packages/api-stack/api/api-plugin-database/src/retry/retry.md

@@ -0,0 +1,85 @@
+# `retry`
+
+<!-- begin:auto-docs -->
+_(no submodules)_
+<!-- end:auto-docs -->
+
+## Purpose
+
+Provides `withRetry`, a small exponential-backoff helper used by the connection pool, the migration runner, and any caller-supplied wrapping of adapter calls. The helper is intentionally minimal: it owns the loop, the backoff calculation, the optional jitter, and the abort plumbing. Higher-level concerns (circuit breakers, dead-letter routing, OTel spans) are layered on top by the caller — keeping retry stateless makes it composable.
+
+Backoff follows the canonical formula `min(baseBackoffMs * 2^(attempt-1), maxBackoffMs)`. Jittered mode multiplies the result by a `[0, 1)` random factor so concurrent callers don't synchronise their retry windows. Both the RNG and the `setTimeout` setter are injectable so tests run deterministically without time travel.
+
+## Features
+
+- Exponential backoff capped at `maxBackoffMs`.
+- Optional full-jitter randomisation through an injectable `Rng`.
+- `isRetryable(err, attempt)` predicate for bail-out on permanent failures.
+- `AbortSignal` propagation cancels pending sleeps and the loop.
+- Pure async — no global state, no side effects beyond the optional sleep.
+
+## Usage: Basic
+
+```ts
+import { withRetry } from "@suaveplan/api-plugin-database/retry/retry";
+
+const result = await withRetry(() => db.query("SELECT 1"), {
+    maxAttempts: 3,
+    baseBackoffMs: 50,
+    maxBackoffMs: 1_000,
+    jitter: true,
+});
+```
+
+## Usage: Advanced
+
+```ts
+import { withRetry } from "@suaveplan/api-plugin-database";
+
+const controller = new AbortController();
+setTimeout(() => controller.abort(), 30_000);
+
+const data = await withRetry(
+    async (attempt) => {
+        const adapter = await pool.acquire();
+        try {
+            return await adapter.query("SELECT * FROM users WHERE active");
+        } finally {
+            pool.release(adapter);
+        }
+    },
+    {
+        maxAttempts: 5,
+        baseBackoffMs: 100,
+        maxBackoffMs: 5_000,
+        jitter: true,
+        signal: controller.signal,
+        isRetryable: (err) =>
+            err instanceof Error && /timeout|reset/i.test(err.message),
+    }
+);
+```
+
+## API Reference
+
+### `withRetry<T>(fn, options): Promise<T>`
+
+Invokes `fn(attempt)` (1-indexed) until success or budget exhaustion. Returns the resolved value or rethrows the final error.
+
+### `interface RetryOptions`
+
+- `maxAttempts` (number, >=1) — total attempt budget.
+- `baseBackoffMs` (number, >=1) — base interval.
+- `maxBackoffMs` (number, >=1) — interval ceiling.
+- `jitter` (boolean) — multiply by `random()` for full jitter.
+- `isRetryable?` (`(err, attempt) => boolean`) — bail predicate.
+- `rng?` (`Rng`) — injectable random source.
+- `signal?` (`AbortSignal`) — cancels in-flight sleep + loop.
+- `schedule?` (`(cb, ms) => void`) — injectable `setTimeout` for tests.
+
+## Implementation Notes
+
+- Sleeps are interrupted promptly by `AbortSignal`; the inner promise rejects with the signal's `reason` (when an `Error`) or a generic `Error("withRetry aborted")`.
+- Aborts at the top of the loop (before `fn(attempt)`) short-circuit the retry budget immediately.
+- The helper does not catch or rewrap thrown values — callers see the underlying error. This keeps the helper independent of the package's error registry, which composes downstream.
+- Helper functions are intentionally inline-sized; the file stays under the 300-line hard limit.

packages/api-stack/api/api-plugin-database/src/retry/retry.test.ts

@@ -0,0 +1,341 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { withRetry } from "./retry.js";
+
+const immediate = (cb: () => void): void => {
+    cb();
+};
+
+describe("withRetry", () => {
+    it("first-attempt success returns the value", {
+        timeout: 5000,
+    }, async () => {
+        const result = await withRetry(async () => 42, {
+            maxAttempts: 3,
+            baseBackoffMs: 1,
+            maxBackoffMs: 10,
+            jitter: false,
+            schedule: immediate,
+        });
+        expect(result).toBe(42);
+    });
+
+    it("retries on transient failure", { timeout: 5000 }, async () => {
+        let calls = 0;
+        const result = await withRetry(
+            async () => {
+                calls += 1;
+                if (calls < 3) throw new Error("transient");
+                return "ok";
+            },
+            {
+                maxAttempts: 5,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+                schedule: immediate,
+            }
+        );
+        expect(result).toBe("ok");
+        expect(calls).toBe(3);
+    });
+
+    it("throws after maxAttempts exhausted", { timeout: 5000 }, async () => {
+        let calls = 0;
+        await expect(
+            withRetry(
+                async () => {
+                    calls += 1;
+                    throw new Error(`fail-${calls}`);
+                },
+                {
+                    maxAttempts: 3,
+                    baseBackoffMs: 1,
+                    maxBackoffMs: 10,
+                    jitter: false,
+                    schedule: immediate,
+                }
+            )
+        ).rejects.toThrow("fail-3");
+        expect(calls).toBe(3);
+    });
+
+    it("isRetryable false bails out early", { timeout: 5000 }, async () => {
+        let calls = 0;
+        await expect(
+            withRetry(
+                async () => {
+                    calls += 1;
+                    throw new Error("permanent");
+                },
+                {
+                    maxAttempts: 5,
+                    baseBackoffMs: 1,
+                    maxBackoffMs: 10,
+                    jitter: false,
+                    isRetryable: () => false,
+                    schedule: immediate,
+                }
+            )
+        ).rejects.toThrow("permanent");
+        expect(calls).toBe(1);
+    });
+
+    it("jitter mode multiplies by injected RNG", {
+        timeout: 5000,
+    }, async () => {
+        const recorded: number[] = [];
+        const schedule = (cb: () => void, ms: number): void => {
+            recorded.push(ms);
+            cb();
+        };
+        let calls = 0;
+        await withRetry(
+            async () => {
+                calls += 1;
+                if (calls < 3) throw new Error("transient");
+                return "ok";
+            },
+            {
+                maxAttempts: 5,
+                baseBackoffMs: 100,
+                maxBackoffMs: 10_000,
+                jitter: true,
+                rng: () => 0.5,
+                schedule,
+            }
+        );
+        // Attempt 1 backoff = 0.5 * 100 = 50; Attempt 2 = 0.5 * 200 = 100.
+        expect(recorded).toEqual([50, 100]);
+    });
+
+    it("respects an AbortSignal that fires between attempts", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        let calls = 0;
+        const schedule = (cb: () => void): void => {
+            controller.abort();
+            cb();
+        };
+        await expect(
+            withRetry(
+                async () => {
+                    calls += 1;
+                    throw new Error("transient");
+                },
+                {
+                    maxAttempts: 5,
+                    baseBackoffMs: 1,
+                    maxBackoffMs: 10,
+                    jitter: false,
+                    signal: controller.signal,
+                    schedule,
+                }
+            )
+        ).rejects.toThrow(/aborted/);
+        expect(calls).toBe(1);
+    });
+
+    it("rejects immediately when signal is pre-aborted", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        controller.abort();
+        await expect(
+            withRetry(async () => "ok", {
+                maxAttempts: 3,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+                signal: controller.signal,
+                schedule: immediate,
+            })
+        ).rejects.toThrow(/aborted/);
+    });
+
+    it("propagates signal.reason when it is an Error", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        const reason = new Error("custom-abort-reason");
+        controller.abort(reason);
+        await expect(
+            withRetry(async () => "ok", {
+                maxAttempts: 3,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+                signal: controller.signal,
+                schedule: immediate,
+            })
+        ).rejects.toBe(reason);
+    });
+
+    it("no jitter uses the raw exp interval", { timeout: 5000 }, async () => {
+        const recorded: number[] = [];
+        const schedule = (cb: () => void, ms: number): void => {
+            recorded.push(ms);
+            cb();
+        };
+        let calls = 0;
+        await withRetry(
+            async () => {
+                calls += 1;
+                if (calls < 3) throw new Error("transient");
+                return "ok";
+            },
+            {
+                maxAttempts: 5,
+                baseBackoffMs: 100,
+                maxBackoffMs: 10_000,
+                jitter: false,
+                schedule,
+            }
+        );
+        expect(recorded).toEqual([100, 200]);
+    });
+
+    it("default rng path executes when rng is not supplied", {
+        timeout: 5000,
+    }, async () => {
+        let calls = 0;
+        const recorded: number[] = [];
+        const schedule = (cb: () => void, ms: number): void => {
+            recorded.push(ms);
+            cb();
+        };
+        await withRetry(
+            async () => {
+                calls += 1;
+                if (calls < 2) throw new Error("transient");
+                return "ok";
+            },
+            {
+                maxAttempts: 3,
+                baseBackoffMs: 50,
+                maxBackoffMs: 10_000,
+                jitter: true,
+                schedule,
+            }
+        );
+        expect(recorded.length).toBe(1);
+        expect(recorded[0]).toBeGreaterThanOrEqual(0);
+        expect(recorded[0]).toBeLessThan(50);
+    });
+
+    it("abort during sleep with non-Error reason throws generic", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        const schedule = (cb: () => void): void => {
+            controller.abort("non-error-reason");
+            // never call cb so the sleep is interrupted only by abort
+            void cb;
+        };
+        await expect(
+            withRetry(
+                async () => {
+                    throw new Error("transient");
+                },
+                {
+                    maxAttempts: 3,
+                    baseBackoffMs: 1,
+                    maxBackoffMs: 10,
+                    jitter: false,
+                    signal: controller.signal,
+                    schedule,
+                }
+            )
+        ).rejects.toThrow("withRetry aborted");
+    });
+
+    it("uses globalThis.setTimeout when no schedule is supplied", {
+        timeout: 5000,
+    }, async () => {
+        let calls = 0;
+        const result = await withRetry(
+            async () => {
+                calls += 1;
+                if (calls < 2) throw new Error("transient");
+                return "ok";
+            },
+            {
+                maxAttempts: 3,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+            }
+        );
+        expect(result).toBe("ok");
+        expect(calls).toBe(2);
+    });
+
+    it("default top-of-loop abort emits 'withRetry aborted' for non-Error reason", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        controller.abort("non-error");
+        await expect(
+            withRetry(async () => "ok", {
+                maxAttempts: 3,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+                signal: controller.signal,
+                schedule: immediate,
+            })
+        ).rejects.toThrow("withRetry aborted");
+    });
+
+    it("sleep short-circuits when signal aborts between attempts", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        let calls = 0;
+        const fn = async (): Promise<string> => {
+            calls += 1;
+            if (calls === 1) {
+                controller.abort(new Error("between attempts"));
+                throw new Error("transient");
+            }
+            return "ok";
+        };
+        await expect(
+            withRetry(fn, {
+                maxAttempts: 3,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+                signal: controller.signal,
+                schedule: immediate,
+            })
+        ).rejects.toThrow("between attempts");
+        expect(calls).toBe(1);
+    });
+
+    it("sleep short-circuit handles non-Error abort reason", {
+        timeout: 5000,
+    }, async () => {
+        const controller = new AbortController();
+        let calls = 0;
+        const fn = async (): Promise<string> => {
+            calls += 1;
+            if (calls === 1) {
+                controller.abort("non-error");
+                throw new Error("transient");
+            }
+            return "ok";
+        };
+        await expect(
+            withRetry(fn, {
+                maxAttempts: 3,
+                baseBackoffMs: 1,
+                maxBackoffMs: 10,
+                jitter: false,
+                signal: controller.signal,
+                schedule: immediate,
+            })
+        ).rejects.toThrow("withRetry aborted");
+        expect(calls).toBe(1);
+    });
+});

packages/api-stack/api/api-plugin-database/src/retry/retry.ts

@@ -0,0 +1,103 @@
+/**
+ * Construction options for {@link withRetry}.
+ */
+export interface RetryOptions {
+    /** Maximum total attempts including the first try. Must be >= 1. */
+    readonly maxAttempts: number;
+    /** Base backoff in ms (also the floor when attempt=1). */
+    readonly baseBackoffMs: number;
+    /** Hard ceiling on a single backoff interval. */
+    readonly maxBackoffMs: number;
+    /** When true, the delay is `random() * exp` (full jitter). */
+    readonly jitter: boolean;
+    /**
+     * Optional predicate. If supplied and returns `false`, `withRetry`
+     * stops retrying and rethrows the current error.
+     */
+    readonly isRetryable?: (error: unknown, attempt: number) => boolean;
+    /** Optional RNG callable for deterministic jitter in tests. Defaults to `Math.random`. */
+    readonly rng?: () => number;
+    /** Optional abort signal — cancels pending sleeps and stops further retries. */
+    readonly signal?: AbortSignal;
+    /**
+     * Injectable timer setter for tests. Defaults to `globalThis.setTimeout`.
+     */
+    readonly schedule?: (cb: () => void, ms: number) => void;
+}
+
+/**
+ * Run `fn` with exponential-backoff retry. Returns the resolved value on
+ * the first attempt that succeeds. Rethrows the final error if every
+ * attempt fails, or the current error if `isRetryable` returns false.
+ *
+ * Backoff for attempt `n` (1-indexed) is
+ * `min(baseBackoffMs * 2^(n-1), maxBackoffMs)`, optionally multiplied by
+ * a `[0, 1)` random factor when `jitter` is true.
+ */
+export async function withRetry<T>(
+    fn: (attempt: number) => Promise<T>,
+    options: RetryOptions
+): Promise<T> {
+    const {
+        maxAttempts,
+        baseBackoffMs,
+        maxBackoffMs,
+        jitter,
+        isRetryable,
+        rng,
+        signal,
+        schedule,
+    } = options;
+    let lastError: unknown;
+    for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+        if (signal?.aborted === true) {
+            throw signal.reason instanceof Error
+                ? signal.reason
+                : new Error("withRetry aborted");
+        }
+        try {
+            return await fn(attempt);
+        } catch (err) {
+            lastError = err;
+            if (attempt === maxAttempts) break;
+            if (isRetryable !== undefined && !isRetryable(err, attempt)) break;
+            const exp = Math.min(
+                maxBackoffMs,
+                baseBackoffMs * 2 ** (attempt - 1)
+            );
+            const random = (rng ?? Math.random)();
+            const delay = jitter ? random * exp : exp;
+            await sleep(delay, schedule, signal);
+        }
+    }
+    throw lastError;
+}
+
+function sleep(
+    ms: number,
+    schedule: ((cb: () => void, ms: number) => void) | undefined,
+    signal: AbortSignal | undefined
+): Promise<void> {
+    return new Promise<void>((resolve, reject) => {
+        const onAbort = (): void => {
+            reject(
+                signal?.reason instanceof Error
+                    ? signal.reason
+                    : new Error("withRetry aborted")
+            );
+        };
+        if (signal?.aborted === true) {
+            onAbort();
+            return;
+        }
+        const tick = (): void => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        };
+        if (signal !== undefined) {
+            signal.addEventListener("abort", onAbort, { once: true });
+        }
+        const setter = schedule ?? globalThis.setTimeout;
+        setter(tick, ms);
+    });
+}