Initial Commit
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
# @suaveplan/network-plugin-content-delivery - Mesh-as-CDN core: ChunkIndex (gossip-propagated holder map), ChunkRequestRouter (latency-aware routing with server fallback), RedundancyManager (N-copy enforcement + re-replication on peer churn), IntegrityVerifier (hash-agnostic verification + sender blacklisting), and ContentDeliveryPlugin facade. Drive-store, packer, prefetch, and bandwidth-throttle integrations live in follow-up changes.
|
||||
|
||||
## Summary
|
||||
|
||||
Mesh-as-CDN core for the Suaveplan network. `ChunkIndex` maintains a distributed holder map keyed by chunk hash with gossip-style merge over `ChunkAdvertisement` records (peer + timestamp ordering); `ChunkRequestRouter` resolves a chunk request to the lowest-latency holder via injected `LatencyOracle`, falling back to the configured server when no mesh peer holds the chunk; `RedundancyManager` enforces the per-chunk `redundancyTarget` (configurable preset map for `critical` / `shared` / `user-content` / `telemetry` classes), detects peer departures via the bus, and emits `ReplicationTask` orders with deterministic replacement-holder selection; `IntegrityVerifier` runs an injected `HashFn` (SHA-256 by default; BLAKE3 plug-compatible) over received chunk bytes, blacklists senders that ship corrupted chunks for a TTL window, and exposes a clear-text trust map for downstream observability. Wrapped in a single `ContentDeliveryPlugin` satisfying the `NetworkPlugin` lifecycle.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [index](./src/index/chunk-index.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [chunk-index](./src/index/chunk-index.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [integrity](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [integrity-verifier](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [packer](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [packer-integration](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [content-delivery-plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [prefetch](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [prefetch-engine](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [redundancy](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [redundancy-manager](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [chunk-request-router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [chunk-store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [throttle](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [bandwidth-throttler](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [error-codes](./src/error-codes.md) — Frozen `NET_CD` error registry for `@suaveplan/network-plugin-content-delivery`. Every failure path the content-deliver…
|
||||
<!-- end:auto-docs --> - [chunk-index](./src/index/chunk-index.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity-verifier](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer-integration](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [content-delivery-plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch-engine](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy-manager](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [chunk-request-router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [chunk-store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [throttle](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [bandwidth-throttler](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen `NET_CD` error registry for `@suaveplan/network-plugin-content-delivery`. Every failure path the content-deliver…
|
||||
- [chunk-index](./src/index/chunk-index.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity-verifier](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer-integration](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [content-delivery-plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch-engine](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy-manager](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [chunk-request-router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [chunk-store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [throttle](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [bandwidth-throttler](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen `NET_CD` error registry for `@suaveplan/network-plugin-content-delivery`. Every failure path the content-deliver…
|
||||
- [chunk-index](./src/index/chunk-index.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity-verifier](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer-integration](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [content-delivery-plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch-engine](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy-manager](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [chunk-request-router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [chunk-store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [throttle](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [bandwidth-throttler](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen `NET_CD` error registry for `@suaveplan/network-plugin-content-delivery`. Every failure path the content-deliver…
|
||||
- [chunk-index](./src/index/chunk-index.md) — <!-- begin:auto-docs --> _(no submodules)_- [integrity](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_ - [integrity-verifier](./src/integrity/integrity-verifier.md) — <!-- begin:auto-docs --> _(no submodules)_- [packer](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_ - [packer-integration](./src/packer/packer-integration.md) — <!-- begin:auto-docs --> _(no submodules)_- [plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_ - [content-delivery-plugin](./src/plugin/content-delivery-plugin.md) — <!-- begin:auto-docs --> _(no submodules)_- [prefetch](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ - [prefetch-engine](./src/prefetch/prefetch-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [redundancy](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_ - [redundancy-manager](./src/redundancy/redundancy-manager.md) — <!-- begin:auto-docs --> _(no submodules)_- [router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_ - [chunk-request-router](./src/router/chunk-request-router.md) — <!-- begin:auto-docs --> _(no submodules)_- [store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_ - [chunk-store](./src/store/chunk-store.md) — <!-- begin:auto-docs --> _(no submodules)_- [throttle](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_ - [bandwidth-throttler](./src/throttle/bandwidth-throttler.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Frozen `NET_CD` error registry for `@suaveplan/network-plugin-content-delivery`. Every failure path the content-deliver…
|
||||
|
||||
## Modules
|
||||
|
||||
- `index/chunk-index` — distributed holder map.
|
||||
- `router/chunk-request-router` — latency-aware request routing.
|
||||
- `redundancy/redundancy-manager` — N-copy enforcement + re-replication.
|
||||
- `integrity/integrity-verifier` — hash verification + sender blacklisting.
|
||||
- `packer/packer-integration` — adapts `@suaveplan/packer` output into `ChunkManifest` and broadcasts `cd:manifest`.
|
||||
- `store/chunk-store` — local chunk persistence over a pluggable `ChunkStoreAdapter` (`@suaveplan/drive` in production) with TTL + LRU eviction; emits `cd:chunk-stored` / `cd:chunk-evicted`.
|
||||
- `prefetch/prefetch-engine` — proactive fetch path with pluggable `PredictionModel` and session-scoped fan-out; emits `cd:prefetch-issued`.
|
||||
- `throttle/bandwidth-throttler` — per-peer + global token-bucket throttling tagging every admitted dispatch with `priority: "bulk"`; emits `cd:throttled`.
|
||||
- `plugin/content-delivery-plugin` — `NetworkPlugin` facade wiring every subsystem.
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"name": "@suaveplan/network-plugin-content-delivery",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Mesh-as-CDN core: ChunkIndex (gossip-propagated holder map), ChunkRequestRouter (latency-aware routing with server fallback), RedundancyManager (N-copy enforcement + re-replication on peer churn), IntegrityVerifier (hash-agnostic verification + sender blacklisting), and ContentDeliveryPlugin facade. Drive-store, packer, prefetch, and bandwidth-throttle integrations live in follow-up changes.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"files": ["dist", "src", "README.md"],
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
|
||||
"require": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "bunx biome check src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/network": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
# error-codes
|
||||
|
||||
## Purpose
|
||||
|
||||
Frozen `NET_CD` error registry for `@suaveplan/network-plugin-content-delivery`. Every failure path the content-delivery subsystem can surface — a request for an unknown chunk, no holder available across the mesh, an integrity hash mismatch, an attempt to receive bytes from a blacklisted peer, an invalid redundancy target, a missing index entry, or a router with no fallback configured — is mapped onto one of these stable codes.
|
||||
|
||||
The registry is locked at module load time via `defineCodes("NET_CD", [...] as const)` so consumers downstream (logger redaction maps, i18n catalogs, the OpenSpec conformance checker) can rely on the set never silently growing without a code change.
|
||||
|
||||
## Features
|
||||
|
||||
- Seven codes covering the chunk-discovery path (`CHUNK_NOT_FOUND`, `INDEX_ENTRY_MISSING`), routing failures (`NO_HOLDER_AVAILABLE`, `ROUTER_NO_FALLBACK`), integrity gates (`INTEGRITY_HASH_MISMATCH`, `SENDER_BLACKLISTED`), and redundancy configuration validation (`REDUNDANCY_TARGET_INVALID`).
|
||||
- Sibling `NetworkCdError` extends `SuaveplanError<NetworkCdErrorCode>` so consumers can write `instanceof NetworkCdError` and access `error.code` with full type narrowing.
|
||||
- Codes can be referenced via `error.code` matching as well as carried as outcome discriminants on report-style result types — both count as "used" for the conformance check.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
NETWORK_CD_CODES,
|
||||
NetworkCdError,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.NO_HOLDER_AVAILABLE,
|
||||
"cd.no_holder",
|
||||
{ context: { chunkHash: "aaa" } }
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
NetworkCdError,
|
||||
type NetworkCdErrorCode,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
function classify(err: unknown): NetworkCdErrorCode | null {
|
||||
return err instanceof NetworkCdError ? err.code : null;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `NETWORK_CD_CODES` — frozen object with seven keys: `CHUNK_NOT_FOUND`, `NO_HOLDER_AVAILABLE`, `INTEGRITY_HASH_MISMATCH`, `SENDER_BLACKLISTED`, `REDUNDANCY_TARGET_INVALID`, `INDEX_ENTRY_MISSING`, `ROUTER_NO_FALLBACK`. Values are `"NET_CD.<KEY>"` strings.
|
||||
- `NetworkCdErrorCode` — string-literal union of the registry values.
|
||||
- `NetworkCdError` — error class extending `SuaveplanError<NetworkCdErrorCode>`. Constructor: `(code, messageKey, options?)`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Per `AGENTS.md §2c`, every code in the registry must be referenced from `src/` somewhere — typically as the argument to `new NetworkCdError(...)` (thrown classes) but `INTEGRITY_HASH_MISMATCH` and `SENDER_BLACKLISTED` may be carried on `IntegrityVerifier` outcome unions when the verifier reports rather than throws. Either form satisfies the gate.
|
||||
- The list grows only by adding to the array literal; there is no programmatic `register()` API by design — the `as const` plus `defineCodes` arrangement is the contract.
|
||||
- Code messages use dotted i18n keys (e.g. `cd.no_holder`) so the logger's i18n adapter can resolve them at format time without coupling this package to a specific locale catalog.
|
||||
@@ -0,0 +1,54 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "./error-codes.js";
|
||||
|
||||
describe("NETWORK_CD_CODES", () => {
|
||||
it("registers under NET_CD", () => {
|
||||
for (const value of Object.values(NETWORK_CD_CODES)) {
|
||||
expect(value).toMatch(/^NET_CD\./);
|
||||
}
|
||||
});
|
||||
|
||||
it("is frozen", () => {
|
||||
expect(Object.isFrozen(NETWORK_CD_CODES)).toBe(true);
|
||||
});
|
||||
|
||||
it("contains the documented codes", () => {
|
||||
for (const key of [
|
||||
"CHUNK_NOT_FOUND",
|
||||
"NO_HOLDER_AVAILABLE",
|
||||
"INTEGRITY_HASH_MISMATCH",
|
||||
"SENDER_BLACKLISTED",
|
||||
"REDUNDANCY_TARGET_INVALID",
|
||||
"INDEX_ENTRY_MISSING",
|
||||
"ROUTER_NO_FALLBACK",
|
||||
"STORE_QUOTA_EXCEEDED",
|
||||
"STORE_CHUNK_NOT_FOUND",
|
||||
"PACKER_MANIFEST_INVALID",
|
||||
"PREFETCH_MODEL_FAILED",
|
||||
"THROTTLE_RATE_INVALID",
|
||||
] as const) {
|
||||
expect(NETWORK_CD_CODES).toHaveProperty(key);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("NetworkCdError", () => {
|
||||
it("extends SuaveplanError and preserves the code", () => {
|
||||
const err = new NetworkCdError(
|
||||
NETWORK_CD_CODES.NO_HOLDER_AVAILABLE,
|
||||
"cd.no_holder"
|
||||
);
|
||||
expect(err.name).toBe("NetworkCdError");
|
||||
expect(err.code).toBe(NETWORK_CD_CODES.NO_HOLDER_AVAILABLE);
|
||||
});
|
||||
|
||||
it("preserves the cause", () => {
|
||||
const cause = new Error("inner");
|
||||
const err = new NetworkCdError(
|
||||
NETWORK_CD_CODES.INTEGRITY_HASH_MISMATCH,
|
||||
"cd.integrity.hash_mismatch",
|
||||
{ cause }
|
||||
);
|
||||
expect(err.cause).toBe(cause);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
import {
|
||||
defineCodes,
|
||||
SuaveplanError,
|
||||
type SuaveplanErrorOptions,
|
||||
} from "@suaveplan/error";
|
||||
|
||||
export const NETWORK_CD_CODES = defineCodes("NET_CD", [
|
||||
"CHUNK_NOT_FOUND",
|
||||
"NO_HOLDER_AVAILABLE",
|
||||
"INTEGRITY_HASH_MISMATCH",
|
||||
"SENDER_BLACKLISTED",
|
||||
"REDUNDANCY_TARGET_INVALID",
|
||||
"INDEX_ENTRY_MISSING",
|
||||
"ROUTER_NO_FALLBACK",
|
||||
"STORE_QUOTA_EXCEEDED",
|
||||
"STORE_CHUNK_NOT_FOUND",
|
||||
"PACKER_MANIFEST_INVALID",
|
||||
"PREFETCH_MODEL_FAILED",
|
||||
"THROTTLE_RATE_INVALID",
|
||||
] as const);
|
||||
|
||||
export type NetworkCdErrorCode =
|
||||
(typeof NETWORK_CD_CODES)[keyof typeof NETWORK_CD_CODES];
|
||||
|
||||
export class NetworkCdError extends SuaveplanError<NetworkCdErrorCode> {
|
||||
public constructor(
|
||||
code: NetworkCdErrorCode,
|
||||
messageKey: string,
|
||||
options?: SuaveplanErrorOptions
|
||||
) {
|
||||
super(code, messageKey, options);
|
||||
this.name = "NetworkCdError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// Side-effect: register error codes
|
||||
import "./error-codes.js";
|
||||
|
||||
// Errors
|
||||
export {
|
||||
NETWORK_CD_CODES,
|
||||
NetworkCdError,
|
||||
type NetworkCdErrorCode,
|
||||
} from "./error-codes.js";
|
||||
// Index
|
||||
export {
|
||||
ChunkIndex,
|
||||
type ChunkIndexBus,
|
||||
type ChunkIndexConfig,
|
||||
} from "./index/chunk-index.js";
|
||||
// Integrity
|
||||
export {
|
||||
defaultEncode,
|
||||
type Encoder,
|
||||
type HashFn,
|
||||
type IntegrityBus,
|
||||
IntegrityVerifier,
|
||||
type IntegrityVerifierConfig,
|
||||
type VerifyOutcome,
|
||||
} from "./integrity/integrity-verifier.js";
|
||||
// Packer
|
||||
export {
|
||||
type PackerBus,
|
||||
PackerIntegration,
|
||||
type PackerIntegrationConfig,
|
||||
type PackerOutputLike,
|
||||
} from "./packer/packer-integration.js";
|
||||
// Plugin
|
||||
export {
|
||||
type ChunkStoreConfig as PluginChunkStoreConfig,
|
||||
type ContentDeliveryBus,
|
||||
ContentDeliveryPlugin,
|
||||
type ContentDeliveryPluginConfig,
|
||||
type PrefetchConfig as PluginPrefetchConfig,
|
||||
type ThrottleConfig as PluginThrottleConfig,
|
||||
} from "./plugin/content-delivery-plugin.js";
|
||||
// Prefetch
|
||||
export {
|
||||
type LocalChunkPredicate,
|
||||
type PredictionModel,
|
||||
type PrefetchBus,
|
||||
type PrefetchContext,
|
||||
type PrefetchDispatchedEvent,
|
||||
PrefetchEngine,
|
||||
type PrefetchEngineConfig,
|
||||
type PrefetchSummary,
|
||||
type SessionScopeProvider,
|
||||
} from "./prefetch/prefetch-engine.js";
|
||||
// Redundancy
|
||||
export {
|
||||
DEFAULT_REDUNDANCY,
|
||||
RedundancyManager,
|
||||
type RedundancyManagerConfig,
|
||||
} from "./redundancy/redundancy-manager.js";
|
||||
// Router
|
||||
export {
|
||||
ChunkRequestRouter,
|
||||
type ChunkRequestRouterConfig,
|
||||
type LatencyOracle,
|
||||
type RouteDecision,
|
||||
type RouteSource,
|
||||
type ServerFallbackFn,
|
||||
} from "./router/chunk-request-router.js";
|
||||
// Store
|
||||
export {
|
||||
type ChunkEvictedEvent,
|
||||
ChunkStore,
|
||||
type ChunkStoreAdapter,
|
||||
type ChunkStoreBus,
|
||||
type ChunkStoreConfig,
|
||||
type ChunkStoredEvent,
|
||||
MemoryChunkStoreAdapter,
|
||||
} from "./store/chunk-store.js";
|
||||
// Throttle
|
||||
export {
|
||||
BandwidthThrottler,
|
||||
type BandwidthThrottlerConfig,
|
||||
type BulkDispatch,
|
||||
type ThrottleBus,
|
||||
type ThrottledEvent,
|
||||
} from "./throttle/bandwidth-throttler.js";
|
||||
// Types
|
||||
export type {
|
||||
ChunkAdvertisement,
|
||||
ChunkBytes,
|
||||
ChunkIndexEntry,
|
||||
ChunkManifest,
|
||||
ChunkRequest,
|
||||
} from "./types/chunk.types.js";
|
||||
export type {
|
||||
RedundancyClass,
|
||||
RedundancyConfig,
|
||||
ReplicationReason,
|
||||
ReplicationTask,
|
||||
} from "./types/redundancy.types.js";
|
||||
@@ -0,0 +1,72 @@
|
||||
# ChunkIndex
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Distributed-by-design index of which peers hold which chunks. Each peer keeps a local copy of the index that it updates by ingesting `ChunkAdvertisement` messages either gossiped over the bus (`cd:advertise`) or piggy-backed on chunk responses. The index is the data structure both `ChunkRequestRouter` and `RedundancyManager` consult to make decisions — the router asks "who has chunk X?" and the redundancy manager asks "is chunk X under-replicated?".
|
||||
|
||||
The index uses a per-holder timestamp ledger to merge advertisements deterministically: a later advertisement from the same holder replaces an earlier one (so a peer that re-broadcasts after a restart wins), but an out-of-order advertisement that arrives late is dropped (so the index converges regardless of message ordering).
|
||||
|
||||
## Features
|
||||
|
||||
- Last-writer-wins gossip merge keyed on `(chunkHash, holderId)` with the holder's `advertisedAt` timestamp as the merge clock.
|
||||
- `ingestAdvertisement` returns a boolean indicating whether the index was updated, so the caller can avoid re-gossiping no-op advertisements.
|
||||
- `removeHolder(chunkHash, holderId)` for graceful chunk-level eviction; `removePeer(holderId)` for whole-peer departures (returns the affected chunk hashes so the redundancy manager can fan out replication tasks).
|
||||
- `setRedundancyTarget(chunkHash, target)` lets a higher-level policy bump or relax replication for individual chunks (e.g. promoting a "shared" chunk to "critical" once it goes mainline).
|
||||
- Optional `bus` re-broadcasts `cd:advertise` on every accepted advertisement, so observability adapters and downstream subsystems hear about the index change without polling.
|
||||
- `defaultRedundancyTarget` enforced at construction time (`>= 1`) and at `setRedundancyTarget` call time (`>= 1`); both raise `NET_CD.REDUNDANCY_TARGET_INVALID`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { ChunkIndex } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
index.ingestAdvertisement({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-1",
|
||||
advertisedAt: Date.now(),
|
||||
});
|
||||
console.log(index.listHolders("aaa")); // ["peer-1"]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ChunkIndex } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const index = new ChunkIndex({
|
||||
defaultRedundancyTarget: 3,
|
||||
clock: () => 1_700_000_000_000,
|
||||
bus: { emit: (topic, payload) => observability.publish(topic, payload) },
|
||||
});
|
||||
|
||||
index.ingestAdvertisement(advA);
|
||||
index.ingestAdvertisement(advB);
|
||||
index.setRedundancyTarget("aaa", 5);
|
||||
const affected = index.removePeer("peer-2");
|
||||
for (const hash of affected) redundancyManager.evaluate(hash);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ChunkIndex(config)` — `config: { defaultRedundancyTarget: number; clock?: () => number; bus?: ChunkIndexBus }`. Throws `NET_CD.REDUNDANCY_TARGET_INVALID` if `defaultRedundancyTarget < 1`.
|
||||
- `ingestAdvertisement(adv)` — returns `true` if the index was updated, `false` if the advertisement was dropped (older than the last-seen for the same holder).
|
||||
- `removeHolder(chunkHash, holderId)` — returns `true` if the holder was removed.
|
||||
- `removePeer(holderId)` — returns the list of chunk hashes the peer was holding.
|
||||
- `setRedundancyTarget(chunkHash, target)` — raises `NET_CD.INDEX_ENTRY_MISSING` if the chunk has no entry; raises `NET_CD.REDUNDANCY_TARGET_INVALID` if `target < 1`.
|
||||
- `getEntry(chunkHash)` — returns the `ChunkIndexEntry` or `undefined`.
|
||||
- `requireEntry(chunkHash)` — same as `getEntry` but raises `NET_CD.INDEX_ENTRY_MISSING` instead of returning `undefined`.
|
||||
- `listHolders(chunkHash)` — `readonly string[]`; empty if the chunk has no entry.
|
||||
- `size()` — number of distinct chunk hashes in the index.
|
||||
- `snapshot()` — defensive copy of every entry for serialization / debugging.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The `holderTimestamps` ledger is parallel to the `entries` map rather than embedded inside `ChunkIndexEntry`, because the entry shape is wire-stable and we don't want serializers to leak per-peer clocks. This keeps the wire-stable shape minimal at the cost of one extra lookup per ingest.
|
||||
- `removePeer` iterates the entire entries map. In a 100k-chunk mesh this is still micro-second work; if it becomes hot we'll add a reverse `holderId → Set<chunkHash>` index.
|
||||
- `setRedundancyTarget` is the only mutator that can raise — every other mutator silently coalesces invalid input. This matches the proposal's "best-effort" availability model: a stale or partially-formed advertisement should not break the index.
|
||||
@@ -0,0 +1,163 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkAdvertisement } from "../types/chunk.types.js";
|
||||
import { ChunkIndex } from "./chunk-index.js";
|
||||
|
||||
const adv = (
|
||||
overrides: Partial<ChunkAdvertisement> = {}
|
||||
): ChunkAdvertisement => ({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-1",
|
||||
advertisedAt: 1,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("ChunkIndex construction", () => {
|
||||
it("rejects defaultRedundancyTarget < 1", () => {
|
||||
expect(() => new ChunkIndex({ defaultRedundancyTarget: 0 })).toThrow(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkIndex.ingestAdvertisement", () => {
|
||||
it("adds a holder for a new chunk", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const updated = idx.ingestAdvertisement(adv());
|
||||
expect(updated).toBe(true);
|
||||
expect(idx.listHolders("aaa")).toEqual(["peer-1"]);
|
||||
});
|
||||
|
||||
it("merges two holders into the same entry", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ holderId: "peer-1", advertisedAt: 1 }));
|
||||
idx.ingestAdvertisement(adv({ holderId: "peer-2", advertisedAt: 1 }));
|
||||
expect(idx.listHolders("aaa").sort()).toEqual(["peer-1", "peer-2"]);
|
||||
});
|
||||
|
||||
it("drops an out-of-order older advertisement from the same holder", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ advertisedAt: 5 }));
|
||||
const updated = idx.ingestAdvertisement(adv({ advertisedAt: 3 }));
|
||||
expect(updated).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts a later advertisement from the same holder", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ advertisedAt: 3 }));
|
||||
const updated = idx.ingestAdvertisement(adv({ advertisedAt: 5 }));
|
||||
expect(updated).toBe(true);
|
||||
});
|
||||
|
||||
it("treats an equal-timestamp duplicate as a no-op", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ advertisedAt: 5 }));
|
||||
const updated = idx.ingestAdvertisement(adv({ advertisedAt: 5 }));
|
||||
expect(updated).toBe(false);
|
||||
});
|
||||
|
||||
it("does not duplicate the holder list when the same peer re-announces", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ advertisedAt: 1 }));
|
||||
idx.ingestAdvertisement(adv({ advertisedAt: 2 }));
|
||||
expect(idx.listHolders("aaa")).toEqual(["peer-1"]);
|
||||
});
|
||||
|
||||
it("emits cd:advertise on accepted advertisements", () => {
|
||||
const events: unknown[] = [];
|
||||
const idx = new ChunkIndex({
|
||||
defaultRedundancyTarget: 3,
|
||||
bus: { emit: (_t, p) => events.push(p) },
|
||||
});
|
||||
idx.ingestAdvertisement(adv());
|
||||
expect(events.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkIndex.removeHolder / removePeer", () => {
|
||||
it("removeHolder returns true when the holder existed and false otherwise", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv());
|
||||
expect(idx.removeHolder("aaa", "peer-1")).toBe(true);
|
||||
expect(idx.removeHolder("aaa", "peer-1")).toBe(false);
|
||||
expect(idx.removeHolder("nonexistent", "peer-1")).toBe(false);
|
||||
});
|
||||
|
||||
it("removePeer drops the peer from every chunk it held", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ chunkHash: "aaa", holderId: "peer-1" }));
|
||||
idx.ingestAdvertisement(adv({ chunkHash: "bbb", holderId: "peer-1" }));
|
||||
idx.ingestAdvertisement(adv({ chunkHash: "ccc", holderId: "peer-2" }));
|
||||
const affected = idx.removePeer("peer-1");
|
||||
expect(affected.sort()).toEqual(["aaa", "bbb"]);
|
||||
expect(idx.listHolders("aaa")).toEqual([]);
|
||||
expect(idx.listHolders("ccc")).toEqual(["peer-2"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkIndex.setRedundancyTarget", () => {
|
||||
it("updates the target on an existing entry", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv());
|
||||
idx.setRedundancyTarget("aaa", 5);
|
||||
expect(idx.requireEntry("aaa").redundancyTarget).toBe(5);
|
||||
});
|
||||
|
||||
it("rejects target < 1", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv());
|
||||
expect(() => idx.setRedundancyTarget("aaa", 0)).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("raises INDEX_ENTRY_MISSING when the chunk is unknown", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
try {
|
||||
idx.setRedundancyTarget("zzz", 5);
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.INDEX_ENTRY_MISSING
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkIndex inspectors", () => {
|
||||
it("requireEntry throws when missing, returns when present", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
expect(() => idx.requireEntry("zzz")).toThrow(NetworkCdError);
|
||||
idx.ingestAdvertisement(adv());
|
||||
expect(idx.requireEntry("aaa").chunkHash).toBe("aaa");
|
||||
});
|
||||
|
||||
it("getEntry returns undefined for missing chunks", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
expect(idx.getEntry("zzz")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("size and snapshot reflect the current state", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
idx.ingestAdvertisement(adv({ chunkHash: "aaa" }));
|
||||
idx.ingestAdvertisement(adv({ chunkHash: "bbb" }));
|
||||
expect(idx.size()).toBe(2);
|
||||
expect(idx.snapshot().length).toBe(2);
|
||||
});
|
||||
|
||||
it("listHolders returns an empty array for unknown chunks", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
expect(idx.listHolders("zzz")).toEqual([]);
|
||||
});
|
||||
|
||||
it("uses Date.now when no clock is injected", () => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const before = Date.now();
|
||||
idx.ingestAdvertisement(adv());
|
||||
const entry = idx.requireEntry("aaa");
|
||||
const after = Date.now();
|
||||
expect(entry.lastUpdated).toBeGreaterThanOrEqual(before);
|
||||
expect(entry.lastUpdated).toBeLessThanOrEqual(after);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,143 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type {
|
||||
ChunkAdvertisement,
|
||||
ChunkIndexEntry,
|
||||
} from "../types/chunk.types.js";
|
||||
|
||||
export interface ChunkIndexBus {
|
||||
emit(topic: "cd:advertise", payload: ChunkAdvertisement): void;
|
||||
}
|
||||
|
||||
export interface ChunkIndexConfig {
|
||||
readonly defaultRedundancyTarget: number;
|
||||
readonly clock?: () => number;
|
||||
readonly bus?: ChunkIndexBus;
|
||||
}
|
||||
|
||||
export class ChunkIndex {
|
||||
public readonly defaultRedundancyTarget: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly bus: ChunkIndexBus | undefined;
|
||||
private readonly entries = new Map<string, ChunkIndexEntry>();
|
||||
private readonly holderTimestamps = new Map<string, Map<string, number>>();
|
||||
|
||||
public constructor(config: ChunkIndexConfig) {
|
||||
if (config.defaultRedundancyTarget < 1) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.REDUNDANCY_TARGET_INVALID,
|
||||
"cd.redundancy.default_invalid",
|
||||
{ context: { value: config.defaultRedundancyTarget } }
|
||||
);
|
||||
}
|
||||
this.defaultRedundancyTarget = config.defaultRedundancyTarget;
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
this.bus = config.bus;
|
||||
}
|
||||
|
||||
public ingestAdvertisement(adv: ChunkAdvertisement): boolean {
|
||||
const perHolder =
|
||||
this.holderTimestamps.get(adv.chunkHash) ??
|
||||
new Map<string, number>();
|
||||
const previous = perHolder.get(adv.holderId);
|
||||
if (previous !== undefined && previous >= adv.advertisedAt) {
|
||||
return false;
|
||||
}
|
||||
perHolder.set(adv.holderId, adv.advertisedAt);
|
||||
this.holderTimestamps.set(adv.chunkHash, perHolder);
|
||||
const existing = this.entries.get(adv.chunkHash);
|
||||
const holders = existing?.holders.includes(adv.holderId)
|
||||
? existing.holders
|
||||
: [...(existing?.holders ?? []), adv.holderId];
|
||||
const next: ChunkIndexEntry = {
|
||||
chunkHash: adv.chunkHash,
|
||||
fileId: adv.fileId,
|
||||
chunkIndex: adv.chunkIndex,
|
||||
holders,
|
||||
redundancyTarget:
|
||||
existing?.redundancyTarget ?? this.defaultRedundancyTarget,
|
||||
lastUpdated: this.clock(),
|
||||
};
|
||||
this.entries.set(adv.chunkHash, next);
|
||||
this.bus?.emit("cd:advertise", adv);
|
||||
return true;
|
||||
}
|
||||
|
||||
public removeHolder(chunkHash: string, holderId: string): boolean {
|
||||
const entry = this.entries.get(chunkHash);
|
||||
if (entry === undefined) return false;
|
||||
if (!entry.holders.includes(holderId)) return false;
|
||||
const holders = entry.holders.filter((h) => h !== holderId);
|
||||
const next: ChunkIndexEntry = {
|
||||
...entry,
|
||||
holders,
|
||||
lastUpdated: this.clock(),
|
||||
};
|
||||
this.entries.set(chunkHash, next);
|
||||
const perHolder = this.holderTimestamps.get(chunkHash);
|
||||
perHolder?.delete(holderId);
|
||||
return true;
|
||||
}
|
||||
|
||||
public removePeer(holderId: string): readonly string[] {
|
||||
const affected: string[] = [];
|
||||
for (const [hash, entry] of this.entries) {
|
||||
if (entry.holders.includes(holderId)) {
|
||||
this.removeHolder(hash, holderId);
|
||||
affected.push(hash);
|
||||
}
|
||||
}
|
||||
return affected;
|
||||
}
|
||||
|
||||
public setRedundancyTarget(chunkHash: string, target: number): void {
|
||||
if (target < 1) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.REDUNDANCY_TARGET_INVALID,
|
||||
"cd.redundancy.target_invalid",
|
||||
{ context: { chunkHash, target } }
|
||||
);
|
||||
}
|
||||
const entry = this.entries.get(chunkHash);
|
||||
if (entry === undefined) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.INDEX_ENTRY_MISSING,
|
||||
"cd.index.entry_missing",
|
||||
{ context: { chunkHash } }
|
||||
);
|
||||
}
|
||||
const next: ChunkIndexEntry = {
|
||||
...entry,
|
||||
redundancyTarget: target,
|
||||
lastUpdated: this.clock(),
|
||||
};
|
||||
this.entries.set(chunkHash, next);
|
||||
}
|
||||
|
||||
public getEntry(chunkHash: string): ChunkIndexEntry | undefined {
|
||||
return this.entries.get(chunkHash);
|
||||
}
|
||||
|
||||
public requireEntry(chunkHash: string): ChunkIndexEntry {
|
||||
const entry = this.entries.get(chunkHash);
|
||||
if (entry === undefined) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.INDEX_ENTRY_MISSING,
|
||||
"cd.index.entry_missing",
|
||||
{ context: { chunkHash } }
|
||||
);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
public listHolders(chunkHash: string): readonly string[] {
|
||||
return this.entries.get(chunkHash)?.holders ?? [];
|
||||
}
|
||||
|
||||
public size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
public snapshot(): readonly ChunkIndexEntry[] {
|
||||
return [...this.entries.values()];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
# IntegrityVerifier
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Verifies that chunk bytes received from a peer match the expected hash, and blacklists peers that ship corrupted chunks for a TTL window. The verifier is hash-agnostic by design: callers inject a `HashFn` (typically `bytes => sha256(bytes)`, but the same interface accepts BLAKE3, SHA-3, or a hardware-backed verifier) so the package itself depends only on `@suaveplan/error` and `@suaveplan/types`.
|
||||
|
||||
The blacklist is the verifier's only piece of state. It expires automatically — a peer that ships one bad chunk gets a temporary cool-down rather than a permanent ban — which matches the proposal's "blacklists the sender temporarily" wording and lets the mesh self-heal across transient failures.
|
||||
|
||||
## Features
|
||||
|
||||
- Hash-agnostic verification via injectable `HashFn`; the verifier never imports a specific hashing library.
|
||||
- Hex-encoding by default (matches the rest of the audit / drive chain hashing); callers can override `encodeHash` to use base64 / multibase / etc.
|
||||
- TTL-based blacklist: a sender that ships a corrupted chunk is blacklisted for `blacklistTtlMs`; subsequent `verify` calls from the same sender raise `NET_CD.SENDER_BLACKLISTED` until the TTL expires.
|
||||
- `verify(chunk)` returns a discriminated `VerifyOutcome` so non-throwing call sites can branch cleanly; `verifyOrThrow(chunk)` is the throwing variant for "I expect this to be valid" sites.
|
||||
- `isBlacklisted(peerId)` lazily expires entries on read so the blacklist size stays bounded without a sweep timer.
|
||||
- `blacklistedPeers()` is a snapshot helper for observability dashboards.
|
||||
- `clearBlacklist()` for operator-driven recovery (e.g. after a deploy that shipped a bad hash).
|
||||
- Optional `bus` emits `cd:integrity-violation` per failure for fan-out to alerting.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { sha256 } from "@suaveplan/crypto-hash";
|
||||
import { IntegrityVerifier } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: (bytes) => new Uint8Array(sha256(bytes)),
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
|
||||
const outcome = verifier.verify({
|
||||
chunkHash: "expected-hex",
|
||||
bytes: receivedBytes,
|
||||
senderId: "peer-2",
|
||||
});
|
||||
if (!outcome.ok) console.warn(`bad chunk from ${outcome.senderId}`);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { IntegrityVerifier } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: (bytes) => blake3(bytes),
|
||||
encodeHash: (bytes) => base64.encode(bytes),
|
||||
blacklistTtlMs: 5 * 60_000,
|
||||
clock: () => 1_700_000_000_000,
|
||||
bus: { emit: (topic, payload) => observability.publish(topic, payload) },
|
||||
});
|
||||
|
||||
bus.on("cd:chunk-received", (chunk) => {
|
||||
try {
|
||||
verifier.verifyOrThrow(chunk);
|
||||
chunkStore.put(chunk);
|
||||
} catch (err) {
|
||||
metrics.increment("cd.integrity.failures");
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `IntegrityVerifier(config)` — `config: { hash; encodeHash?; blacklistTtlMs; clock?; bus? }`. A negative `blacklistTtlMs` is clamped to `0` (effectively no blacklist).
|
||||
- `verify(chunk)` — returns `VerifyOutcome` (`{ ok: true; chunkHash } | { ok: false; chunkHash; observedHash; senderId }`). Raises `NET_CD.SENDER_BLACKLISTED` when the sender is in the blacklist.
|
||||
- `verifyOrThrow(chunk)` — throws `NET_CD.INTEGRITY_HASH_MISMATCH` on hash mismatch (sender is also blacklisted as a side effect); throws `NET_CD.SENDER_BLACKLISTED` if already blacklisted.
|
||||
- `isBlacklisted(peerId)` — boolean; lazily expires the entry.
|
||||
- `blacklistedPeers()` — snapshot of currently-blacklisted peer IDs.
|
||||
- `clearBlacklist()` — drops every entry.
|
||||
- `defaultEncode(bytes)` — exported lowercase-hex helper that the verifier uses by default.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The verifier deliberately couples the blacklist to verification (a bad chunk auto-blacklists the sender) rather than exposing a separate `blacklist(peerId)` mutator; this avoids the foot-gun of "we noticed something off but forgot to actually blacklist."
|
||||
- `verify` and `verifyOrThrow` share the same code path — the throwing variant just unwraps the outcome — so semantics stay aligned.
|
||||
- Comparing the encoded hash as a string (rather than as raw bytes) means the encoding scheme determines equality. Mixing two callers with different encoders against the same chunk-hash field will produce false negatives; document `encodeHash` once at the boundary and stick with it.
|
||||
@@ -0,0 +1,234 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, type NetworkCdError } from "../error-codes.js";
|
||||
import { defaultEncode, IntegrityVerifier } from "./integrity-verifier.js";
|
||||
|
||||
const hashFn = (bytes: Uint8Array): Uint8Array => {
|
||||
let acc = 0;
|
||||
for (const b of bytes) acc = (acc * 31 + b) & 0xffff;
|
||||
return new Uint8Array([
|
||||
bytes[0] ?? 0,
|
||||
bytes.length & 0xff,
|
||||
(acc >>> 8) & 0xff,
|
||||
acc & 0xff,
|
||||
]);
|
||||
};
|
||||
|
||||
const expectedHashOf = (bytes: Uint8Array): string =>
|
||||
defaultEncode(hashFn(bytes));
|
||||
|
||||
describe("IntegrityVerifier.verify (happy path)", () => {
|
||||
it("accepts a chunk whose hash matches", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
const bytes = new Uint8Array([1, 2, 3]);
|
||||
const outcome = verifier.verify({
|
||||
chunkHash: expectedHashOf(bytes),
|
||||
bytes,
|
||||
senderId: "peer-2",
|
||||
});
|
||||
expect(outcome.ok).toBe(true);
|
||||
expect(verifier.isBlacklisted("peer-2")).toBe(false);
|
||||
});
|
||||
|
||||
it("supports a custom encoder", () => {
|
||||
const customEncode = (bytes: Uint8Array): string =>
|
||||
Array.from(bytes).join(",");
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
encodeHash: customEncode,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
const bytes = new Uint8Array([4, 5, 6]);
|
||||
const outcome = verifier.verify({
|
||||
chunkHash: customEncode(hashFn(bytes)),
|
||||
bytes,
|
||||
senderId: "peer-2",
|
||||
});
|
||||
expect(outcome.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("IntegrityVerifier.verify (failure paths)", () => {
|
||||
it("returns ok=false on hash mismatch and blacklists the sender", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
const bytes = new Uint8Array([1, 2, 3]);
|
||||
const outcome = verifier.verify({
|
||||
chunkHash: "not-the-real-hash",
|
||||
bytes,
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(outcome.ok).toBe(false);
|
||||
expect(verifier.isBlacklisted("peer-bad")).toBe(true);
|
||||
});
|
||||
|
||||
it("emits cd:integrity-violation on the bus", () => {
|
||||
const events: unknown[] = [];
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
bus: { emit: (_t, p) => events.push(p) },
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([9]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(events.length).toBe(1);
|
||||
});
|
||||
|
||||
it("throws SENDER_BLACKLISTED when verifying a blacklisted sender", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([9]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
try {
|
||||
verifier.verify({
|
||||
chunkHash: expectedHashOf(new Uint8Array([1])),
|
||||
bytes: new Uint8Array([1]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.SENDER_BLACKLISTED
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("IntegrityVerifier.verifyOrThrow", () => {
|
||||
it("returns void on success", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
const bytes = new Uint8Array([1, 2, 3]);
|
||||
verifier.verifyOrThrow({
|
||||
chunkHash: expectedHashOf(bytes),
|
||||
bytes,
|
||||
senderId: "peer-2",
|
||||
});
|
||||
});
|
||||
|
||||
it("throws INTEGRITY_HASH_MISMATCH on bad hash", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
try {
|
||||
verifier.verifyOrThrow({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([9]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.INTEGRITY_HASH_MISMATCH
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("IntegrityVerifier.blacklist lifecycle", () => {
|
||||
it("expires entries after the TTL", () => {
|
||||
let now = 1000;
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([1]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(verifier.isBlacklisted("peer-bad")).toBe(true);
|
||||
now = 1_500;
|
||||
expect(verifier.isBlacklisted("peer-bad")).toBe(false);
|
||||
});
|
||||
|
||||
it("blacklistedPeers returns currently-banned peers and excludes expired ones", () => {
|
||||
let now = 1000;
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 100,
|
||||
clock: () => now,
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([1]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(verifier.blacklistedPeers()).toEqual(["peer-bad"]);
|
||||
now = 1_500;
|
||||
expect(verifier.blacklistedPeers()).toEqual([]);
|
||||
});
|
||||
|
||||
it("clearBlacklist removes every entry", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([1]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
verifier.clearBlacklist();
|
||||
expect(verifier.isBlacklisted("peer-bad")).toBe(false);
|
||||
});
|
||||
|
||||
it("isBlacklisted returns false for unknown peers", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
expect(verifier.isBlacklisted("never-seen")).toBe(false);
|
||||
});
|
||||
|
||||
it("clamps a negative TTL to zero (blacklist effectively disabled)", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: -100,
|
||||
clock: () => 1000,
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([1]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(verifier.isBlacklisted("peer-bad")).toBe(false);
|
||||
});
|
||||
|
||||
it("uses Date.now when no clock is injected", () => {
|
||||
const verifier = new IntegrityVerifier({
|
||||
hash: hashFn,
|
||||
blacklistTtlMs: 60_000,
|
||||
});
|
||||
verifier.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([1]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(verifier.isBlacklisted("peer-bad")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("defaultEncode", () => {
|
||||
it("hex-encodes lowercase with single-digit padding", () => {
|
||||
expect(defaultEncode(new Uint8Array([0x00, 0x0f, 0xff]))).toBe(
|
||||
"000fff"
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkBytes } from "../types/chunk.types.js";
|
||||
|
||||
export type HashFn = (bytes: Uint8Array) => Uint8Array;
|
||||
|
||||
export type Encoder = (bytes: Uint8Array) => string;
|
||||
|
||||
export interface IntegrityBus {
|
||||
emit(
|
||||
topic: "cd:integrity-violation",
|
||||
payload: { chunkHash: string; senderId: string; observedHash: string }
|
||||
): void;
|
||||
}
|
||||
|
||||
export interface IntegrityVerifierConfig {
|
||||
readonly hash: HashFn;
|
||||
readonly encodeHash?: Encoder;
|
||||
readonly blacklistTtlMs: number;
|
||||
readonly clock?: () => number;
|
||||
readonly bus?: IntegrityBus;
|
||||
}
|
||||
|
||||
export type VerifyOutcome =
|
||||
| { readonly ok: true; readonly chunkHash: string }
|
||||
| {
|
||||
readonly ok: false;
|
||||
readonly chunkHash: string;
|
||||
readonly observedHash: string;
|
||||
readonly senderId: string;
|
||||
};
|
||||
|
||||
const HEX_TABLE = "0123456789abcdef";
|
||||
|
||||
export function defaultEncode(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (let i = 0; i < bytes.length; i += 1) {
|
||||
const b = bytes[i] as number;
|
||||
out += HEX_TABLE[(b >>> 4) & 0xf];
|
||||
out += HEX_TABLE[b & 0xf];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export class IntegrityVerifier {
|
||||
private readonly hash: HashFn;
|
||||
private readonly encodeHash: Encoder;
|
||||
private readonly blacklistTtlMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly bus: IntegrityBus | undefined;
|
||||
private readonly blacklist = new Map<string, number>();
|
||||
|
||||
public constructor(config: IntegrityVerifierConfig) {
|
||||
this.hash = config.hash;
|
||||
this.encodeHash = config.encodeHash ?? defaultEncode;
|
||||
this.blacklistTtlMs = Math.max(0, config.blacklistTtlMs);
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
this.bus = config.bus;
|
||||
}
|
||||
|
||||
public verify(chunk: ChunkBytes): VerifyOutcome {
|
||||
if (this.isBlacklisted(chunk.senderId)) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.SENDER_BLACKLISTED,
|
||||
"cd.integrity.sender_blacklisted",
|
||||
{
|
||||
context: {
|
||||
senderId: chunk.senderId,
|
||||
chunkHash: chunk.chunkHash,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
const observedHash = this.encodeHash(this.hash(chunk.bytes));
|
||||
if (observedHash === chunk.chunkHash) {
|
||||
return { ok: true, chunkHash: chunk.chunkHash };
|
||||
}
|
||||
this.blacklistSender(chunk.senderId);
|
||||
this.bus?.emit("cd:integrity-violation", {
|
||||
chunkHash: chunk.chunkHash,
|
||||
senderId: chunk.senderId,
|
||||
observedHash,
|
||||
});
|
||||
return {
|
||||
ok: false,
|
||||
chunkHash: chunk.chunkHash,
|
||||
observedHash,
|
||||
senderId: chunk.senderId,
|
||||
};
|
||||
}
|
||||
|
||||
public verifyOrThrow(chunk: ChunkBytes): void {
|
||||
const outcome = this.verify(chunk);
|
||||
if (outcome.ok) return;
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.INTEGRITY_HASH_MISMATCH,
|
||||
"cd.integrity.hash_mismatch",
|
||||
{
|
||||
context: {
|
||||
chunkHash: chunk.chunkHash,
|
||||
observedHash: outcome.observedHash,
|
||||
senderId: chunk.senderId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public isBlacklisted(peerId: string): boolean {
|
||||
const expiresAt = this.blacklist.get(peerId);
|
||||
if (expiresAt === undefined) return false;
|
||||
if (expiresAt <= this.clock()) {
|
||||
this.blacklist.delete(peerId);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public blacklistedPeers(): readonly string[] {
|
||||
const now = this.clock();
|
||||
const out: string[] = [];
|
||||
for (const [peerId, expiresAt] of this.blacklist) {
|
||||
if (expiresAt > now) out.push(peerId);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
public clearBlacklist(): void {
|
||||
this.blacklist.clear();
|
||||
}
|
||||
|
||||
private blacklistSender(senderId: string): void {
|
||||
this.blacklist.set(senderId, this.clock() + this.blacklistTtlMs);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
# PackerIntegration
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`PackerIntegration` adapts the chunked output of `@suaveplan/packer` into the
|
||||
wire-stable `ChunkManifest` consumed by the rest of the content-delivery
|
||||
subsystem. The packer pipeline is owned by a separate package (so any future
|
||||
change to compression, encryption, or signing flows independently of the
|
||||
mesh-CDN), and that package's output shape is not locked to `ChunkManifest`.
|
||||
This integration sits at the seam — it validates the packer payload, builds a
|
||||
manifest with stable field ordering, and broadcasts it on the
|
||||
`cd:manifest` bus topic so peers can warm their `ChunkIndex` ahead of the first
|
||||
peer-to-peer advertisement.
|
||||
|
||||
## Features
|
||||
|
||||
- Structural `PackerOutputLike` input — no hard runtime dep on `@suaveplan/packer`.
|
||||
- Defensive validation: empty `fileId`, empty `hashAlgorithm`, empty
|
||||
`packerVersion`, empty `chunkHashes` array, and non-string entries in the
|
||||
hash list all surface as `NETWORK_CD_CODES.PACKER_MANIFEST_INVALID` with
|
||||
rich `context` rather than producing a corrupted manifest.
|
||||
- `buildManifest()` is pure (return-only); `publish()` builds AND broadcasts on
|
||||
the optional bus so non-observability consumers can drop the bus and skip
|
||||
the side effect.
|
||||
- Defensive copy of `chunkHashes` so the returned manifest is not aliased to
|
||||
the caller's array.
|
||||
- Injectable clock for deterministic test wiring even when the field itself is
|
||||
not embedded today.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { PackerIntegration } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const packer = new PackerIntegration();
|
||||
const manifest = packer.buildManifest({
|
||||
fileId: "f-1",
|
||||
chunkHashes: ["h0", "h1", "h2"],
|
||||
hashAlgorithm: "sha256",
|
||||
packerVersion: "1.0.0",
|
||||
});
|
||||
// manifest.totalChunks === 3
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
Wired through a `ContentDeliveryBus` so peers receive the manifest on the wire
|
||||
the moment the upload completes, and constructed with an injected clock for
|
||||
deterministic replay in property tests:
|
||||
|
||||
```ts
|
||||
import {
|
||||
PackerIntegration,
|
||||
type ChunkManifest,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const seen: ChunkManifest[] = [];
|
||||
const packer = new PackerIntegration({
|
||||
bus: { emit: (_topic, payload) => seen.push(payload) },
|
||||
clock: () => 0,
|
||||
});
|
||||
const manifest = packer.publish({
|
||||
fileId: "release.tar",
|
||||
chunkHashes: ["h0", "h1"],
|
||||
hashAlgorithm: "blake3",
|
||||
packerVersion: "2.0.0-rc.4",
|
||||
});
|
||||
// seen[0] === manifest, seen[0].hashAlgorithm === "blake3"
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `new PackerIntegration(config?: PackerIntegrationConfig)` — construct. All
|
||||
fields optional.
|
||||
- `buildManifest(output: PackerOutputLike): ChunkManifest` — validate +
|
||||
convert. Throws `NetworkCdError(PACKER_MANIFEST_INVALID)` on any
|
||||
field-level failure.
|
||||
- `publish(output: PackerOutputLike): ChunkManifest` — `buildManifest()` then
|
||||
`bus.emit("cd:manifest", manifest)` if a bus was supplied. Returns the same
|
||||
manifest the bus saw so callers can confirm round-trip equality.
|
||||
- `PackerOutputLike` — `{ fileId, chunkHashes, hashAlgorithm, packerVersion }`.
|
||||
- `PackerBus` — `{ emit(topic: "cd:manifest", payload: ChunkManifest): void }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The integration keeps zero hard dep on `@suaveplan/packer`. The packer's
|
||||
real `PackedDataMetadata` already carries the four fields this contract
|
||||
requires; consumers either pass the metadata object directly or wrap it.
|
||||
- Validation is total — any malformed field triggers a typed throw so an
|
||||
upstream packer regression cannot silently poison the mesh-CDN holder map.
|
||||
- `hashAlgorithm` is propagated verbatim. The mesh trusts the packer's choice
|
||||
— the `IntegrityVerifier` is wired with the matching `HashFn` separately.
|
||||
- The field-order of the returned `ChunkManifest` matches the type definition
|
||||
(`fileId`, `totalChunks`, `chunkHashes`, `hashAlgorithm`, `packerVersion`)
|
||||
so JSON-serialised manifests are stable for content-addressed caching.
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkManifest } from "../types/chunk.types.js";
|
||||
import {
|
||||
PackerIntegration,
|
||||
type PackerOutputLike,
|
||||
} from "./packer-integration.js";
|
||||
|
||||
const out = (overrides: Partial<PackerOutputLike> = {}): PackerOutputLike => ({
|
||||
fileId: "f-1",
|
||||
chunkHashes: ["h0", "h1", "h2"],
|
||||
hashAlgorithm: "sha256",
|
||||
packerVersion: "1.0.0",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("PackerIntegration.buildManifest", () => {
|
||||
it("constructs a ChunkManifest from a valid packer output", () => {
|
||||
const p = new PackerIntegration();
|
||||
const manifest = p.buildManifest(out());
|
||||
expect(manifest.fileId).toBe("f-1");
|
||||
expect(manifest.totalChunks).toBe(3);
|
||||
expect(manifest.chunkHashes).toEqual(["h0", "h1", "h2"]);
|
||||
expect(manifest.hashAlgorithm).toBe("sha256");
|
||||
expect(manifest.packerVersion).toBe("1.0.0");
|
||||
});
|
||||
|
||||
it("propagates the hashAlgorithm verbatim", () => {
|
||||
const p = new PackerIntegration();
|
||||
const manifest = p.buildManifest(out({ hashAlgorithm: "blake3" }));
|
||||
expect(manifest.hashAlgorithm).toBe("blake3");
|
||||
});
|
||||
|
||||
it("returns a defensive copy of chunkHashes", () => {
|
||||
const p = new PackerIntegration();
|
||||
const source: string[] = ["h0", "h1"];
|
||||
const manifest = p.buildManifest(out({ chunkHashes: source }));
|
||||
source.push("mutated");
|
||||
expect(manifest.chunkHashes).toEqual(["h0", "h1"]);
|
||||
});
|
||||
|
||||
it("uses the injected clock", () => {
|
||||
const ticks: number[] = [];
|
||||
const p = new PackerIntegration({
|
||||
clock: () => {
|
||||
ticks.push(1);
|
||||
return 42;
|
||||
},
|
||||
});
|
||||
p.buildManifest(out());
|
||||
expect(ticks.length).toBe(1);
|
||||
});
|
||||
|
||||
it("uses Date.now when no clock injected", () => {
|
||||
const p = new PackerIntegration();
|
||||
// Build doesn't throw — proves default clock callable.
|
||||
expect(() => p.buildManifest(out())).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects empty fileId", () => {
|
||||
const p = new PackerIntegration();
|
||||
try {
|
||||
p.buildManifest(out({ fileId: "" }));
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.PACKER_MANIFEST_INVALID
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-string fileId", () => {
|
||||
const p = new PackerIntegration();
|
||||
const bad = { ...out(), fileId: 123 as unknown as string };
|
||||
expect(() => p.buildManifest(bad)).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects empty hashAlgorithm", () => {
|
||||
const p = new PackerIntegration();
|
||||
expect(() => p.buildManifest(out({ hashAlgorithm: "" }))).toThrow(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects non-string hashAlgorithm", () => {
|
||||
const p = new PackerIntegration();
|
||||
const bad = {
|
||||
...out(),
|
||||
hashAlgorithm: null as unknown as string,
|
||||
};
|
||||
expect(() => p.buildManifest(bad)).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects empty packerVersion", () => {
|
||||
const p = new PackerIntegration();
|
||||
expect(() => p.buildManifest(out({ packerVersion: "" }))).toThrow(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects non-string packerVersion", () => {
|
||||
const p = new PackerIntegration();
|
||||
const bad = {
|
||||
...out(),
|
||||
packerVersion: undefined as unknown as string,
|
||||
};
|
||||
expect(() => p.buildManifest(bad)).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects empty chunkHashes array", () => {
|
||||
const p = new PackerIntegration();
|
||||
expect(() => p.buildManifest(out({ chunkHashes: [] }))).toThrow(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects non-array chunkHashes", () => {
|
||||
const p = new PackerIntegration();
|
||||
const bad = {
|
||||
...out(),
|
||||
chunkHashes: "not-array" as unknown as readonly string[],
|
||||
};
|
||||
expect(() => p.buildManifest(bad)).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects empty entry in chunkHashes", () => {
|
||||
const p = new PackerIntegration();
|
||||
expect(() =>
|
||||
p.buildManifest(out({ chunkHashes: ["h0", "", "h2"] }))
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects non-string entry in chunkHashes", () => {
|
||||
const p = new PackerIntegration();
|
||||
const bad = {
|
||||
...out(),
|
||||
chunkHashes: ["h0", 99 as unknown as string],
|
||||
};
|
||||
expect(() => p.buildManifest(bad)).toThrow(NetworkCdError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PackerIntegration.publish", () => {
|
||||
it("emits cd:manifest on the bus", () => {
|
||||
const seen: ChunkManifest[] = [];
|
||||
const p = new PackerIntegration({
|
||||
bus: { emit: (_t, payload) => seen.push(payload) },
|
||||
});
|
||||
const manifest = p.publish(out());
|
||||
expect(seen.length).toBe(1);
|
||||
expect(seen[0]).toBe(manifest);
|
||||
});
|
||||
|
||||
it("does not throw when no bus configured", () => {
|
||||
const p = new PackerIntegration();
|
||||
expect(() => p.publish(out())).not.toThrow();
|
||||
});
|
||||
|
||||
it("does not emit when buildManifest throws", () => {
|
||||
const seen: ChunkManifest[] = [];
|
||||
const p = new PackerIntegration({
|
||||
bus: { emit: (_t, payload) => seen.push(payload) },
|
||||
});
|
||||
expect(() => p.publish(out({ fileId: "" }))).toThrow(NetworkCdError);
|
||||
expect(seen.length).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkManifest } from "../types/chunk.types.js";
|
||||
|
||||
/**
|
||||
* Structural shape of a `@suaveplan/packer` chunked output that this
|
||||
* integration can consume. Kept structural (no hard dep) so callers may pass
|
||||
* the `PackedData`-style record directly OR a thin adapter shape — anything
|
||||
* that exposes the four fields below.
|
||||
*/
|
||||
export interface PackerOutputLike {
|
||||
readonly fileId: string;
|
||||
readonly chunkHashes: readonly string[];
|
||||
readonly hashAlgorithm: string;
|
||||
readonly packerVersion: string;
|
||||
}
|
||||
|
||||
export interface PackerBus {
|
||||
emit(topic: "cd:manifest", payload: ChunkManifest): void;
|
||||
}
|
||||
|
||||
export interface PackerIntegrationConfig {
|
||||
readonly bus?: PackerBus;
|
||||
/**
|
||||
* Optional clock — currently unused by manifest construction but reserved
|
||||
* so future enrichment fields (e.g. issuance time) can be added without
|
||||
* breaking callers.
|
||||
*/
|
||||
readonly clock?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `PackerIntegration` adapts a `@suaveplan/packer` output into the wire-stable
|
||||
* `ChunkManifest` consumed by `ChunkIndex` + `ChunkRequestRouter`. It performs
|
||||
* defensive validation of the packer payload (so a malformed upstream produces
|
||||
* a typed `PACKER_MANIFEST_INVALID` error rather than silently corrupting the
|
||||
* mesh-CDN holder map) and broadcasts the manifest on `cd:manifest` so peers
|
||||
* can warm their indexes before the first advertisement arrives.
|
||||
*/
|
||||
export class PackerIntegration {
|
||||
private readonly bus: PackerBus | undefined;
|
||||
private readonly clock: () => number;
|
||||
|
||||
public constructor(config: PackerIntegrationConfig = {}) {
|
||||
this.bus = config.bus;
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
}
|
||||
|
||||
public buildManifest(output: PackerOutputLike): ChunkManifest {
|
||||
if (typeof output.fileId !== "string" || output.fileId.length === 0) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PACKER_MANIFEST_INVALID,
|
||||
"cd.packer.fileId_invalid",
|
||||
{ context: { fileId: output.fileId } }
|
||||
);
|
||||
}
|
||||
if (
|
||||
typeof output.hashAlgorithm !== "string" ||
|
||||
output.hashAlgorithm.length === 0
|
||||
) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PACKER_MANIFEST_INVALID,
|
||||
"cd.packer.hashAlgorithm_invalid",
|
||||
{ context: { hashAlgorithm: output.hashAlgorithm } }
|
||||
);
|
||||
}
|
||||
if (
|
||||
typeof output.packerVersion !== "string" ||
|
||||
output.packerVersion.length === 0
|
||||
) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PACKER_MANIFEST_INVALID,
|
||||
"cd.packer.packerVersion_invalid",
|
||||
{ context: { packerVersion: output.packerVersion } }
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Array.isArray(output.chunkHashes) ||
|
||||
output.chunkHashes.length === 0
|
||||
) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PACKER_MANIFEST_INVALID,
|
||||
"cd.packer.chunkHashes_empty",
|
||||
{ context: { fileId: output.fileId } }
|
||||
);
|
||||
}
|
||||
for (let i = 0; i < output.chunkHashes.length; i += 1) {
|
||||
const hash = output.chunkHashes[i];
|
||||
if (typeof hash !== "string" || hash.length === 0) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PACKER_MANIFEST_INVALID,
|
||||
"cd.packer.chunkHash_invalid",
|
||||
{ context: { fileId: output.fileId, chunkIndex: i } }
|
||||
);
|
||||
}
|
||||
}
|
||||
// Touch the clock so deterministic-clock tests can pin the call. The
|
||||
// value itself is reserved for future manifest enrichment; today it is
|
||||
// intentionally not embedded in the wire payload.
|
||||
void this.clock();
|
||||
const manifest: ChunkManifest = {
|
||||
fileId: output.fileId,
|
||||
totalChunks: output.chunkHashes.length,
|
||||
chunkHashes: [...output.chunkHashes],
|
||||
hashAlgorithm: output.hashAlgorithm,
|
||||
packerVersion: output.packerVersion,
|
||||
};
|
||||
return manifest;
|
||||
}
|
||||
|
||||
public publish(output: PackerOutputLike): ChunkManifest {
|
||||
const manifest = this.buildManifest(output);
|
||||
this.bus?.emit("cd:manifest", manifest);
|
||||
return manifest;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
# ContentDeliveryPlugin
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Single `NetworkPlugin` facade that wires `ChunkIndex`, `ChunkRequestRouter`, `RedundancyManager`, and `IntegrityVerifier` into a cohesive subsystem and registers them with a `NetworkNode` via the standard plugin lifecycle (`install` / `start` / `stop` / `dispose`). Consumers register exactly one of these per host; everything else is reachable via the public properties on the plugin instance.
|
||||
|
||||
The plugin keeps its own state minimal: the lifecycle just remembers the node and flips a started flag. The interesting state — the chunk index, the blacklist, the redundancy classifications — lives in the composed subsystems and survives `start` / `stop` cycles. Only `dispose` clears state, by emptying the integrity blacklist (so a re-installed plugin starts with a fresh slate).
|
||||
|
||||
## Features
|
||||
|
||||
- One-shot construction wires every subsystem from a single config object, threading the local peer id and clock through to each.
|
||||
- Public read-only access to `index`, `router`, `redundancy`, `integrity` so consumers can drop directly into any subsystem.
|
||||
- The router is auto-wired with `isBlacklisted: (peerId) => integrity.isBlacklisted(peerId)` so a blacklisted peer is automatically excluded from holder ranking — no manual wiring at the call site.
|
||||
- Optional `redundancy` config; defaults to the proposal's reference table (`critical: 5, shared: 3, user-content: 2, telemetry: 1`).
|
||||
- `dispose` clears the integrity blacklist so a re-installed plugin doesn't carry a stale ban list.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { sha256 } from "@suaveplan/crypto-hash";
|
||||
import { NetworkNode } from "@suaveplan/network";
|
||||
import { ContentDeliveryPlugin } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const node = new NetworkNode({ id: "peer-1" });
|
||||
const cd = new ContentDeliveryPlugin({
|
||||
localPeerId: "peer-1",
|
||||
defaultRedundancyTarget: 3,
|
||||
latency: webrtc.latencyForPeer,
|
||||
hash: (bytes) => new Uint8Array(sha256(bytes)),
|
||||
blacklistTtlMs: 60_000,
|
||||
candidatePool: () => peerRoster.snapshot(),
|
||||
});
|
||||
|
||||
await node.use(cd).start();
|
||||
cd.index.ingestAdvertisement(adv);
|
||||
const decision = cd.router.route("aaa");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ContentDeliveryPlugin } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const cd = new ContentDeliveryPlugin({
|
||||
localPeerId: "peer-1",
|
||||
defaultRedundancyTarget: 3,
|
||||
redundancy: {
|
||||
defaultClass: "shared",
|
||||
perClass: { critical: 7, shared: 4, "user-content": 2, telemetry: 1 },
|
||||
},
|
||||
latency: (peerId) => webrtc.latencyMs(peerId),
|
||||
fallback: (hash) => server.fetchChunk(hash),
|
||||
hash: (bytes) => blake3(bytes),
|
||||
blacklistTtlMs: 5 * 60_000,
|
||||
candidatePool: () => peerRoster.healthy(),
|
||||
clock: () => 1_700_000_000_000,
|
||||
bus: { emit: (topic, payload) => observability.publish(topic, payload) },
|
||||
});
|
||||
|
||||
cd.redundancy.classify("aaa", "critical");
|
||||
const tasks = cd.redundancy.onPeerDeparture("peer-3");
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ContentDeliveryPlugin(config)` — `config: ContentDeliveryPluginConfig`. See type definition for the full surface.
|
||||
- `name` — `"content-delivery"` (constant).
|
||||
- Subsystem properties: `index`, `router`, `redundancy`, `integrity`.
|
||||
- `install(node)` / `start()` / `stop()` / `dispose()` — `NetworkPlugin` lifecycle.
|
||||
- `host()` — returns the installed `NetworkNode` or `null`.
|
||||
- `isStarted()` — boolean lifecycle inspector.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The plugin uses spread-with-conditional-key (`...(config.x === undefined ? {} : { x: config.x })`) so that under `exactOptionalPropertyTypes` an `undefined` value never reaches a sub-component that expects an _absent_ key.
|
||||
- The `bus` is forwarded to both the index and the integrity verifier; their event topics are different (`cd:advertise` vs `cd:integrity-violation`) so they don't conflict.
|
||||
- `dispose` does NOT reset the chunk index — only the integrity blacklist. The chunk index is part of the durable mesh state and would be wrong to drop on a reload.
|
||||
@@ -0,0 +1,321 @@
|
||||
import type { NetworkNode } from "@suaveplan/network";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { ContentDeliveryPlugin } from "./content-delivery-plugin.js";
|
||||
|
||||
const stubHash = (bytes: Uint8Array): Uint8Array =>
|
||||
new Uint8Array([bytes[0] ?? 0, bytes.length & 0xff]);
|
||||
|
||||
const baseConfig = () => ({
|
||||
localPeerId: "peer-1",
|
||||
defaultRedundancyTarget: 3,
|
||||
latency: (_p: string) => 10,
|
||||
hash: stubHash,
|
||||
blacklistTtlMs: 60_000,
|
||||
candidatePool: () => ["peer-9"],
|
||||
});
|
||||
|
||||
describe("ContentDeliveryPlugin", () => {
|
||||
it("exposes its name as 'content-delivery'", () => {
|
||||
const plugin = new ContentDeliveryPlugin(baseConfig());
|
||||
expect(plugin.name).toBe("content-delivery");
|
||||
});
|
||||
|
||||
it("wires up every subsystem", () => {
|
||||
const plugin = new ContentDeliveryPlugin(baseConfig());
|
||||
expect(plugin.index).toBeDefined();
|
||||
expect(plugin.router).toBeDefined();
|
||||
expect(plugin.redundancy).toBeDefined();
|
||||
expect(plugin.integrity).toBeDefined();
|
||||
});
|
||||
|
||||
it("auto-wires the integrity blacklist into the router", () => {
|
||||
const plugin = new ContentDeliveryPlugin(baseConfig());
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-3",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
plugin.integrity.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([0]),
|
||||
senderId: "peer-2",
|
||||
});
|
||||
expect(plugin.router.route("aaa").peerId).toBe("peer-3");
|
||||
});
|
||||
|
||||
it("forwards an optional clock through to subsystems", () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
clock: () => 42,
|
||||
});
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
expect(plugin.index.requireEntry("aaa").lastUpdated).toBe(42);
|
||||
});
|
||||
|
||||
it("forwards an optional bus through to subsystems", () => {
|
||||
const events: Array<{ topic: string }> = [];
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
bus: { emit: (topic) => events.push({ topic }) },
|
||||
});
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
plugin.integrity.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([0]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
expect(events.find((e) => e.topic === "cd:advertise")).toBeDefined();
|
||||
expect(
|
||||
events.find((e) => e.topic === "cd:integrity-violation")
|
||||
).toBeDefined();
|
||||
});
|
||||
|
||||
it("forwards an optional fallback through to the router", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
fallback: () => new Uint8Array([7]),
|
||||
});
|
||||
const decision = plugin.router.route("zzz");
|
||||
expect(decision.source).toBe("server-fallback");
|
||||
const bytes = await plugin.router.fetchViaFallback("zzz");
|
||||
expect(bytes[0]).toBe(7);
|
||||
});
|
||||
|
||||
it("forwards an optional redundancy config through to the manager", () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
redundancy: {
|
||||
defaultClass: "shared",
|
||||
perClass: {
|
||||
critical: 7,
|
||||
shared: 4,
|
||||
"user-content": 2,
|
||||
telemetry: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(plugin.redundancy.targetFor("any")).toBe(4);
|
||||
});
|
||||
|
||||
it("exposes the packer integration getter", () => {
|
||||
const plugin = new ContentDeliveryPlugin(baseConfig());
|
||||
expect(plugin.packer).toBeDefined();
|
||||
const manifest = plugin.packer.buildManifest({
|
||||
fileId: "f-1",
|
||||
chunkHashes: ["h0"],
|
||||
hashAlgorithm: "sha256",
|
||||
packerVersion: "1.0.0",
|
||||
});
|
||||
expect(manifest.totalChunks).toBe(1);
|
||||
});
|
||||
|
||||
it("does not construct optional subsystems when their config is absent", () => {
|
||||
const plugin = new ContentDeliveryPlugin(baseConfig());
|
||||
expect(plugin.store).toBeUndefined();
|
||||
expect(plugin.prefetch).toBeUndefined();
|
||||
expect(plugin.throttler).toBeUndefined();
|
||||
});
|
||||
|
||||
it("constructs the store subsystem when configured", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
store: { maxBytes: 1024 },
|
||||
});
|
||||
expect(plugin.store).toBeDefined();
|
||||
await plugin.store?.put("h0", new Uint8Array([1]));
|
||||
expect(plugin.store?.has("h0")).toBe(true);
|
||||
});
|
||||
|
||||
it("constructs the store with maxAgeMs and adapter overrides", async () => {
|
||||
const inner = new Map<string, Uint8Array>();
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
store: {
|
||||
maxBytes: 1024,
|
||||
maxAgeMs: 1000,
|
||||
adapter: {
|
||||
put: (h, b) => {
|
||||
inner.set(h, b);
|
||||
},
|
||||
get: (h) => inner.get(h),
|
||||
delete: (h) => {
|
||||
inner.delete(h);
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await plugin.store?.put("h0", new Uint8Array([1]));
|
||||
expect(inner.has("h0")).toBe(true);
|
||||
});
|
||||
|
||||
it("constructs the prefetch engine and uses store.has when available", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
store: { maxBytes: 1024 },
|
||||
prefetch: {
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-x"],
|
||||
},
|
||||
});
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "h0",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-x",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
await plugin.store?.put("h0", new Uint8Array([1]));
|
||||
const out = await plugin.prefetch?.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
// store.has=true so prefetch skips locally
|
||||
expect(out?.skippedLocal).toEqual(["h0"]);
|
||||
});
|
||||
|
||||
it("constructs the prefetch engine with custom hasLocally + maxBatchSize", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
prefetch: {
|
||||
model: () => ["h0", "h1", "h2"],
|
||||
sessionScope: () => ["peer-x"],
|
||||
hasLocally: () => false,
|
||||
maxBatchSize: 2,
|
||||
},
|
||||
});
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "h0",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-x",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
plugin.index.ingestAdvertisement({
|
||||
chunkHash: "h1",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 1,
|
||||
holderId: "peer-x",
|
||||
advertisedAt: 1,
|
||||
});
|
||||
const out = await plugin.prefetch?.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out?.truncated).toBe(1);
|
||||
});
|
||||
|
||||
it("constructs the prefetch engine without store and without hasLocally override", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
prefetch: {
|
||||
model: () => [],
|
||||
sessionScope: () => [],
|
||||
},
|
||||
});
|
||||
const out = await plugin.prefetch?.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out?.peersAvailable).toBe(0);
|
||||
});
|
||||
|
||||
it("constructs the throttler with all override fields", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
throttle: {
|
||||
perPeerBytesPerSec: 1000,
|
||||
perPeerBurstBytes: 500,
|
||||
globalBytesPerSec: 5000,
|
||||
globalBurstBytes: 2500,
|
||||
sleep: async () => {
|
||||
/* no-op */
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(plugin.throttler).toBeDefined();
|
||||
const dispatch = await plugin.throttler?.admit("peer-a", "h0", 100);
|
||||
expect(dispatch?.priority).toBe("bulk");
|
||||
});
|
||||
|
||||
it("forwards clock + bus into store / prefetch / throttler subsystems", async () => {
|
||||
const events: Array<{ topic: string }> = [];
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
clock: () => 7,
|
||||
bus: { emit: (topic) => events.push({ topic }) },
|
||||
store: { maxBytes: 1024 },
|
||||
prefetch: {
|
||||
model: () => [],
|
||||
sessionScope: () => [],
|
||||
},
|
||||
throttle: {
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
},
|
||||
});
|
||||
await plugin.store?.put("h0", new Uint8Array([1]));
|
||||
await plugin.throttler?.admit("peer-a", "h0", 100);
|
||||
await plugin.prefetch?.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
// store emits cd:chunk-stored using the shared bus
|
||||
expect(events.find((e) => e.topic === "cd:chunk-stored")).toBeDefined();
|
||||
});
|
||||
|
||||
it("constructs the throttler with minimal config", async () => {
|
||||
const plugin = new ContentDeliveryPlugin({
|
||||
...baseConfig(),
|
||||
throttle: {
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
},
|
||||
});
|
||||
const dispatch = await plugin.throttler?.admit("peer-a", "h0", 100);
|
||||
expect(dispatch?.priority).toBe("bulk");
|
||||
});
|
||||
|
||||
it("install / start / stop / dispose lifecycle", () => {
|
||||
const plugin = new ContentDeliveryPlugin(baseConfig());
|
||||
const fakeNode = { id: "peer-1" } as unknown as NetworkNode;
|
||||
plugin.install(fakeNode);
|
||||
expect(plugin.host()).toBe(fakeNode);
|
||||
plugin.start();
|
||||
expect(plugin.isStarted()).toBe(true);
|
||||
plugin.stop();
|
||||
expect(plugin.isStarted()).toBe(false);
|
||||
plugin.integrity.verify({
|
||||
chunkHash: "wrong",
|
||||
bytes: new Uint8Array([0]),
|
||||
senderId: "peer-bad",
|
||||
});
|
||||
plugin.dispose();
|
||||
expect(plugin.host()).toBeNull();
|
||||
expect(plugin.integrity.isBlacklisted("peer-bad")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,205 @@
|
||||
import type { NetworkNode, NetworkPlugin } from "@suaveplan/network";
|
||||
import { ChunkIndex } from "../index/chunk-index.js";
|
||||
import {
|
||||
type HashFn,
|
||||
IntegrityVerifier,
|
||||
} from "../integrity/integrity-verifier.js";
|
||||
import { PackerIntegration } from "../packer/packer-integration.js";
|
||||
import {
|
||||
type LocalChunkPredicate,
|
||||
type PredictionModel,
|
||||
PrefetchEngine,
|
||||
type SessionScopeProvider,
|
||||
} from "../prefetch/prefetch-engine.js";
|
||||
import {
|
||||
DEFAULT_REDUNDANCY,
|
||||
RedundancyManager,
|
||||
} from "../redundancy/redundancy-manager.js";
|
||||
import {
|
||||
ChunkRequestRouter,
|
||||
type LatencyOracle,
|
||||
type ServerFallbackFn,
|
||||
} from "../router/chunk-request-router.js";
|
||||
import { ChunkStore, type ChunkStoreAdapter } from "../store/chunk-store.js";
|
||||
import { BandwidthThrottler } from "../throttle/bandwidth-throttler.js";
|
||||
import type { RedundancyConfig } from "../types/redundancy.types.js";
|
||||
|
||||
export interface ContentDeliveryBus {
|
||||
emit(topic: "cd:advertise", payload: unknown): void;
|
||||
emit(topic: "cd:integrity-violation", payload: unknown): void;
|
||||
emit(topic: "cd:manifest", payload: unknown): void;
|
||||
emit(topic: "cd:prefetch-issued", payload: unknown): void;
|
||||
emit(topic: "cd:chunk-stored", payload: unknown): void;
|
||||
emit(topic: "cd:chunk-evicted", payload: unknown): void;
|
||||
emit(topic: "cd:throttled", payload: unknown): void;
|
||||
}
|
||||
|
||||
export interface ChunkStoreConfig {
|
||||
readonly maxBytes: number;
|
||||
readonly maxAgeMs?: number;
|
||||
readonly adapter?: ChunkStoreAdapter;
|
||||
}
|
||||
|
||||
export interface PrefetchConfig {
|
||||
readonly model: PredictionModel;
|
||||
readonly sessionScope: SessionScopeProvider;
|
||||
readonly hasLocally?: LocalChunkPredicate;
|
||||
readonly maxBatchSize?: number;
|
||||
}
|
||||
|
||||
export interface ThrottleConfig {
|
||||
readonly perPeerBytesPerSec: number;
|
||||
readonly perPeerBurstBytes?: number;
|
||||
readonly globalBytesPerSec: number;
|
||||
readonly globalBurstBytes?: number;
|
||||
readonly sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface ContentDeliveryPluginConfig {
|
||||
readonly localPeerId: string;
|
||||
readonly defaultRedundancyTarget: number;
|
||||
readonly redundancy?: RedundancyConfig;
|
||||
readonly latency: LatencyOracle;
|
||||
readonly fallback?: ServerFallbackFn;
|
||||
readonly hash: HashFn;
|
||||
readonly blacklistTtlMs: number;
|
||||
readonly candidatePool: () => readonly string[];
|
||||
readonly clock?: () => number;
|
||||
readonly bus?: ContentDeliveryBus;
|
||||
readonly store?: ChunkStoreConfig;
|
||||
readonly prefetch?: PrefetchConfig;
|
||||
readonly throttle?: ThrottleConfig;
|
||||
}
|
||||
|
||||
export class ContentDeliveryPlugin implements NetworkPlugin {
|
||||
public readonly name = "content-delivery";
|
||||
public readonly index: ChunkIndex;
|
||||
public readonly router: ChunkRequestRouter;
|
||||
public readonly redundancy: RedundancyManager;
|
||||
public readonly integrity: IntegrityVerifier;
|
||||
public readonly packer: PackerIntegration;
|
||||
public readonly store: ChunkStore | undefined;
|
||||
public readonly prefetch: PrefetchEngine | undefined;
|
||||
public readonly throttler: BandwidthThrottler | undefined;
|
||||
private node: NetworkNode | null = null;
|
||||
private started = false;
|
||||
|
||||
public constructor(config: ContentDeliveryPluginConfig) {
|
||||
const bus = config.bus;
|
||||
const indexBus =
|
||||
bus === undefined ? undefined : { emit: bus.emit.bind(bus) };
|
||||
const integrityBus =
|
||||
bus === undefined ? undefined : { emit: bus.emit.bind(bus) };
|
||||
this.index = new ChunkIndex({
|
||||
defaultRedundancyTarget: config.defaultRedundancyTarget,
|
||||
...(config.clock === undefined ? {} : { clock: config.clock }),
|
||||
...(indexBus === undefined ? {} : { bus: indexBus }),
|
||||
});
|
||||
this.integrity = new IntegrityVerifier({
|
||||
hash: config.hash,
|
||||
blacklistTtlMs: config.blacklistTtlMs,
|
||||
...(config.clock === undefined ? {} : { clock: config.clock }),
|
||||
...(integrityBus === undefined ? {} : { bus: integrityBus }),
|
||||
});
|
||||
this.router = new ChunkRequestRouter({
|
||||
index: this.index,
|
||||
latency: config.latency,
|
||||
localPeerId: config.localPeerId,
|
||||
isBlacklisted: (peerId) => this.integrity.isBlacklisted(peerId),
|
||||
...(config.fallback === undefined
|
||||
? {}
|
||||
: { fallback: config.fallback }),
|
||||
});
|
||||
this.redundancy = new RedundancyManager({
|
||||
index: this.index,
|
||||
redundancy: config.redundancy ?? DEFAULT_REDUNDANCY,
|
||||
candidatePool: config.candidatePool,
|
||||
localPeerId: config.localPeerId,
|
||||
});
|
||||
this.packer = new PackerIntegration({
|
||||
...(config.clock === undefined ? {} : { clock: config.clock }),
|
||||
...(bus === undefined ? {} : { bus: { emit: bus.emit.bind(bus) } }),
|
||||
});
|
||||
if (config.store !== undefined) {
|
||||
this.store = new ChunkStore({
|
||||
maxBytes: config.store.maxBytes,
|
||||
...(config.store.maxAgeMs === undefined
|
||||
? {}
|
||||
: { maxAgeMs: config.store.maxAgeMs }),
|
||||
...(config.store.adapter === undefined
|
||||
? {}
|
||||
: { adapter: config.store.adapter }),
|
||||
...(config.clock === undefined ? {} : { clock: config.clock }),
|
||||
...(bus === undefined
|
||||
? {}
|
||||
: { bus: { emit: bus.emit.bind(bus) } }),
|
||||
});
|
||||
}
|
||||
if (config.prefetch !== undefined) {
|
||||
this.prefetch = new PrefetchEngine({
|
||||
model: config.prefetch.model,
|
||||
sessionScope: config.prefetch.sessionScope,
|
||||
index: this.index,
|
||||
...(() => {
|
||||
if (config.prefetch.hasLocally !== undefined) {
|
||||
return { hasLocally: config.prefetch.hasLocally };
|
||||
}
|
||||
const store = this.store;
|
||||
if (store === undefined) return {};
|
||||
return { hasLocally: (h: string) => store.has(h) };
|
||||
})(),
|
||||
...(config.prefetch.maxBatchSize === undefined
|
||||
? {}
|
||||
: { maxBatchSize: config.prefetch.maxBatchSize }),
|
||||
...(config.clock === undefined ? {} : { clock: config.clock }),
|
||||
...(bus === undefined
|
||||
? {}
|
||||
: { bus: { emit: bus.emit.bind(bus) } }),
|
||||
});
|
||||
}
|
||||
if (config.throttle !== undefined) {
|
||||
this.throttler = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: config.throttle.perPeerBytesPerSec,
|
||||
...(config.throttle.perPeerBurstBytes === undefined
|
||||
? {}
|
||||
: { perPeerBurstBytes: config.throttle.perPeerBurstBytes }),
|
||||
globalBytesPerSec: config.throttle.globalBytesPerSec,
|
||||
...(config.throttle.globalBurstBytes === undefined
|
||||
? {}
|
||||
: { globalBurstBytes: config.throttle.globalBurstBytes }),
|
||||
...(config.throttle.sleep === undefined
|
||||
? {}
|
||||
: { sleep: config.throttle.sleep }),
|
||||
...(config.clock === undefined ? {} : { clock: config.clock }),
|
||||
...(bus === undefined
|
||||
? {}
|
||||
: { bus: { emit: bus.emit.bind(bus) } }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public install(node: NetworkNode): void {
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public start(): void {
|
||||
this.started = true;
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
this.started = false;
|
||||
}
|
||||
|
||||
public dispose(): void {
|
||||
this.integrity.clearBlacklist();
|
||||
this.node = null;
|
||||
}
|
||||
|
||||
public host(): NetworkNode | null {
|
||||
return this.node;
|
||||
}
|
||||
|
||||
public isStarted(): boolean {
|
||||
return this.started;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
# PrefetchEngine
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`PrefetchEngine` is the proactive-fetch path for the mesh-CDN. Whenever a peer
|
||||
opens or advances a session, the engine consults a pluggable
|
||||
`PredictionModel` to ask "what chunks is this peer about to need?", filters the
|
||||
predictions against the local store + known holders + the requester's session
|
||||
scope, and broadcasts `cd:prefetch-issued` events targeting only peers that
|
||||
share the requester's session. Without this layer, the mesh services every
|
||||
chunk request reactively — every chunk waits for the requester's first miss.
|
||||
With it, hot working-set chunks arrive ahead of demand and the perceived tail
|
||||
latency of large file transfers collapses to the cost of a single ack
|
||||
round-trip.
|
||||
|
||||
## Features
|
||||
|
||||
- Pluggable `PredictionModel` returning an ordered list of chunk hashes (sync
|
||||
or async).
|
||||
- Session-scoped fan-out via injected `SessionScopeProvider` — peers outside
|
||||
the requester's session are never targeted.
|
||||
- Self-skip: the requester is filtered out of the dispatch even when the
|
||||
scope provider returns it.
|
||||
- Local-store dedupe via injected `LocalChunkPredicate` — chunks the local
|
||||
node already holds are skipped (no useless self-issued requests).
|
||||
- Index-aware dedupe: chunks no in-scope peer holds yet are reported as
|
||||
`skippedAlreadyHeld` (the wire request would just bounce).
|
||||
- Hard `maxBatchSize` cap (default 16) with truncation count returned to the
|
||||
caller.
|
||||
- Per-dispatch summary: `{ issued, skippedLocal, skippedAlreadyHeld, truncated,
|
||||
peersAvailable }` so callers can attach metrics/decision-trace logging.
|
||||
- Defensive validation: a model that throws or returns a non-array surfaces
|
||||
as `NETWORK_CD_CODES.PREFETCH_MODEL_FAILED` with the original error chained.
|
||||
- Empty `cd:prefetch-issued` event volume when scope is empty — never burn
|
||||
bus capacity on no-op dispatches.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
ChunkIndex,
|
||||
PrefetchEngine,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0", "h1"],
|
||||
sessionScope: () => ["peer-a", "peer-b"],
|
||||
index,
|
||||
});
|
||||
await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
Async prediction model, local-store dedupe, custom batch cap, and observability
|
||||
bus wiring:
|
||||
|
||||
```ts
|
||||
import {
|
||||
ChunkIndex,
|
||||
PrefetchEngine,
|
||||
type PredictionModel,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const localCache = new Set<string>(["h0"]);
|
||||
const model: PredictionModel = async (ctx) => {
|
||||
return ctx.recentChunkHashes.flatMap((h) => [`${h}-next`, `${h}-after`]);
|
||||
};
|
||||
const issued: unknown[] = [];
|
||||
const engine = new PrefetchEngine({
|
||||
model,
|
||||
sessionScope: async (sessionId) => {
|
||||
// Look up scope from your session manager.
|
||||
return sessionId === "s-1" ? ["peer-a", "peer-b"] : [];
|
||||
},
|
||||
index,
|
||||
hasLocally: (h) => localCache.has(h),
|
||||
bus: { emit: (_t, payload) => issued.push(payload) },
|
||||
maxBatchSize: 8,
|
||||
clock: () => 0,
|
||||
});
|
||||
const summary = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: ["h0", "h1"],
|
||||
hint: { route: "warm" },
|
||||
});
|
||||
// summary.issued is the subset that was actually dispatched
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `new PrefetchEngine(config: PrefetchEngineConfig)` — `model`, `sessionScope`,
|
||||
`index` are required. Other fields optional.
|
||||
- `dispatch(context: PrefetchContext): Promise<PrefetchSummary>` — runs the
|
||||
pipeline. Always returns; never throws on empty scope or empty predictions.
|
||||
Throws `NetworkCdError(PREFETCH_MODEL_FAILED)` on model failure.
|
||||
- `PrefetchContext` — `{ sessionId, requesterId, recentChunkHashes, hint? }`.
|
||||
- `PredictionModel` — `(ctx) => readonly string[] | Promise<readonly string[]>`.
|
||||
- `SessionScopeProvider` — `(sessionId) => readonly string[] | Promise<...>`.
|
||||
- `LocalChunkPredicate` — `(chunkHash) => boolean`.
|
||||
- `PrefetchSummary` — `{ issued, skippedLocal, skippedAlreadyHeld, truncated,
|
||||
peersAvailable }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The engine never dispatches to the requester itself — `peersFiltered`
|
||||
removes `context.requesterId` from the scope list before any fan-out.
|
||||
- Dedupe is order-preserving: the first occurrence of a chunk hash in the
|
||||
prediction list wins; subsequent duplicates are skipped silently.
|
||||
- Empty/non-string entries in the prediction list are skipped silently rather
|
||||
than triggering `PREFETCH_MODEL_FAILED` — the model is allowed to emit
|
||||
defensive empty placeholders without trapping the engine.
|
||||
- `truncated` is computed BEFORE dedupe / skip filtering so callers can tell
|
||||
when the model is over-producing relative to `maxBatchSize`.
|
||||
- `cd:prefetch-issued` carries `targetPeers` already filtered to in-scope
|
||||
holders so the transport layer can fan out without re-querying the index.
|
||||
@@ -0,0 +1,363 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import { ChunkIndex } from "../index/chunk-index.js";
|
||||
import {
|
||||
type PrefetchDispatchedEvent,
|
||||
PrefetchEngine,
|
||||
} from "./prefetch-engine.js";
|
||||
|
||||
const seedHolders = (
|
||||
index: ChunkIndex,
|
||||
pairs: ReadonlyArray<readonly [string, string]>
|
||||
): void => {
|
||||
for (const [chunkHash, holderId] of pairs) {
|
||||
index.ingestAdvertisement({
|
||||
chunkHash,
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId,
|
||||
advertisedAt: 1,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
describe("PrefetchEngine construction", () => {
|
||||
it("throws on maxBatchSize < 1", () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
expect(
|
||||
() =>
|
||||
new PrefetchEngine({
|
||||
model: () => [],
|
||||
sessionScope: () => [],
|
||||
index,
|
||||
maxBatchSize: 0,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("throws on non-finite maxBatchSize", () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
expect(
|
||||
() =>
|
||||
new PrefetchEngine({
|
||||
model: () => [],
|
||||
sessionScope: () => [],
|
||||
index,
|
||||
maxBatchSize: Number.POSITIVE_INFINITY,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PrefetchEngine.dispatch — model invocation", () => {
|
||||
it("invokes the model with the supplied context", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
let captured: unknown;
|
||||
const engine = new PrefetchEngine({
|
||||
model: (ctx) => {
|
||||
captured = ctx;
|
||||
return [];
|
||||
},
|
||||
sessionScope: () => [],
|
||||
index,
|
||||
});
|
||||
await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: ["h0"],
|
||||
});
|
||||
expect(captured).toEqual({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: ["h0"],
|
||||
});
|
||||
});
|
||||
|
||||
it("supports an async model", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: async () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual(["h0"]);
|
||||
});
|
||||
|
||||
it("wraps a model that throws as PREFETCH_MODEL_FAILED", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => {
|
||||
throw new Error("boom");
|
||||
},
|
||||
sessionScope: () => [],
|
||||
index,
|
||||
});
|
||||
try {
|
||||
await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.PREFETCH_MODEL_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a model that returns a non-array", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => "not-array" as unknown as readonly string[],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
await expect(
|
||||
engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
})
|
||||
).rejects.toBeInstanceOf(NetworkCdError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PrefetchEngine.dispatch — scope filtering", () => {
|
||||
it("returns peersAvailable=0 when scope is empty", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => [],
|
||||
index,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.peersAvailable).toBe(0);
|
||||
expect(out.issued).toEqual([]);
|
||||
});
|
||||
|
||||
it("filters self out of the scope", async () => {
|
||||
const events: PrefetchDispatchedEvent[] = [];
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-self", "peer-a"],
|
||||
index,
|
||||
bus: { emit: (_t, payload) => events.push(payload) },
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual(["h0"]);
|
||||
expect(events[0]?.targetPeers).toEqual(["peer-a"]);
|
||||
});
|
||||
|
||||
it("supports an async scope provider", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: async () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.peersAvailable).toBe(1);
|
||||
});
|
||||
|
||||
it("reports skippedAlreadyHeld for chunks no in-scope peer holds", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-out-of-scope"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual([]);
|
||||
expect(out.skippedAlreadyHeld).toEqual(["h0"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PrefetchEngine.dispatch — local store dedupe", () => {
|
||||
it("skips chunks the local node already holds", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
hasLocally: (h) => h === "h0",
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.skippedLocal).toEqual(["h0"]);
|
||||
expect(out.issued).toEqual([]);
|
||||
});
|
||||
|
||||
it("uses the default permissive predicate when none provided", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual(["h0"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PrefetchEngine.dispatch — dedupe + truncation", () => {
|
||||
it("dedupes repeated predictions", async () => {
|
||||
const events: PrefetchDispatchedEvent[] = [];
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0", "h0", "h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
bus: { emit: (_t, payload) => events.push(payload) },
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual(["h0"]);
|
||||
expect(events.length).toBe(1);
|
||||
});
|
||||
|
||||
it("truncates predictions beyond maxBatchSize", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [
|
||||
["h0", "peer-a"],
|
||||
["h1", "peer-a"],
|
||||
["h2", "peer-a"],
|
||||
]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0", "h1", "h2"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
maxBatchSize: 2,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual(["h0", "h1"]);
|
||||
expect(out.truncated).toBe(1);
|
||||
});
|
||||
|
||||
it("skips empty / non-string predictions silently", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h1", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => [
|
||||
"",
|
||||
"h1",
|
||||
null as unknown as string,
|
||||
undefined as unknown as string,
|
||||
],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
const out = await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(out.issued).toEqual(["h1"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("PrefetchEngine.dispatch — observability", () => {
|
||||
it("emits cd:prefetch-issued with issuedAt from the injected clock", async () => {
|
||||
const events: PrefetchDispatchedEvent[] = [];
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
let tick = 100;
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
bus: { emit: (_t, payload) => events.push(payload) },
|
||||
clock: () => {
|
||||
tick += 1;
|
||||
return tick;
|
||||
},
|
||||
});
|
||||
await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(events[0]?.issuedAt).toBe(101);
|
||||
});
|
||||
|
||||
it("uses Date.now when no clock injected", async () => {
|
||||
const events: PrefetchDispatchedEvent[] = [];
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
bus: { emit: (_t, payload) => events.push(payload) },
|
||||
});
|
||||
const before = Date.now();
|
||||
await engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
});
|
||||
expect(events[0]?.issuedAt).toBeGreaterThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("does not throw when no bus configured", async () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
seedHolders(index, [["h0", "peer-a"]]);
|
||||
const engine = new PrefetchEngine({
|
||||
model: () => ["h0"],
|
||||
sessionScope: () => ["peer-a"],
|
||||
index,
|
||||
});
|
||||
await expect(
|
||||
engine.dispatch({
|
||||
sessionId: "s-1",
|
||||
requesterId: "peer-self",
|
||||
recentChunkHashes: [],
|
||||
})
|
||||
).resolves.toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,193 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkIndex } from "../index/chunk-index.js";
|
||||
|
||||
/**
|
||||
* Pluggable prediction model. Receives a session-scoped context and returns
|
||||
* an ordered list of chunk hashes the model believes the requester is most
|
||||
* likely to need next. Order matters: the engine prefetches in the order
|
||||
* returned (highest confidence first).
|
||||
*/
|
||||
export type PredictionModel = (
|
||||
context: PrefetchContext
|
||||
) => readonly string[] | Promise<readonly string[]>;
|
||||
|
||||
export interface PrefetchContext {
|
||||
readonly sessionId: string;
|
||||
readonly requesterId: string;
|
||||
readonly recentChunkHashes: readonly string[];
|
||||
readonly hint?: Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the set of peers in the requester's current session scope. Returning
|
||||
* an empty array short-circuits the dispatch — the engine reports
|
||||
* `peersAvailable: 0` and emits no prefetch requests.
|
||||
*/
|
||||
export type SessionScopeProvider = (
|
||||
sessionId: string
|
||||
) => readonly string[] | Promise<readonly string[]>;
|
||||
|
||||
/**
|
||||
* Returns true when the local node already holds the chunk (so no prefetch is
|
||||
* needed). Caller plugs this into the local `ChunkStore`.
|
||||
*/
|
||||
export type LocalChunkPredicate = (chunkHash: string) => boolean;
|
||||
|
||||
export interface PrefetchDispatchedEvent {
|
||||
readonly sessionId: string;
|
||||
readonly requesterId: string;
|
||||
readonly chunkHash: string;
|
||||
readonly targetPeers: readonly string[];
|
||||
readonly issuedAt: number;
|
||||
}
|
||||
|
||||
export interface PrefetchBus {
|
||||
emit(topic: "cd:prefetch-issued", payload: PrefetchDispatchedEvent): void;
|
||||
}
|
||||
|
||||
export interface PrefetchEngineConfig {
|
||||
readonly model: PredictionModel;
|
||||
readonly sessionScope: SessionScopeProvider;
|
||||
readonly index: ChunkIndex;
|
||||
/**
|
||||
* Returns true when the local store already holds the chunk. Defaults to
|
||||
* a permissive `() => false` so an unwired prefetch engine never silently
|
||||
* skips dispatches.
|
||||
*/
|
||||
readonly hasLocally?: LocalChunkPredicate;
|
||||
readonly clock?: () => number;
|
||||
readonly bus?: PrefetchBus;
|
||||
/**
|
||||
* Hard cap on chunks dispatched per `dispatch()` call. Defaults to 16.
|
||||
* Predictions beyond this cap are silently truncated; the engine reports
|
||||
* the truncated count via the returned summary.
|
||||
*/
|
||||
readonly maxBatchSize?: number;
|
||||
}
|
||||
|
||||
export interface PrefetchSummary {
|
||||
readonly issued: readonly string[];
|
||||
readonly skippedLocal: readonly string[];
|
||||
readonly skippedAlreadyHeld: readonly string[];
|
||||
readonly truncated: number;
|
||||
readonly peersAvailable: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `PrefetchEngine` proactively requests chunks the prediction model thinks the
|
||||
* requester is about to need. Predictions are filtered against the local
|
||||
* `ChunkStore` (skip what we already have) and the `ChunkIndex` (skip chunks
|
||||
* already held by every peer in scope), then fanned out to the session-scoped
|
||||
* peer set so prefetch traffic never crosses session boundaries.
|
||||
*/
|
||||
export class PrefetchEngine {
|
||||
private readonly model: PredictionModel;
|
||||
private readonly sessionScope: SessionScopeProvider;
|
||||
private readonly index: ChunkIndex;
|
||||
private readonly hasLocally: LocalChunkPredicate;
|
||||
private readonly clock: () => number;
|
||||
private readonly bus: PrefetchBus | undefined;
|
||||
private readonly maxBatchSize: number;
|
||||
|
||||
public constructor(config: PrefetchEngineConfig) {
|
||||
this.model = config.model;
|
||||
this.sessionScope = config.sessionScope;
|
||||
this.index = config.index;
|
||||
this.hasLocally = config.hasLocally ?? (() => false);
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
this.bus = config.bus;
|
||||
const cap = config.maxBatchSize ?? 16;
|
||||
if (!Number.isFinite(cap) || cap < 1) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PREFETCH_MODEL_FAILED,
|
||||
"cd.prefetch.maxBatchSize_invalid",
|
||||
{ context: { maxBatchSize: cap } }
|
||||
);
|
||||
}
|
||||
this.maxBatchSize = Math.floor(cap);
|
||||
}
|
||||
|
||||
public async dispatch(context: PrefetchContext): Promise<PrefetchSummary> {
|
||||
let predictions: readonly string[];
|
||||
try {
|
||||
predictions = await this.model(context);
|
||||
} catch (cause) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PREFETCH_MODEL_FAILED,
|
||||
"cd.prefetch.model_threw",
|
||||
{
|
||||
cause,
|
||||
context: {
|
||||
sessionId: context.sessionId,
|
||||
requesterId: context.requesterId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
if (!Array.isArray(predictions)) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.PREFETCH_MODEL_FAILED,
|
||||
"cd.prefetch.model_returned_non_array",
|
||||
{
|
||||
context: {
|
||||
sessionId: context.sessionId,
|
||||
requesterId: context.requesterId,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
const peers = await this.sessionScope(context.sessionId);
|
||||
const peersFiltered = peers.filter((p) => p !== context.requesterId);
|
||||
const truncated = Math.max(0, predictions.length - this.maxBatchSize);
|
||||
const capped = predictions.slice(0, this.maxBatchSize);
|
||||
const seen = new Set<string>();
|
||||
const issued: string[] = [];
|
||||
const skippedLocal: string[] = [];
|
||||
const skippedAlreadyHeld: string[] = [];
|
||||
if (peersFiltered.length === 0) {
|
||||
return {
|
||||
issued,
|
||||
skippedLocal,
|
||||
skippedAlreadyHeld,
|
||||
truncated,
|
||||
peersAvailable: 0,
|
||||
};
|
||||
}
|
||||
for (const chunkHash of capped) {
|
||||
if (typeof chunkHash !== "string" || chunkHash.length === 0) {
|
||||
continue;
|
||||
}
|
||||
if (seen.has(chunkHash)) continue;
|
||||
seen.add(chunkHash);
|
||||
if (this.hasLocally(chunkHash)) {
|
||||
skippedLocal.push(chunkHash);
|
||||
continue;
|
||||
}
|
||||
const holders = this.index.listHolders(chunkHash);
|
||||
const peerHolders = holders.filter((h) =>
|
||||
peersFiltered.includes(h)
|
||||
);
|
||||
if (peerHolders.length === 0) {
|
||||
// No in-scope peer holds it — useless to prefetch.
|
||||
skippedAlreadyHeld.push(chunkHash);
|
||||
continue;
|
||||
}
|
||||
const issuedAt = this.clock();
|
||||
this.bus?.emit("cd:prefetch-issued", {
|
||||
sessionId: context.sessionId,
|
||||
requesterId: context.requesterId,
|
||||
chunkHash,
|
||||
targetPeers: peerHolders,
|
||||
issuedAt,
|
||||
});
|
||||
issued.push(chunkHash);
|
||||
}
|
||||
return {
|
||||
issued,
|
||||
skippedLocal,
|
||||
skippedAlreadyHeld,
|
||||
truncated,
|
||||
peersAvailable: peersFiltered.length,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
# RedundancyManager
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Enforces "every chunk lives on at least N peers" across the mesh. The manager classifies chunks (`critical` / `shared` / `user-content` / `telemetry`) per the proposal's table, watches the index for under-replication, and emits `ReplicationTask` orders that the consuming `drive-mesh` scheduler executes. The manager itself is policy-only: it never fetches bytes, never opens transports, and never picks the _order_ of fetches — it just decides _when_ and _to whom_ a copy should propagate.
|
||||
|
||||
The four-class taxonomy is closed by design: adding a new class is a code change, not a configuration change. This keeps the redundancy table auditable in source rather than in dynamic configuration that can drift.
|
||||
|
||||
## Features
|
||||
|
||||
- `DEFAULT_REDUNDANCY` exports the proposal's reference table (`critical: 5, shared: 3, user-content: 2, telemetry: 1`); consumers can pass it as-is or override per-class.
|
||||
- `classify(chunkHash, klass)` records the classification AND propagates the implied `redundancyTarget` into the `ChunkIndex`, so the index stays the single source of truth for per-chunk targets.
|
||||
- `evaluate(chunkHash)` answers "is this chunk under-replicated, and if so where should new copies go?" — returns `null` when the chunk is already at or above target.
|
||||
- `onPeerDeparture(holderId)` is the single bus-listener entry point: ingests a peer-left event, removes the peer from the index, evaluates every affected chunk, and returns the bundle of replication tasks ready to dispatch.
|
||||
- `requestManualReplication(chunkHash)` is an operator escape hatch (e.g. a forced re-shard of a stuck chunk) that produces the same `ReplicationTask` shape with `reason: "manual"`.
|
||||
- `candidatePool` is an injected callback (typically wired to the live peer roster from `network-core`) so the manager picks new holders from the actually-connected mesh, not a stale snapshot.
|
||||
- Construction-time validation: every `perClass` value must be `>= 1`, otherwise the constructor throws `NET_CD.REDUNDANCY_TARGET_INVALID`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
ChunkIndex,
|
||||
DEFAULT_REDUNDANCY,
|
||||
RedundancyManager,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const manager = new RedundancyManager({
|
||||
index,
|
||||
redundancy: DEFAULT_REDUNDANCY,
|
||||
candidatePool: () => peerRoster.snapshot(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
|
||||
manager.classify("aaa", "critical");
|
||||
const task = manager.evaluate("aaa");
|
||||
if (task !== null) driveMesh.dispatch(task);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { RedundancyManager } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const manager = new RedundancyManager({
|
||||
index,
|
||||
redundancy: {
|
||||
defaultClass: "shared",
|
||||
perClass: { critical: 7, shared: 4, "user-content": 2, telemetry: 1 },
|
||||
},
|
||||
candidatePool: () => peerRoster.healthy(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
|
||||
bus.on("peer:left", (peerId) => {
|
||||
const tasks = manager.onPeerDeparture(peerId);
|
||||
for (const task of tasks) driveMesh.dispatch(task);
|
||||
});
|
||||
|
||||
const adminTask = manager.requestManualReplication("aaa");
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `RedundancyManager(config)` — `config: { index; redundancy; candidatePool; localPeerId }`. Throws `NET_CD.REDUNDANCY_TARGET_INVALID` if any `perClass` value is `< 1`.
|
||||
- `classify(chunkHash, klass)` — records the class and sets the index's `redundancyTarget`.
|
||||
- `targetFor(chunkHash)` — returns the target copies for the chunk's class (or the default class for unclassified chunks).
|
||||
- `evaluate(chunkHash)` — returns a `ReplicationTask` (`reason: "below-target"`) or `null` if at target.
|
||||
- `onPeerDeparture(holderId)` — removes the peer from the index AND evaluates every affected chunk; returns the bundle of replication tasks.
|
||||
- `requestManualReplication(chunkHash)` — same as `evaluate` but with `reason: "manual"`.
|
||||
- `DEFAULT_REDUNDANCY` — frozen `RedundancyConfig` with the proposal's reference table.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The exclude set inside `evaluateWithReason` excludes both the current holders AND the local peer; the local peer should not be picked as a "new" holder because it's the one running the manager. This avoids self-replication tasks that downstream schedulers would have to filter out.
|
||||
- `recommendedCandidates` is sliced to exactly `need = target - currentHolders.length` so the consumer doesn't have to second-guess which candidates were the manager's first choice. If the candidate pool runs short, the slice truncates silently — the proposal calls this "best-effort" availability.
|
||||
- The `chunkClass` map is in-memory only; persisting class assignments across restarts is the responsibility of the caller (typically by re-applying classifications from a manifest at start-up).
|
||||
@@ -0,0 +1,205 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import { ChunkIndex } from "../index/chunk-index.js";
|
||||
import type { ChunkAdvertisement } from "../types/chunk.types.js";
|
||||
import { DEFAULT_REDUNDANCY, RedundancyManager } from "./redundancy-manager.js";
|
||||
|
||||
const setup = (
|
||||
advertisements: readonly ChunkAdvertisement[],
|
||||
candidatePool: readonly string[]
|
||||
) => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
for (const a of advertisements) index.ingestAdvertisement(a);
|
||||
const manager = new RedundancyManager({
|
||||
index,
|
||||
redundancy: DEFAULT_REDUNDANCY,
|
||||
candidatePool: () => candidatePool,
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
return { index, manager };
|
||||
};
|
||||
|
||||
const adv = (
|
||||
overrides: Partial<ChunkAdvertisement> = {}
|
||||
): ChunkAdvertisement => ({
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("RedundancyManager construction", () => {
|
||||
it("rejects a perClass entry < 1", () => {
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
try {
|
||||
new RedundancyManager({
|
||||
index,
|
||||
redundancy: {
|
||||
defaultClass: "shared",
|
||||
perClass: {
|
||||
critical: 5,
|
||||
shared: 3,
|
||||
"user-content": 2,
|
||||
telemetry: 0,
|
||||
},
|
||||
},
|
||||
candidatePool: () => [],
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.REDUNDANCY_TARGET_INVALID
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("RedundancyManager.classify / targetFor", () => {
|
||||
it("propagates target into the index", () => {
|
||||
const { index, manager } = setup([adv()], []);
|
||||
manager.classify("aaa", "critical");
|
||||
expect(index.requireEntry("aaa").redundancyTarget).toBe(5);
|
||||
expect(manager.targetFor("aaa")).toBe(5);
|
||||
});
|
||||
|
||||
it("falls back to the default class for unclassified chunks", () => {
|
||||
const { manager } = setup([adv()], []);
|
||||
expect(manager.targetFor("aaa")).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RedundancyManager.evaluate", () => {
|
||||
it("returns null when at or above target", () => {
|
||||
const { manager } = setup(
|
||||
[
|
||||
adv({ holderId: "peer-2" }),
|
||||
adv({ holderId: "peer-3" }),
|
||||
adv({ holderId: "peer-4" }),
|
||||
],
|
||||
["peer-5", "peer-6"]
|
||||
);
|
||||
expect(manager.evaluate("aaa")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for unknown chunks", () => {
|
||||
const { manager } = setup([], []);
|
||||
expect(manager.evaluate("zzz")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns a task with reason=below-target when under-replicated", () => {
|
||||
const { manager } = setup(
|
||||
[adv({ holderId: "peer-2" })],
|
||||
["peer-3", "peer-4", "peer-5"]
|
||||
);
|
||||
const task = manager.evaluate("aaa");
|
||||
expect(task).not.toBeNull();
|
||||
expect(task?.reason).toBe("below-target");
|
||||
expect(task?.targetCount).toBe(3);
|
||||
expect(task?.recommendedCandidates.length).toBe(2);
|
||||
});
|
||||
|
||||
it("excludes existing holders and the local peer from candidates", () => {
|
||||
const { manager } = setup(
|
||||
[adv({ holderId: "peer-2" })],
|
||||
["peer-1", "peer-2", "peer-3", "peer-4"]
|
||||
);
|
||||
const task = manager.evaluate("aaa");
|
||||
expect(task?.recommendedCandidates).toEqual(["peer-3", "peer-4"]);
|
||||
});
|
||||
|
||||
it("truncates recommendations when the candidate pool is short", () => {
|
||||
const { manager } = setup([adv({ holderId: "peer-2" })], ["peer-3"]);
|
||||
const task = manager.evaluate("aaa");
|
||||
expect(task?.recommendedCandidates).toEqual(["peer-3"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RedundancyManager.onPeerDeparture", () => {
|
||||
it("removes the peer from the index and emits replication tasks", () => {
|
||||
const { index, manager } = setup(
|
||||
[
|
||||
adv({ holderId: "peer-2" }),
|
||||
adv({
|
||||
chunkHash: "bbb",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 1,
|
||||
holderId: "peer-2",
|
||||
}),
|
||||
adv({
|
||||
chunkHash: "ccc",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 2,
|
||||
holderId: "peer-3",
|
||||
}),
|
||||
],
|
||||
["peer-4", "peer-5"]
|
||||
);
|
||||
const tasks = manager.onPeerDeparture("peer-2");
|
||||
expect(tasks.length).toBe(2);
|
||||
for (const task of tasks) {
|
||||
expect(task.reason).toBe("holder-departed");
|
||||
}
|
||||
expect(index.listHolders("aaa")).toEqual([]);
|
||||
expect(index.listHolders("ccc")).toEqual(["peer-3"]);
|
||||
});
|
||||
|
||||
it("returns an empty array when the peer holds nothing", () => {
|
||||
const { manager } = setup([], []);
|
||||
expect(manager.onPeerDeparture("peer-9")).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips chunks that are still at target after the removal", () => {
|
||||
const { manager } = setup(
|
||||
[
|
||||
adv({ holderId: "peer-2" }),
|
||||
adv({ holderId: "peer-3" }),
|
||||
adv({ holderId: "peer-4" }),
|
||||
adv({ holderId: "peer-5" }),
|
||||
],
|
||||
["peer-6"]
|
||||
);
|
||||
const tasks = manager.onPeerDeparture("peer-5");
|
||||
expect(tasks.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("RedundancyManager.requestManualReplication", () => {
|
||||
it("returns a task with reason=manual", () => {
|
||||
const { manager } = setup(
|
||||
[adv({ holderId: "peer-2" })],
|
||||
["peer-3", "peer-4"]
|
||||
);
|
||||
const task = manager.requestManualReplication("aaa");
|
||||
expect(task?.reason).toBe("manual");
|
||||
});
|
||||
|
||||
it("returns null when at target", () => {
|
||||
const { manager } = setup(
|
||||
[
|
||||
adv({ holderId: "peer-2" }),
|
||||
adv({ holderId: "peer-3" }),
|
||||
adv({ holderId: "peer-4" }),
|
||||
],
|
||||
[]
|
||||
);
|
||||
expect(manager.requestManualReplication("aaa")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("DEFAULT_REDUNDANCY", () => {
|
||||
it("matches the proposal's reference table", () => {
|
||||
expect(DEFAULT_REDUNDANCY.perClass.critical).toBe(5);
|
||||
expect(DEFAULT_REDUNDANCY.perClass.shared).toBe(3);
|
||||
expect(DEFAULT_REDUNDANCY.perClass["user-content"]).toBe(2);
|
||||
expect(DEFAULT_REDUNDANCY.perClass.telemetry).toBe(1);
|
||||
});
|
||||
|
||||
it("is frozen", () => {
|
||||
expect(Object.isFrozen(DEFAULT_REDUNDANCY)).toBe(true);
|
||||
expect(Object.isFrozen(DEFAULT_REDUNDANCY.perClass)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkIndex } from "../index/chunk-index.js";
|
||||
import type {
|
||||
RedundancyClass,
|
||||
RedundancyConfig,
|
||||
ReplicationReason,
|
||||
ReplicationTask,
|
||||
} from "../types/redundancy.types.js";
|
||||
|
||||
export interface RedundancyManagerConfig {
|
||||
readonly index: ChunkIndex;
|
||||
readonly redundancy: RedundancyConfig;
|
||||
readonly candidatePool: () => readonly string[];
|
||||
readonly localPeerId: string;
|
||||
}
|
||||
|
||||
export const DEFAULT_REDUNDANCY: RedundancyConfig = Object.freeze({
|
||||
defaultClass: "shared" as const,
|
||||
perClass: Object.freeze({
|
||||
critical: 5,
|
||||
shared: 3,
|
||||
"user-content": 2,
|
||||
telemetry: 1,
|
||||
}),
|
||||
});
|
||||
|
||||
export class RedundancyManager {
|
||||
private readonly index: ChunkIndex;
|
||||
private readonly redundancy: RedundancyConfig;
|
||||
private readonly candidatePool: () => readonly string[];
|
||||
private readonly localPeerId: string;
|
||||
private readonly chunkClass = new Map<string, RedundancyClass>();
|
||||
|
||||
public constructor(config: RedundancyManagerConfig) {
|
||||
for (const value of Object.values(config.redundancy.perClass)) {
|
||||
if (value < 1) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.REDUNDANCY_TARGET_INVALID,
|
||||
"cd.redundancy.class_target_invalid",
|
||||
{ context: { perClass: config.redundancy.perClass } }
|
||||
);
|
||||
}
|
||||
}
|
||||
this.index = config.index;
|
||||
this.redundancy = config.redundancy;
|
||||
this.candidatePool = config.candidatePool;
|
||||
this.localPeerId = config.localPeerId;
|
||||
}
|
||||
|
||||
public classify(chunkHash: string, klass: RedundancyClass): void {
|
||||
this.chunkClass.set(chunkHash, klass);
|
||||
const target = this.redundancy.perClass[klass];
|
||||
this.index.setRedundancyTarget(chunkHash, target);
|
||||
}
|
||||
|
||||
public targetFor(chunkHash: string): number {
|
||||
const klass =
|
||||
this.chunkClass.get(chunkHash) ?? this.redundancy.defaultClass;
|
||||
return this.redundancy.perClass[klass];
|
||||
}
|
||||
|
||||
public evaluate(chunkHash: string): ReplicationTask | null {
|
||||
return this.evaluateWithReason(chunkHash, "below-target");
|
||||
}
|
||||
|
||||
public onPeerDeparture(holderId: string): readonly ReplicationTask[] {
|
||||
const affected = this.index.removePeer(holderId);
|
||||
const tasks: ReplicationTask[] = [];
|
||||
for (const hash of affected) {
|
||||
const task = this.evaluateWithReason(hash, "holder-departed");
|
||||
if (task !== null) tasks.push(task);
|
||||
}
|
||||
return tasks;
|
||||
}
|
||||
|
||||
public requestManualReplication(chunkHash: string): ReplicationTask | null {
|
||||
return this.evaluateWithReason(chunkHash, "manual");
|
||||
}
|
||||
|
||||
private evaluateWithReason(
|
||||
chunkHash: string,
|
||||
reason: ReplicationReason
|
||||
): ReplicationTask | null {
|
||||
const entry = this.index.getEntry(chunkHash);
|
||||
if (entry === undefined) return null;
|
||||
const target = this.targetFor(chunkHash);
|
||||
const currentHolders = entry.holders;
|
||||
if (currentHolders.length >= target) return null;
|
||||
const exclude = new Set<string>([...currentHolders, this.localPeerId]);
|
||||
const candidates = this.candidatePool().filter((p) => !exclude.has(p));
|
||||
const need = target - currentHolders.length;
|
||||
const recommended = candidates.slice(0, need);
|
||||
return {
|
||||
chunkHash,
|
||||
fileId: entry.fileId,
|
||||
chunkIndex: entry.chunkIndex,
|
||||
currentHolders,
|
||||
targetCount: target,
|
||||
recommendedCandidates: recommended,
|
||||
reason,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
# ChunkRequestRouter
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Decides where to fetch a chunk from. Given a chunk hash, the router asks the `ChunkIndex` who holds it, ranks the holders by injected `LatencyOracle` (typically the per-peer latency table from `network-plugin-webrtc`), and returns a `RouteDecision` carrying the chosen peer. When no mesh peer holds the chunk, the router emits a `"server-fallback"` decision (or throws `NET_CD.NO_HOLDER_AVAILABLE` when no fallback is configured) so the caller can hand off to the central server.
|
||||
|
||||
The router is deliberately read-only with respect to the index — it does not register holders, evict peers, or trigger replication. Its sole job is to answer "who should I ask?" deterministically.
|
||||
|
||||
## Features
|
||||
|
||||
- Latency-aware ranking: sort holders by injected oracle, lowest latency first; ties break by `peerId` lex order so two routers seeing the same input always make the same call.
|
||||
- Self-exclusion: the local peer is never returned as a holder even if the index has it (a router that routed back to itself would deadlock).
|
||||
- Blacklist-aware: an injected `isBlacklisted(peerId)` predicate (typically wired to the `IntegrityVerifier`'s blacklist) filters bad senders out of the candidate set without mutating the index.
|
||||
- Server fallback opt-in: pass `fallback: ServerFallbackFn` to enable; without it, the router throws `NET_CD.CHUNK_NOT_FOUND` when the index has no entry for the chunk at all, or `NET_CD.NO_HOLDER_AVAILABLE` when the entry exists but every holder is filtered out (self / blacklisted / latency-unknown). The two codes let callers tell "we've never heard of this chunk" apart from "we know it but can't reach anyone who has it."
|
||||
- `fetchViaFallback(hash)` is the explicit fallback path, raising `NET_CD.ROUTER_NO_FALLBACK` if no fallback was configured at construction time so the caller can detect the misconfiguration loudly.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
ChunkIndex,
|
||||
ChunkRequestRouter,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
index.ingestAdvertisement({ chunkHash: "aaa", fileId: "f-1", chunkIndex: 0, holderId: "peer-2", advertisedAt: Date.now() });
|
||||
index.ingestAdvertisement({ chunkHash: "aaa", fileId: "f-1", chunkIndex: 0, holderId: "peer-3", advertisedAt: Date.now() });
|
||||
|
||||
const router = new ChunkRequestRouter({
|
||||
index,
|
||||
latency: (p) => ({ "peer-2": 80, "peer-3": 20 } as Record<string, number>)[p],
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
|
||||
const decision = router.route("aaa");
|
||||
console.log(decision.peerId, decision.latencyMs); // "peer-3", 20
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { ChunkRequestRouter } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const router = new ChunkRequestRouter({
|
||||
index,
|
||||
latency: webrtc.latencyForPeer,
|
||||
fallback: (hash) => server.fetchChunk(hash),
|
||||
isBlacklisted: integrityVerifier.isBlacklisted,
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
|
||||
const decision = router.route(hash);
|
||||
const bytes = decision.source === "server-fallback"
|
||||
? await router.fetchViaFallback(hash)
|
||||
: await transport.requestChunk(decision.peerId!, hash);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ChunkRequestRouter(config)` — `config: { index; latency; fallback?; localPeerId; isBlacklisted? }`.
|
||||
- `route(chunkHash)` — returns `RouteDecision`. Raises `NET_CD.CHUNK_NOT_FOUND` when the index has no entry at all and no fallback is configured; raises `NET_CD.NO_HOLDER_AVAILABLE` when the entry exists but no holder is usable and no fallback is configured.
|
||||
- `fetchViaFallback(chunkHash)` — returns `Promise<Uint8Array>`. Raises `NET_CD.ROUTER_NO_FALLBACK` when no fallback is configured.
|
||||
- `rankedHolders(chunkHash)` — pure ranking helper for diagnostics; returns the ranked `{ peerId, latencyMs }` list (empty if the chunk has no entry or no usable holder).
|
||||
- `LatencyOracle` — `(peerId) => number | undefined`; `undefined` means "latency unknown" and that peer is dropped from the candidate list.
|
||||
- `RouteDecision` — `{ chunkHash; source: "mesh" | "server-fallback"; peerId: string | null; latencyMs: number | null }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Holders without latency data are dropped from the candidate list rather than ranked at the bottom. The reasoning: a peer for whom we have no latency reading is one we have not connected to recently, so trying to fetch from it is at best high-latency and at worst a cold-connect dance.
|
||||
- The `peerId.localeCompare` tiebreak keeps the router deterministic even when two peers report identical latency, which is important for property-based tests and replay debugging.
|
||||
- Blacklist filtering happens before latency ranking, not after — a blacklisted peer with the lowest latency would be a footgun if we ranked first then filtered.
|
||||
@@ -0,0 +1,299 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import { ChunkIndex } from "../index/chunk-index.js";
|
||||
import type { ChunkAdvertisement } from "../types/chunk.types.js";
|
||||
import { ChunkRequestRouter } from "./chunk-request-router.js";
|
||||
|
||||
const populated = (
|
||||
advertisements: readonly ChunkAdvertisement[]
|
||||
): ChunkIndex => {
|
||||
const idx = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
for (const a of advertisements) idx.ingestAdvertisement(a);
|
||||
return idx;
|
||||
};
|
||||
|
||||
const basicLatency = (): ((p: string) => number | undefined) => {
|
||||
const map: Record<string, number> = {
|
||||
"peer-2": 80,
|
||||
"peer-3": 20,
|
||||
"peer-4": 50,
|
||||
};
|
||||
return (p) => map[p];
|
||||
};
|
||||
|
||||
describe("ChunkRequestRouter.route", () => {
|
||||
it("picks the lowest-latency holder", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-3",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
const decision = router.route("aaa");
|
||||
expect(decision.source).toBe("mesh");
|
||||
expect(decision.peerId).toBe("peer-3");
|
||||
expect(decision.latencyMs).toBe(20);
|
||||
});
|
||||
|
||||
it("breaks ties on latency by hostId alphabetically", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-z",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-a",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: () => 50,
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
expect(router.route("aaa").peerId).toBe("peer-a");
|
||||
});
|
||||
|
||||
it("excludes the local peer", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-1",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
expect(router.route("aaa").peerId).toBe("peer-2");
|
||||
});
|
||||
|
||||
it("excludes peers without latency data", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-9",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
expect(router.route("aaa").peerId).toBe("peer-2");
|
||||
});
|
||||
|
||||
it("excludes blacklisted peers", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-3",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
isBlacklisted: (p) => p === "peer-3",
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
expect(router.route("aaa").peerId).toBe("peer-2");
|
||||
});
|
||||
|
||||
it("falls back to the server when the index has no entry", () => {
|
||||
const router = new ChunkRequestRouter({
|
||||
index: populated([]),
|
||||
latency: basicLatency(),
|
||||
fallback: () => new Uint8Array([1]),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
const decision = router.route("zzz");
|
||||
expect(decision.source).toBe("server-fallback");
|
||||
expect(decision.peerId).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to the server when entry exists but no holder is usable", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-1",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
fallback: () => new Uint8Array([1]),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
const decision = router.route("aaa");
|
||||
expect(decision.source).toBe("server-fallback");
|
||||
});
|
||||
|
||||
it("throws CHUNK_NOT_FOUND when no entry and no fallback", () => {
|
||||
const router = new ChunkRequestRouter({
|
||||
index: populated([]),
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
try {
|
||||
router.route("zzz");
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.CHUNK_NOT_FOUND
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("throws NO_HOLDER_AVAILABLE when entry exists but no holder usable and no fallback", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-1",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
try {
|
||||
router.route("aaa");
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.NO_HOLDER_AVAILABLE
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkRequestRouter.fetchViaFallback", () => {
|
||||
it("calls the fallback when configured", async () => {
|
||||
const router = new ChunkRequestRouter({
|
||||
index: populated([]),
|
||||
latency: basicLatency(),
|
||||
fallback: async (hash) => new TextEncoder().encode(hash),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
const bytes = await router.fetchViaFallback("aaa");
|
||||
expect(new TextDecoder().decode(bytes)).toBe("aaa");
|
||||
});
|
||||
|
||||
it("supports a synchronous fallback", async () => {
|
||||
const router = new ChunkRequestRouter({
|
||||
index: populated([]),
|
||||
latency: basicLatency(),
|
||||
fallback: () => new Uint8Array([7]),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
const bytes = await router.fetchViaFallback("aaa");
|
||||
expect(bytes[0]).toBe(7);
|
||||
});
|
||||
|
||||
it("throws ROUTER_NO_FALLBACK when not configured", async () => {
|
||||
const router = new ChunkRequestRouter({
|
||||
index: populated([]),
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
await expect(router.fetchViaFallback("aaa")).rejects.toMatchObject({
|
||||
code: NETWORK_CD_CODES.ROUTER_NO_FALLBACK,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkRequestRouter.rankedHolders", () => {
|
||||
it("returns the ranked holder list for a known chunk", () => {
|
||||
const idx = populated([
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-2",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
{
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-3",
|
||||
advertisedAt: 1,
|
||||
},
|
||||
]);
|
||||
const router = new ChunkRequestRouter({
|
||||
index: idx,
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
const ranked = router.rankedHolders("aaa");
|
||||
expect(ranked.map((r) => r.peerId)).toEqual(["peer-3", "peer-2"]);
|
||||
});
|
||||
|
||||
it("returns an empty array for an unknown chunk", () => {
|
||||
const router = new ChunkRequestRouter({
|
||||
index: populated([]),
|
||||
latency: basicLatency(),
|
||||
localPeerId: "peer-1",
|
||||
});
|
||||
expect(router.rankedHolders("zzz")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import type { ChunkIndex } from "../index/chunk-index.js";
|
||||
|
||||
export type LatencyOracle = (peerId: string) => number | undefined;
|
||||
|
||||
export type ServerFallbackFn = (
|
||||
chunkHash: string
|
||||
) => Promise<Uint8Array> | Uint8Array;
|
||||
|
||||
export interface ChunkRequestRouterConfig {
|
||||
readonly index: ChunkIndex;
|
||||
readonly latency: LatencyOracle;
|
||||
readonly fallback?: ServerFallbackFn;
|
||||
readonly localPeerId: string;
|
||||
readonly isBlacklisted?: (peerId: string) => boolean;
|
||||
}
|
||||
|
||||
export type RouteSource = "mesh" | "server-fallback";
|
||||
|
||||
export interface RouteDecision {
|
||||
readonly chunkHash: string;
|
||||
readonly source: RouteSource;
|
||||
readonly peerId: string | null;
|
||||
readonly latencyMs: number | null;
|
||||
}
|
||||
|
||||
export class ChunkRequestRouter {
|
||||
public readonly localPeerId: string;
|
||||
private readonly index: ChunkIndex;
|
||||
private readonly latency: LatencyOracle;
|
||||
private readonly fallback: ServerFallbackFn | undefined;
|
||||
private readonly isBlacklisted: (peerId: string) => boolean;
|
||||
|
||||
public constructor(config: ChunkRequestRouterConfig) {
|
||||
this.index = config.index;
|
||||
this.latency = config.latency;
|
||||
this.fallback = config.fallback;
|
||||
this.localPeerId = config.localPeerId;
|
||||
this.isBlacklisted = config.isBlacklisted ?? (() => false);
|
||||
}
|
||||
|
||||
public route(chunkHash: string): RouteDecision {
|
||||
const entry = this.index.getEntry(chunkHash);
|
||||
if (entry === undefined) {
|
||||
if (this.fallback !== undefined) {
|
||||
return {
|
||||
chunkHash,
|
||||
source: "server-fallback",
|
||||
peerId: null,
|
||||
latencyMs: null,
|
||||
};
|
||||
}
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.CHUNK_NOT_FOUND,
|
||||
"cd.router.chunk_not_found",
|
||||
{ context: { chunkHash } }
|
||||
);
|
||||
}
|
||||
const ranked = this.rankHolders(entry.holders);
|
||||
if (ranked.length > 0) {
|
||||
const top = ranked[0]!;
|
||||
return {
|
||||
chunkHash,
|
||||
source: "mesh",
|
||||
peerId: top.peerId,
|
||||
latencyMs: top.latencyMs,
|
||||
};
|
||||
}
|
||||
if (this.fallback !== undefined) {
|
||||
return {
|
||||
chunkHash,
|
||||
source: "server-fallback",
|
||||
peerId: null,
|
||||
latencyMs: null,
|
||||
};
|
||||
}
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.NO_HOLDER_AVAILABLE,
|
||||
"cd.router.no_holder",
|
||||
{ context: { chunkHash } }
|
||||
);
|
||||
}
|
||||
|
||||
public async fetchViaFallback(chunkHash: string): Promise<Uint8Array> {
|
||||
if (this.fallback === undefined) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.ROUTER_NO_FALLBACK,
|
||||
"cd.router.no_fallback",
|
||||
{ context: { chunkHash } }
|
||||
);
|
||||
}
|
||||
return await Promise.resolve(this.fallback(chunkHash));
|
||||
}
|
||||
|
||||
public rankedHolders(
|
||||
chunkHash: string
|
||||
): readonly { peerId: string; latencyMs: number }[] {
|
||||
const entry = this.index.getEntry(chunkHash);
|
||||
if (entry === undefined) return [];
|
||||
return this.rankHolders(entry.holders);
|
||||
}
|
||||
|
||||
private rankHolders(
|
||||
holders: readonly string[]
|
||||
): { peerId: string; latencyMs: number }[] {
|
||||
const usable = holders
|
||||
.filter((p) => p !== this.localPeerId && !this.isBlacklisted(p))
|
||||
.map((peerId) => ({ peerId, latencyMs: this.latency(peerId) }))
|
||||
.filter(
|
||||
(h): h is { peerId: string; latencyMs: number } =>
|
||||
h.latencyMs !== undefined
|
||||
);
|
||||
usable.sort((a, b) => {
|
||||
if (a.latencyMs !== b.latencyMs) return a.latencyMs - b.latencyMs;
|
||||
return a.peerId.localeCompare(b.peerId);
|
||||
});
|
||||
return usable;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
type ChunkEvictedEvent,
|
||||
ChunkStore,
|
||||
type ChunkStoreAdapter,
|
||||
MemoryChunkStoreAdapter,
|
||||
} from "./chunk-store.js";
|
||||
|
||||
const u8 = (size: number, fill: number): Uint8Array => {
|
||||
const arr = new Uint8Array(size);
|
||||
arr.fill(fill);
|
||||
return arr;
|
||||
};
|
||||
|
||||
describe("ChunkStore LRU eviction order", () => {
|
||||
it("evicts oldest insert first when full", async () => {
|
||||
const events: ChunkEvictedEvent[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 6,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
events.push(payload as ChunkEvictedEvent);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("h0", u8(2, 0));
|
||||
await store.put("h1", u8(2, 1));
|
||||
await store.put("h2", u8(2, 2));
|
||||
// Now full: 6 bytes. Inserting h3 (2 bytes) should evict h0.
|
||||
await store.put("h3", u8(2, 3));
|
||||
expect(events.length).toBe(1);
|
||||
expect(events[0]?.chunkHash).toBe("h0");
|
||||
expect(events[0]?.reason).toBe("lru");
|
||||
expect(store.has("h0")).toBe(false);
|
||||
expect(store.has("h1")).toBe(true);
|
||||
expect(store.has("h2")).toBe(true);
|
||||
expect(store.has("h3")).toBe(true);
|
||||
});
|
||||
|
||||
it("touches an entry on get so it survives the next eviction", async () => {
|
||||
const events: ChunkEvictedEvent[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 6,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
events.push(payload as ChunkEvictedEvent);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("h0", u8(2, 0));
|
||||
await store.put("h1", u8(2, 1));
|
||||
await store.put("h2", u8(2, 2));
|
||||
// Touch h0 so h1 becomes oldest.
|
||||
await store.get("h0");
|
||||
await store.put("h3", u8(2, 3));
|
||||
expect(events[0]?.chunkHash).toBe("h1");
|
||||
expect(store.has("h0")).toBe(true);
|
||||
expect(store.has("h1")).toBe(false);
|
||||
});
|
||||
|
||||
it("evicts multiple oldest entries when one large put requires it", async () => {
|
||||
const evicted: string[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 6,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
evicted.push((payload as ChunkEvictedEvent).chunkHash);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("h0", u8(2, 0));
|
||||
await store.put("h1", u8(2, 1));
|
||||
await store.put("h2", u8(2, 2));
|
||||
// Insert a 6-byte chunk: must evict ALL three smaller chunks.
|
||||
await store.put("big", u8(6, 9));
|
||||
expect(evicted).toEqual(["h0", "h1", "h2"]);
|
||||
expect(store.size()).toBe(1);
|
||||
expect(store.bytes()).toBe(6);
|
||||
});
|
||||
|
||||
it("under mixed traffic preserves access-recency order", async () => {
|
||||
const evicted: string[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 8,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
evicted.push((payload as ChunkEvictedEvent).chunkHash);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("a", u8(2, 1)); // order: a
|
||||
await store.put("b", u8(2, 2)); // order: a, b
|
||||
await store.put("c", u8(2, 3)); // order: a, b, c
|
||||
await store.put("d", u8(2, 4)); // order: a, b, c, d (full)
|
||||
await store.get("a"); // order: b, c, d, a
|
||||
await store.get("c"); // order: b, d, a, c
|
||||
// Insert e — evicts b first (head).
|
||||
await store.put("e", u8(2, 5));
|
||||
expect(evicted).toEqual(["b"]);
|
||||
// Insert f — evicts d next.
|
||||
await store.put("f", u8(2, 6));
|
||||
expect(evicted).toEqual(["b", "d"]);
|
||||
// Remaining live entries: a, c, e, f.
|
||||
expect(store.has("a")).toBe(true);
|
||||
expect(store.has("c")).toBe(true);
|
||||
expect(store.has("e")).toBe(true);
|
||||
expect(store.has("f")).toBe(true);
|
||||
});
|
||||
|
||||
it("manual evict moves the entry out without affecting LRU position of others", async () => {
|
||||
const evicted: string[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 6,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
evicted.push((payload as ChunkEvictedEvent).chunkHash);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("a", u8(2, 1));
|
||||
await store.put("b", u8(2, 2));
|
||||
await store.put("c", u8(2, 3));
|
||||
// Manually evict b (middle).
|
||||
await store.evict("b");
|
||||
evicted.length = 0;
|
||||
// Insert d — fills the freed slot, no further LRU eviction needed.
|
||||
await store.put("d", u8(2, 4));
|
||||
expect(evicted).toEqual([]);
|
||||
expect(store.has("a")).toBe(true);
|
||||
expect(store.has("c")).toBe(true);
|
||||
expect(store.has("d")).toBe(true);
|
||||
});
|
||||
|
||||
it("survives an adapter that throws on delete during LRU eviction", async () => {
|
||||
const inner = new MemoryChunkStoreAdapter();
|
||||
const adapter: ChunkStoreAdapter = {
|
||||
put: (h, b) => inner.put(h, b),
|
||||
get: (h) => inner.get(h),
|
||||
delete: () =>
|
||||
Promise.reject(new Error("simulated adapter failure")),
|
||||
};
|
||||
const store = new ChunkStore({ maxBytes: 4, adapter });
|
||||
await store.put("a", u8(2, 1));
|
||||
await store.put("b", u8(2, 2));
|
||||
// This forces an LRU eviction whose adapter.delete rejects — should
|
||||
// NOT throw out of put().
|
||||
await store.put("c", u8(2, 3));
|
||||
// a evicted from in-memory bookkeeping despite adapter delete failure.
|
||||
expect(store.has("a")).toBe(false);
|
||||
expect(store.has("b")).toBe(true);
|
||||
expect(store.has("c")).toBe(true);
|
||||
// Yield once so the swallowed rejection is processed before test ends.
|
||||
await Promise.resolve();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
# ChunkStore
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`ChunkStore` is the persistence layer of the mesh-CDN. It holds chunk bytes
|
||||
locally so this peer can serve other peers (without re-fetching from origin)
|
||||
and so it can resume large transfers across process restarts. Bookkeeping —
|
||||
LRU ordering, TTL expiry, byte-budget enforcement — is owned by `ChunkStore`
|
||||
itself; only the byte payload is delegated to a pluggable
|
||||
`ChunkStoreAdapter` (the production wiring uses `@suaveplan/drive` for
|
||||
OPFS / IndexedDB / Node filesystem; tests and Worker-less environments use
|
||||
the bundled `MemoryChunkStoreAdapter`). This split guarantees that an
|
||||
in-memory store and a Drive-backed store evict identically — bugs in
|
||||
eviction policy can be reproduced without booting the storage backend.
|
||||
|
||||
## Features
|
||||
|
||||
- Byte-budget LRU eviction: every `put` that would push usage past `maxBytes`
|
||||
evicts oldest-touched chunks first until the new chunk fits.
|
||||
- TTL eviction: chunks older than `maxAgeMs` are evicted on read (in-band) or
|
||||
via explicit `sweepExpired()`.
|
||||
- Pluggable `ChunkStoreAdapter` — `put`, `get`, `delete` only. Sync OR async.
|
||||
- Bus events: `cd:chunk-stored` on every accepted `put`; `cd:chunk-evicted`
|
||||
on every removal with `reason: "ttl" | "lru" | "manual"` so observability
|
||||
consumers can attribute capacity pressure correctly.
|
||||
- Defensive: `put` of a chunk larger than `maxBytes` throws
|
||||
`STORE_QUOTA_EXCEEDED` rather than evicting everything in a futile attempt.
|
||||
- Replace-in-place: putting a hash that already exists updates bookkeeping
|
||||
without double-counting bytes.
|
||||
- `has()` is age-aware — expired entries report absent without firing the
|
||||
eviction event (the next `get` performs the actual eviction).
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { ChunkStore } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const store = new ChunkStore({ maxBytes: 1024 * 1024 });
|
||||
await store.put("h0", new Uint8Array([1, 2, 3]));
|
||||
const bytes = await store.get("h0");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
Wired with the Drive-backed adapter, a 5 MiB byte budget, 10-minute TTL, an
|
||||
injected clock, and a bus that updates the local `ChunkIndex`:
|
||||
|
||||
```ts
|
||||
import {
|
||||
ChunkIndex,
|
||||
ChunkStore,
|
||||
type ChunkStoreAdapter,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const driveAdapter: ChunkStoreAdapter = {
|
||||
put: (h, bytes) => drive.write(`chunks/${h}`, bytes),
|
||||
get: (h) => drive.read(`chunks/${h}`),
|
||||
delete: (h) => drive.delete(`chunks/${h}`),
|
||||
};
|
||||
const index = new ChunkIndex({ defaultRedundancyTarget: 3 });
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 5 * 1024 * 1024,
|
||||
maxAgeMs: 10 * 60 * 1000,
|
||||
adapter: driveAdapter,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-stored") {
|
||||
// index.ingestAdvertisement({...})
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `new ChunkStore(config: ChunkStoreConfig)` — `maxBytes` required; rest
|
||||
optional. Throws `STORE_QUOTA_EXCEEDED` on `maxBytes < 1`.
|
||||
- `put(chunkHash, bytes): Promise<void>` — store chunk; evict oldest until
|
||||
fits; emit `cd:chunk-stored`.
|
||||
- `get(chunkHash): Promise<Uint8Array | undefined>` — return bytes or
|
||||
`undefined`. TTL-expired entries are evicted in-band.
|
||||
- `has(chunkHash): boolean` — true if present and not expired.
|
||||
- `evict(chunkHash, reason?): Promise<boolean>` — manual removal; emits
|
||||
`cd:chunk-evicted`.
|
||||
- `sweepExpired(): Promise<readonly string[]>` — sweep all TTL-expired
|
||||
entries; returns the hashes evicted.
|
||||
- `size(): number` / `bytes(): number` — current bookkeeping.
|
||||
- `MemoryChunkStoreAdapter` — bundled in-memory adapter for tests and Worker-
|
||||
less environments.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The `entries` Map is insertion-ordered by spec. Every `get` that finds a
|
||||
live entry re-inserts it, so the head of the map is always the LRU
|
||||
candidate. `evictOldest` pops the head.
|
||||
- Byte counting is authoritative in-process. Adapter `delete` failures during
|
||||
LRU eviction are swallowed — the in-memory bookkeeping wins so a transient
|
||||
storage error cannot wedge the byte budget.
|
||||
- `STORE_QUOTA_EXCEEDED` is reused for both "single chunk larger than budget"
|
||||
and "internal eviction failed" — both are quota-class failures the caller
|
||||
must respond to.
|
||||
- `cd:chunk-evicted` carries the reason so a single bus subscriber can drive
|
||||
both a "warm cache pressure" gauge (from `lru` events) and a "data freshness
|
||||
policy" trace (from `ttl` events) without needing two subscriptions.
|
||||
@@ -0,0 +1,227 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import {
|
||||
type ChunkEvictedEvent,
|
||||
ChunkStore,
|
||||
type ChunkStoreAdapter,
|
||||
type ChunkStoredEvent,
|
||||
} from "./chunk-store.js";
|
||||
|
||||
const u8 = (...nums: number[]): Uint8Array => Uint8Array.from(nums);
|
||||
|
||||
describe("ChunkStore construction", () => {
|
||||
it("rejects maxBytes < 1", () => {
|
||||
expect(() => new ChunkStore({ maxBytes: 0 })).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects non-finite maxBytes", () => {
|
||||
expect(
|
||||
() => new ChunkStore({ maxBytes: Number.POSITIVE_INFINITY })
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkStore put/get round-trip", () => {
|
||||
it("stores and retrieves a chunk", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
await store.put("h0", u8(1, 2, 3));
|
||||
const out = await store.get("h0");
|
||||
expect(out).toEqual(u8(1, 2, 3));
|
||||
});
|
||||
|
||||
it("returns undefined for an unknown chunk", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
expect(await store.get("zzz")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("emits cd:chunk-stored on put", async () => {
|
||||
const events: ChunkStoredEvent[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
bus: {
|
||||
emit: (_t, p) => events.push(p as ChunkStoredEvent),
|
||||
},
|
||||
});
|
||||
await store.put("h0", u8(1, 2, 3));
|
||||
expect(events.length).toBe(1);
|
||||
expect(events[0]?.bytes).toBe(3);
|
||||
});
|
||||
|
||||
it("replaces an existing chunk in-place", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
await store.put("h0", u8(1, 2, 3));
|
||||
await store.put("h0", u8(4, 5, 6, 7));
|
||||
expect(store.size()).toBe(1);
|
||||
expect(store.bytes()).toBe(4);
|
||||
const out = await store.get("h0");
|
||||
expect(out).toEqual(u8(4, 5, 6, 7));
|
||||
});
|
||||
|
||||
it("rejects a chunk larger than maxBytes", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 4 });
|
||||
try {
|
||||
await store.put("h0", u8(1, 2, 3, 4, 5));
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.STORE_QUOTA_EXCEEDED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses MemoryChunkStoreAdapter when none injected", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
await store.put("h0", u8(1));
|
||||
expect(await store.get("h0")).toEqual(u8(1));
|
||||
});
|
||||
|
||||
it("supports an async adapter", async () => {
|
||||
const inner = new Map<string, Uint8Array>();
|
||||
const adapter: ChunkStoreAdapter = {
|
||||
put: async (h, b) => {
|
||||
inner.set(h, b);
|
||||
},
|
||||
get: async (h) => inner.get(h),
|
||||
delete: async (h) => {
|
||||
inner.delete(h);
|
||||
},
|
||||
};
|
||||
const store = new ChunkStore({ maxBytes: 1024, adapter });
|
||||
await store.put("h0", u8(9));
|
||||
expect(await store.get("h0")).toEqual(u8(9));
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkStore TTL eviction", () => {
|
||||
it("treats reads of expired entries as misses", async () => {
|
||||
let now = 0;
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
maxAgeMs: 10,
|
||||
clock: () => now,
|
||||
});
|
||||
await store.put("h0", u8(1));
|
||||
now = 100;
|
||||
const out = await store.get("h0");
|
||||
expect(out).toBeUndefined();
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("emits cd:chunk-evicted with reason ttl on TTL expiry", async () => {
|
||||
const events: ChunkEvictedEvent[] = [];
|
||||
let now = 0;
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
maxAgeMs: 10,
|
||||
clock: () => now,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
events.push(payload as ChunkEvictedEvent);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("h0", u8(1));
|
||||
now = 100;
|
||||
await store.get("h0");
|
||||
expect(events[0]?.reason).toBe("ttl");
|
||||
});
|
||||
|
||||
it("has() reports false for expired entries without evicting", async () => {
|
||||
let now = 0;
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
maxAgeMs: 10,
|
||||
clock: () => now,
|
||||
});
|
||||
await store.put("h0", u8(1));
|
||||
expect(store.has("h0")).toBe(true);
|
||||
now = 100;
|
||||
expect(store.has("h0")).toBe(false);
|
||||
// Still tracked internally until get() runs.
|
||||
expect(store.size()).toBe(1);
|
||||
});
|
||||
|
||||
it("has() returns false for unknown chunk", () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
expect(store.has("zzz")).toBe(false);
|
||||
});
|
||||
|
||||
it("sweepExpired evicts all TTL-expired entries", async () => {
|
||||
let now = 0;
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
maxAgeMs: 10,
|
||||
clock: () => now,
|
||||
});
|
||||
await store.put("h0", u8(1));
|
||||
await store.put("h1", u8(2));
|
||||
now = 100;
|
||||
const expired = await store.sweepExpired();
|
||||
expect(expired.sort()).toEqual(["h0", "h1"]);
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("sweepExpired only evicts entries past maxAgeMs", async () => {
|
||||
let now = 0;
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
maxAgeMs: 50,
|
||||
clock: () => now,
|
||||
});
|
||||
await store.put("h0", u8(1));
|
||||
now = 60;
|
||||
// h1 is fresh at insert time.
|
||||
await store.put("h1", u8(2));
|
||||
now = 80;
|
||||
// At now=80: h0 stored at 0 -> age 80 (expired); h1 stored at 60 -> age 20 (fresh).
|
||||
const expired = await store.sweepExpired();
|
||||
expect(expired).toEqual(["h0"]);
|
||||
expect(store.has("h1")).toBe(true);
|
||||
});
|
||||
|
||||
it("sweepExpired returns an empty list when no TTL configured", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
await store.put("h0", u8(1));
|
||||
expect(await store.sweepExpired()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkStore manual evict", () => {
|
||||
it("evicts a known chunk and emits cd:chunk-evicted", async () => {
|
||||
const events: ChunkEvictedEvent[] = [];
|
||||
const store = new ChunkStore({
|
||||
maxBytes: 1024,
|
||||
bus: {
|
||||
emit: (topic, payload) => {
|
||||
if (topic === "cd:chunk-evicted") {
|
||||
events.push(payload as ChunkEvictedEvent);
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
await store.put("h0", u8(1, 2));
|
||||
const ok = await store.evict("h0");
|
||||
expect(ok).toBe(true);
|
||||
expect(events[0]?.reason).toBe("manual");
|
||||
expect(events[0]?.bytes).toBe(2);
|
||||
});
|
||||
|
||||
it("returns false when evicting an unknown chunk", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
expect(await store.evict("zzz")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ChunkStore byte budget", () => {
|
||||
it("size and bytes track current state", async () => {
|
||||
const store = new ChunkStore({ maxBytes: 1024 });
|
||||
expect(store.size()).toBe(0);
|
||||
expect(store.bytes()).toBe(0);
|
||||
await store.put("h0", u8(1, 2, 3));
|
||||
expect(store.size()).toBe(1);
|
||||
expect(store.bytes()).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,248 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
|
||||
export interface ChunkStoredEvent {
|
||||
readonly chunkHash: string;
|
||||
readonly bytes: number;
|
||||
readonly storedAt: number;
|
||||
}
|
||||
|
||||
export interface ChunkEvictedEvent {
|
||||
readonly chunkHash: string;
|
||||
readonly bytes: number;
|
||||
readonly reason: "ttl" | "lru" | "manual";
|
||||
readonly evictedAt: number;
|
||||
}
|
||||
|
||||
export interface ChunkStoreBus {
|
||||
emit(topic: "cd:chunk-stored", payload: ChunkStoredEvent): void;
|
||||
emit(topic: "cd:chunk-evicted", payload: ChunkEvictedEvent): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pluggable persistence adapter. The default `ChunkStore` ships an in-memory
|
||||
* implementation; consumers wire `@suaveplan/drive` (OPFS / IndexedDB / Node
|
||||
* filesystem) by passing a `DriveAdapter`-shaped value here. The adapter is
|
||||
* intentionally byte-only: TTL + LRU bookkeeping is owned by `ChunkStore` so
|
||||
* a single eviction policy is enforced regardless of storage backend.
|
||||
*/
|
||||
export interface ChunkStoreAdapter {
|
||||
put(chunkHash: string, bytes: Uint8Array): Promise<void> | void;
|
||||
get(
|
||||
chunkHash: string
|
||||
): Promise<Uint8Array | undefined> | Uint8Array | undefined;
|
||||
delete(chunkHash: string): Promise<void> | void;
|
||||
}
|
||||
|
||||
export class MemoryChunkStoreAdapter implements ChunkStoreAdapter {
|
||||
private readonly bytes = new Map<string, Uint8Array>();
|
||||
|
||||
public put(chunkHash: string, bytes: Uint8Array): void {
|
||||
this.bytes.set(chunkHash, bytes);
|
||||
}
|
||||
|
||||
public get(chunkHash: string): Uint8Array | undefined {
|
||||
return this.bytes.get(chunkHash);
|
||||
}
|
||||
|
||||
public delete(chunkHash: string): void {
|
||||
this.bytes.delete(chunkHash);
|
||||
}
|
||||
}
|
||||
|
||||
export interface ChunkStoreConfig {
|
||||
/**
|
||||
* Hard upper bound on stored bytes. The next `put` that would exceed this
|
||||
* cap evicts oldest-touched chunks first until the new chunk fits. A
|
||||
* single `put` larger than `maxBytes` throws `STORE_QUOTA_EXCEEDED`.
|
||||
*/
|
||||
readonly maxBytes: number;
|
||||
/**
|
||||
* Time-to-live in milliseconds. Reads of chunks older than this are
|
||||
* treated as misses and the entry is evicted in-band. Defaults to
|
||||
* `Number.POSITIVE_INFINITY` (no TTL eviction).
|
||||
*/
|
||||
readonly maxAgeMs?: number;
|
||||
readonly adapter?: ChunkStoreAdapter;
|
||||
readonly clock?: () => number;
|
||||
readonly bus?: ChunkStoreBus;
|
||||
}
|
||||
|
||||
interface Entry {
|
||||
readonly bytes: number;
|
||||
storedAt: number;
|
||||
lastTouched: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* `ChunkStore` persists chunk bytes for the mesh-CDN with byte-budget LRU
|
||||
* eviction and optional TTL. Bookkeeping is held in-memory regardless of the
|
||||
* adapter; only the byte payload itself is delegated to the adapter so a
|
||||
* Drive-backed store and an in-memory store share identical eviction
|
||||
* semantics.
|
||||
*/
|
||||
export class ChunkStore {
|
||||
private readonly adapter: ChunkStoreAdapter;
|
||||
private readonly maxBytes: number;
|
||||
private readonly maxAgeMs: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly bus: ChunkStoreBus | undefined;
|
||||
/**
|
||||
* Insertion-ordered map (Map preserves insertion order). On every `put`
|
||||
* or `get` the entry is deleted-and-re-inserted so the head of the map is
|
||||
* always the LRU candidate.
|
||||
*/
|
||||
private readonly entries = new Map<string, Entry>();
|
||||
private bytesUsed = 0;
|
||||
|
||||
public constructor(config: ChunkStoreConfig) {
|
||||
if (!Number.isFinite(config.maxBytes) || config.maxBytes < 1) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.STORE_QUOTA_EXCEEDED,
|
||||
"cd.store.maxBytes_invalid",
|
||||
{ context: { maxBytes: config.maxBytes } }
|
||||
);
|
||||
}
|
||||
this.maxBytes = Math.floor(config.maxBytes);
|
||||
this.maxAgeMs = config.maxAgeMs ?? Number.POSITIVE_INFINITY;
|
||||
this.adapter = config.adapter ?? new MemoryChunkStoreAdapter();
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
this.bus = config.bus;
|
||||
}
|
||||
|
||||
public async put(chunkHash: string, bytes: Uint8Array): Promise<void> {
|
||||
if (bytes.byteLength > this.maxBytes) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.STORE_QUOTA_EXCEEDED,
|
||||
"cd.store.chunk_exceeds_max_bytes",
|
||||
{
|
||||
context: {
|
||||
chunkHash,
|
||||
bytes: bytes.byteLength,
|
||||
maxBytes: this.maxBytes,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
// If replacing, account for old entry's bytes.
|
||||
const existing = this.entries.get(chunkHash);
|
||||
if (existing !== undefined) {
|
||||
this.bytesUsed -= existing.bytes;
|
||||
this.entries.delete(chunkHash);
|
||||
}
|
||||
// Evict oldest until the new chunk fits. Termination: we already
|
||||
// verified bytes <= maxBytes, and each successful eviction strictly
|
||||
// decreases bytesUsed, so the loop is bounded by the number of
|
||||
// entries currently in the store.
|
||||
while (this.bytesUsed + bytes.byteLength > this.maxBytes) {
|
||||
this.evictOldest("lru");
|
||||
}
|
||||
await this.adapter.put(chunkHash, bytes);
|
||||
const now = this.clock();
|
||||
const entry: Entry = {
|
||||
bytes: bytes.byteLength,
|
||||
storedAt: now,
|
||||
lastTouched: now,
|
||||
};
|
||||
this.entries.set(chunkHash, entry);
|
||||
this.bytesUsed += bytes.byteLength;
|
||||
this.bus?.emit("cd:chunk-stored", {
|
||||
chunkHash,
|
||||
bytes: bytes.byteLength,
|
||||
storedAt: now,
|
||||
});
|
||||
}
|
||||
|
||||
public async get(chunkHash: string): Promise<Uint8Array | undefined> {
|
||||
const entry = this.entries.get(chunkHash);
|
||||
if (entry === undefined) return undefined;
|
||||
const now = this.clock();
|
||||
if (now - entry.storedAt > this.maxAgeMs) {
|
||||
await this.evict(chunkHash, "ttl");
|
||||
return undefined;
|
||||
}
|
||||
// LRU touch: re-insert so it moves to the tail.
|
||||
this.entries.delete(chunkHash);
|
||||
entry.lastTouched = now;
|
||||
this.entries.set(chunkHash, entry);
|
||||
return await this.adapter.get(chunkHash);
|
||||
}
|
||||
|
||||
public has(chunkHash: string): boolean {
|
||||
const entry = this.entries.get(chunkHash);
|
||||
if (entry === undefined) return false;
|
||||
if (this.clock() - entry.storedAt > this.maxAgeMs) {
|
||||
// Expired but not yet swept — report absent to callers; caller
|
||||
// performing `get` will trigger the actual eviction.
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public async evict(
|
||||
chunkHash: string,
|
||||
reason: "ttl" | "lru" | "manual" = "manual"
|
||||
): Promise<boolean> {
|
||||
const entry = this.entries.get(chunkHash);
|
||||
if (entry === undefined) return false;
|
||||
this.entries.delete(chunkHash);
|
||||
this.bytesUsed -= entry.bytes;
|
||||
await this.adapter.delete(chunkHash);
|
||||
this.bus?.emit("cd:chunk-evicted", {
|
||||
chunkHash,
|
||||
bytes: entry.bytes,
|
||||
reason,
|
||||
evictedAt: this.clock(),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
public size(): number {
|
||||
return this.entries.size;
|
||||
}
|
||||
|
||||
public bytes(): number {
|
||||
return this.bytesUsed;
|
||||
}
|
||||
|
||||
public async sweepExpired(): Promise<readonly string[]> {
|
||||
if (this.maxAgeMs === Number.POSITIVE_INFINITY) return [];
|
||||
const now = this.clock();
|
||||
const expired: string[] = [];
|
||||
for (const [chunkHash, entry] of this.entries) {
|
||||
if (now - entry.storedAt > this.maxAgeMs) expired.push(chunkHash);
|
||||
}
|
||||
for (const chunkHash of expired) {
|
||||
await this.evict(chunkHash, "ttl");
|
||||
}
|
||||
return expired;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper: evict the LRU head. Caller MUST guarantee
|
||||
* `entries.size > 0` (the only caller, `put`, only invokes this from
|
||||
* inside a `while (bytesUsed + bytes > maxBytes)` loop after verifying
|
||||
* `bytes <= maxBytes`, which together imply at least one resident entry).
|
||||
*/
|
||||
private evictOldest(reason: "lru" | "manual"): void {
|
||||
const oldest = this.entries.keys().next();
|
||||
const chunkHash = oldest.value as string;
|
||||
const entry = this.entries.get(chunkHash) as Entry;
|
||||
this.entries.delete(chunkHash);
|
||||
this.bytesUsed -= entry.bytes;
|
||||
// Adapter delete is fire-and-forget here — we treat the in-memory
|
||||
// bookkeeping as authoritative. A backend that needs awaited deletes
|
||||
// should call `evict()` directly with the awaited form.
|
||||
const result = this.adapter.delete(chunkHash);
|
||||
if (result instanceof Promise) {
|
||||
result.catch(() => {
|
||||
/* swallow — adapter delete failure does not roll back LRU */
|
||||
});
|
||||
}
|
||||
this.bus?.emit("cd:chunk-evicted", {
|
||||
chunkHash,
|
||||
bytes: entry.bytes,
|
||||
reason,
|
||||
evictedAt: this.clock(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
# BandwidthThrottler
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`BandwidthThrottler` regulates the byte-rate of mesh-CDN chunk dispatches so
|
||||
file traffic cannot starve control-plane messages on the same transport. It
|
||||
maintains two token buckets — one per peer, one global — and forces every
|
||||
admitted dispatch to consume bytes from BOTH before it is allowed onto the
|
||||
wire. Every admitted dispatch is tagged `priority: "bulk"` so the
|
||||
`network-plugin-transport-manager` queue can keep BULK behind CRITICAL
|
||||
without the throttler needing to know the transport's queue topology.
|
||||
Saturation events emit on the bus as `cd:throttled` carrying the precise
|
||||
delay and the responsible bucket so the operator can answer "are we global-
|
||||
limited or peer-limited?" without sampling.
|
||||
|
||||
## Features
|
||||
|
||||
- Per-peer token bucket with configurable rate + burst.
|
||||
- Global token bucket with configurable rate + burst.
|
||||
- Exact wait-time calculation: the throttler computes the precise refill
|
||||
duration required to satisfy both buckets and sleeps that long once,
|
||||
rather than spinning.
|
||||
- `admit()` blocks until both buckets are satisfied and returns a
|
||||
`BulkDispatch` envelope; `tryAdmit()` is the non-blocking probe form.
|
||||
- `cd:throttled` event with `reason: "per-peer" | "global"` so dashboards can
|
||||
attribute saturation correctly.
|
||||
- Defensive: a single chunk request larger than either burst capacity throws
|
||||
`THROTTLE_RATE_INVALID` rather than blocking forever.
|
||||
- Injectable `clock` and `sleep` so tests run in zero wall-clock with full
|
||||
determinism over the wait math.
|
||||
- `availableForPeer(peerId)` and `globalAvailable()` inspectors for capacity
|
||||
planning + admission-control hints.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { BandwidthThrottler } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const throttler = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1024 * 1024, // 1 MiB/s per peer
|
||||
globalBytesPerSec: 8 * 1024 * 1024, // 8 MiB/s aggregate
|
||||
});
|
||||
const dispatch = await throttler.admit("peer-a", "h0", 64 * 1024);
|
||||
// dispatch.priority === "bulk"
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
Deterministic clock + sleep for property tests, with bus-driven observability
|
||||
and a tighter burst configuration:
|
||||
|
||||
```ts
|
||||
import {
|
||||
BandwidthThrottler,
|
||||
type ThrottledEvent,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
let now = 0;
|
||||
const sleeps: number[] = [];
|
||||
const throttled: ThrottledEvent[] = [];
|
||||
const throttler = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
perPeerBurstBytes: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
globalBurstBytes: 5000,
|
||||
clock: () => now,
|
||||
sleep: async (ms) => {
|
||||
sleeps.push(ms);
|
||||
now += ms;
|
||||
},
|
||||
bus: { emit: (_t, payload) => throttled.push(payload) },
|
||||
});
|
||||
await throttler.admit("peer-a", "h0", 800);
|
||||
await throttler.admit("peer-a", "h1", 800);
|
||||
// Second call empties the per-peer bucket: a `cd:throttled` event fires with
|
||||
// reason "per-peer" and the recorded sleep durations sum to the exact refill.
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `new BandwidthThrottler(config: BandwidthThrottlerConfig)` — `perPeerBytesPerSec`
|
||||
and `globalBytesPerSec` required (both > 0). Burst capacities default to
|
||||
one second of rate.
|
||||
- `admit(peerId, chunkHash, bytes): Promise<BulkDispatch>` — blocking admit.
|
||||
- `tryAdmit(peerId, chunkHash, bytes): BulkDispatch | undefined` — probe.
|
||||
- `availableForPeer(peerId): number` — min of peer bucket and global bucket.
|
||||
- `globalAvailable(): number` — current global bucket tokens.
|
||||
- `BulkDispatch` — `{ priority: "bulk", peerId, chunkHash, bytes, admittedAt,
|
||||
waitedMs }`.
|
||||
- `ThrottledEvent` — `{ peerId, chunkHash, bytes, delayMs, reason, observedAt }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The wait-time calculation is exact: the throttler computes
|
||||
`peerShort / perPeerRate` and `globalShort / globalRate` independently and
|
||||
sleeps the larger of the two. After the sleep, both buckets are guaranteed
|
||||
refilled past the threshold, so a single iteration usually suffices.
|
||||
- The loop exists for defensive symmetry: if the injected sleep returns early
|
||||
(e.g. a test stub that does not advance the clock the full amount), the
|
||||
next iteration recomputes the residual short and sleeps again.
|
||||
- `reason` reflects which bucket dictated the wait — the bucket whose
|
||||
required wait was longer wins. Equal waits attribute to `per-peer` (the
|
||||
more actionable signal for an operator).
|
||||
- Per-peer buckets are created lazily on first `admit`/`tryAdmit` so a peer
|
||||
that never sends consumes no bookkeeping memory.
|
||||
- `BulkDispatch.waitedMs` is computed from the clock delta when available
|
||||
(covers real-clock callers); when the injected clock does not advance
|
||||
during the sleep (e.g. deterministic test wiring), the throttler falls
|
||||
back to the sum of requested sleep durations so the value is still
|
||||
meaningful.
|
||||
@@ -0,0 +1,325 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
import {
|
||||
BandwidthThrottler,
|
||||
type ThrottledEvent,
|
||||
} from "./bandwidth-throttler.js";
|
||||
|
||||
interface DeterministicClock {
|
||||
now: number;
|
||||
sleeps: number[];
|
||||
}
|
||||
|
||||
const setupClock = (): {
|
||||
clock: () => number;
|
||||
sleep: (ms: number) => Promise<void>;
|
||||
state: DeterministicClock;
|
||||
} => {
|
||||
const state: DeterministicClock = { now: 0, sleeps: [] };
|
||||
return {
|
||||
clock: () => state.now,
|
||||
sleep: async (ms: number) => {
|
||||
state.sleeps.push(ms);
|
||||
state.now += ms;
|
||||
},
|
||||
state,
|
||||
};
|
||||
};
|
||||
|
||||
describe("BandwidthThrottler construction", () => {
|
||||
it("rejects perPeerBytesPerSec <= 0", () => {
|
||||
expect(
|
||||
() =>
|
||||
new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 0,
|
||||
globalBytesPerSec: 100,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects non-finite perPeerBytesPerSec", () => {
|
||||
expect(
|
||||
() =>
|
||||
new BandwidthThrottler({
|
||||
perPeerBytesPerSec: Number.POSITIVE_INFINITY,
|
||||
globalBytesPerSec: 100,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects globalBytesPerSec <= 0", () => {
|
||||
expect(
|
||||
() =>
|
||||
new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: 0,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects non-finite globalBytesPerSec", () => {
|
||||
expect(
|
||||
() =>
|
||||
new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: Number.NaN,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects burst <= 0", () => {
|
||||
expect(
|
||||
() =>
|
||||
new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
perPeerBurstBytes: 0,
|
||||
globalBytesPerSec: 100,
|
||||
})
|
||||
).toThrow(NetworkCdError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BandwidthThrottler.admit — happy path", () => {
|
||||
it("tags admitted dispatches with priority bulk", async () => {
|
||||
const { clock, sleep } = setupClock();
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
clock,
|
||||
sleep,
|
||||
});
|
||||
const dispatch = await t.admit("peer-a", "h0", 100);
|
||||
expect(dispatch.priority).toBe("bulk");
|
||||
expect(dispatch.peerId).toBe("peer-a");
|
||||
expect(dispatch.chunkHash).toBe("h0");
|
||||
expect(dispatch.bytes).toBe(100);
|
||||
expect(dispatch.waitedMs).toBe(0);
|
||||
});
|
||||
|
||||
it("admits multiple small dispatches without sleeping", async () => {
|
||||
const { clock, sleep, state } = setupClock();
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
clock,
|
||||
sleep,
|
||||
});
|
||||
await t.admit("peer-a", "h0", 100);
|
||||
await t.admit("peer-a", "h1", 100);
|
||||
await t.admit("peer-a", "h2", 100);
|
||||
expect(state.sleeps.length).toBe(0);
|
||||
});
|
||||
|
||||
it("rejects bytes that exceed per-peer burst", async () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
perPeerBurstBytes: 100,
|
||||
globalBytesPerSec: 10000,
|
||||
});
|
||||
try {
|
||||
await t.admit("peer-a", "h0", 200);
|
||||
throw new Error("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(NetworkCdError);
|
||||
expect((err as NetworkCdError).code).toBe(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects bytes that exceed global burst", async () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 10000,
|
||||
globalBytesPerSec: 100,
|
||||
globalBurstBytes: 100,
|
||||
});
|
||||
await expect(t.admit("peer-a", "h0", 200)).rejects.toBeInstanceOf(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects negative bytes", async () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: 100,
|
||||
});
|
||||
await expect(t.admit("peer-a", "h0", -1)).rejects.toBeInstanceOf(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects non-finite bytes", async () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: 100,
|
||||
});
|
||||
await expect(
|
||||
t.admit("peer-a", "h0", Number.POSITIVE_INFINITY)
|
||||
).rejects.toBeInstanceOf(NetworkCdError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BandwidthThrottler — per-peer rate enforcement", () => {
|
||||
it("blocks once the per-peer bucket is exhausted", async () => {
|
||||
const { clock, sleep, state } = setupClock();
|
||||
const events: ThrottledEvent[] = [];
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
perPeerBurstBytes: 1000,
|
||||
globalBytesPerSec: 100000,
|
||||
clock,
|
||||
sleep,
|
||||
bus: { emit: (_topic, payload) => events.push(payload) },
|
||||
});
|
||||
await t.admit("peer-a", "h0", 1000); // empties peer bucket
|
||||
await t.admit("peer-a", "h1", 500); // must wait 500ms (500 / 1000 * 1000)
|
||||
expect(state.sleeps.length).toBeGreaterThanOrEqual(1);
|
||||
expect(events[0]?.reason).toBe("per-peer");
|
||||
expect(events[0]?.delayMs).toBeCloseTo(500, 5);
|
||||
});
|
||||
|
||||
it("isolates per-peer buckets between peers", async () => {
|
||||
const { clock, sleep, state } = setupClock();
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
perPeerBurstBytes: 1000,
|
||||
globalBytesPerSec: 100000,
|
||||
clock,
|
||||
sleep,
|
||||
});
|
||||
await t.admit("peer-a", "h0", 1000); // empties peer-a bucket
|
||||
await t.admit("peer-b", "h1", 1000); // peer-b has full bucket
|
||||
expect(state.sleeps.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BandwidthThrottler — global rate enforcement", () => {
|
||||
it("blocks once the global bucket is exhausted", async () => {
|
||||
const { clock, sleep } = setupClock();
|
||||
const events: ThrottledEvent[] = [];
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100000,
|
||||
globalBytesPerSec: 1000,
|
||||
globalBurstBytes: 1000,
|
||||
clock,
|
||||
sleep,
|
||||
bus: { emit: (_topic, payload) => events.push(payload) },
|
||||
});
|
||||
await t.admit("peer-a", "h0", 1000); // empties global
|
||||
await t.admit("peer-b", "h1", 500); // peer-b's bucket is full but global empty
|
||||
expect(events[0]?.reason).toBe("global");
|
||||
});
|
||||
});
|
||||
|
||||
describe("BandwidthThrottler.tryAdmit", () => {
|
||||
it("returns a tagged BulkDispatch when capacity exists", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
});
|
||||
const out = t.tryAdmit("peer-a", "h0", 500);
|
||||
expect(out?.priority).toBe("bulk");
|
||||
});
|
||||
|
||||
it("returns undefined when peer bucket is too small", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
perPeerBurstBytes: 1000,
|
||||
globalBytesPerSec: 100000,
|
||||
});
|
||||
t.tryAdmit("peer-a", "h0", 1000);
|
||||
expect(t.tryAdmit("peer-a", "h1", 500)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined when global bucket is too small", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100000,
|
||||
globalBytesPerSec: 1000,
|
||||
globalBurstBytes: 1000,
|
||||
});
|
||||
t.tryAdmit("peer-a", "h0", 1000);
|
||||
expect(t.tryAdmit("peer-b", "h1", 500)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects negative bytes", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: 100,
|
||||
});
|
||||
expect(() => t.tryAdmit("peer-a", "h0", -5)).toThrow(NetworkCdError);
|
||||
});
|
||||
|
||||
it("rejects non-finite bytes", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: 100,
|
||||
});
|
||||
expect(() => t.tryAdmit("peer-a", "h0", Number.NaN)).toThrow(
|
||||
NetworkCdError
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BandwidthThrottler — inspectors", () => {
|
||||
it("availableForPeer returns the min of peer + global tokens", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
perPeerBurstBytes: 1000,
|
||||
globalBytesPerSec: 500,
|
||||
globalBurstBytes: 500,
|
||||
});
|
||||
expect(t.availableForPeer("peer-a")).toBe(500);
|
||||
});
|
||||
|
||||
it("globalAvailable returns refilled global tokens", () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100,
|
||||
globalBytesPerSec: 100,
|
||||
globalBurstBytes: 100,
|
||||
});
|
||||
expect(t.globalAvailable()).toBe(100);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BandwidthThrottler — defaults", () => {
|
||||
it("uses Date.now when no clock injected", async () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
});
|
||||
const before = Date.now();
|
||||
const dispatch = await t.admit("peer-a", "h0", 100);
|
||||
expect(dispatch.admittedAt).toBeGreaterThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("uses default sleep that resolves on small ms when zero", async () => {
|
||||
// Default sleep with 0ms still resolves promptly.
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000000,
|
||||
globalBytesPerSec: 1000000,
|
||||
});
|
||||
const out = await t.admit("peer-a", "h0", 1);
|
||||
expect(out.priority).toBe("bulk");
|
||||
});
|
||||
|
||||
it("default clock advances during admit so waitedMs is non-negative", async () => {
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 1000,
|
||||
globalBytesPerSec: 5000,
|
||||
});
|
||||
const dispatch = await t.admit("peer-a", "h0", 1);
|
||||
expect(dispatch.waitedMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it("uses default sleep wired with non-zero ms when throttled", async () => {
|
||||
// Force a real (small) delay through the default sleep path.
|
||||
const t = new BandwidthThrottler({
|
||||
perPeerBytesPerSec: 100000,
|
||||
perPeerBurstBytes: 100,
|
||||
globalBytesPerSec: 1000000,
|
||||
});
|
||||
await t.admit("peer-a", "h0", 100); // empties peer bucket
|
||||
const dispatch = await t.admit("peer-a", "h1", 1);
|
||||
expect(dispatch.priority).toBe("bulk");
|
||||
}, 30000);
|
||||
});
|
||||
@@ -0,0 +1,280 @@
|
||||
import { NETWORK_CD_CODES, NetworkCdError } from "../error-codes.js";
|
||||
|
||||
export interface ThrottledEvent {
|
||||
readonly peerId: string;
|
||||
readonly chunkHash: string;
|
||||
readonly bytes: number;
|
||||
readonly delayMs: number;
|
||||
readonly reason: "per-peer" | "global";
|
||||
readonly observedAt: number;
|
||||
}
|
||||
|
||||
export interface ThrottleBus {
|
||||
emit(topic: "cd:throttled", payload: ThrottledEvent): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tagged dispatch envelope handed back to callers after the throttler admits
|
||||
* a chunk. Transport adapters consume this verbatim — `priority: "bulk"`
|
||||
* marks the message so the transport-manager queues it behind any control-
|
||||
* plane traffic.
|
||||
*/
|
||||
export interface BulkDispatch {
|
||||
readonly priority: "bulk";
|
||||
readonly peerId: string;
|
||||
readonly chunkHash: string;
|
||||
readonly bytes: number;
|
||||
readonly admittedAt: number;
|
||||
readonly waitedMs: number;
|
||||
}
|
||||
|
||||
export interface BandwidthThrottlerConfig {
|
||||
/**
|
||||
* Per-peer steady-state byte rate (bytes per second).
|
||||
*/
|
||||
readonly perPeerBytesPerSec: number;
|
||||
/**
|
||||
* Per-peer burst capacity in bytes. Defaults to `perPeerBytesPerSec`
|
||||
* (one second of headroom).
|
||||
*/
|
||||
readonly perPeerBurstBytes?: number;
|
||||
/**
|
||||
* Global steady-state byte rate (bytes per second).
|
||||
*/
|
||||
readonly globalBytesPerSec: number;
|
||||
/**
|
||||
* Global burst capacity in bytes. Defaults to `globalBytesPerSec`.
|
||||
*/
|
||||
readonly globalBurstBytes?: number;
|
||||
readonly clock?: () => number;
|
||||
readonly bus?: ThrottleBus;
|
||||
/**
|
||||
* Sleep primitive — defaults to a `setTimeout`-based promise. Tests inject
|
||||
* a deterministic sleep that resolves immediately and records the requested
|
||||
* delay, so we can assert "asked for X ms wait" without burning wall-clock.
|
||||
*/
|
||||
readonly sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
interface Bucket {
|
||||
tokens: number;
|
||||
lastRefill: number;
|
||||
}
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
// setTimeout exists in both Node and browser globals; ms is always
|
||||
// strictly positive when defaultSleep is reached because the throttler
|
||||
// only sleeps when at least one bucket reports a positive shortage.
|
||||
setTimeout(resolve, ms);
|
||||
});
|
||||
|
||||
/**
|
||||
* `BandwidthThrottler` admits chunk dispatches under a per-peer + global
|
||||
* token-bucket regime and tags every admitted dispatch with
|
||||
* `priority: "bulk"` so the transport manager queues mesh-CDN traffic behind
|
||||
* the control-plane priority class. When the buckets cannot satisfy a
|
||||
* request, the throttler computes the precise wait time required to refill
|
||||
* either bucket, sleeps, then re-admits — emitting a `cd:throttled` event so
|
||||
* observability can attribute saturation to the peer or to the global cap.
|
||||
*/
|
||||
export class BandwidthThrottler {
|
||||
private readonly perPeerRate: number;
|
||||
private readonly perPeerBurst: number;
|
||||
private readonly globalRate: number;
|
||||
private readonly globalBurst: number;
|
||||
private readonly clock: () => number;
|
||||
private readonly bus: ThrottleBus | undefined;
|
||||
private readonly sleep: (ms: number) => Promise<void>;
|
||||
private readonly perPeer = new Map<string, Bucket>();
|
||||
private readonly global: Bucket;
|
||||
|
||||
public constructor(config: BandwidthThrottlerConfig) {
|
||||
if (
|
||||
!Number.isFinite(config.perPeerBytesPerSec) ||
|
||||
config.perPeerBytesPerSec <= 0
|
||||
) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID,
|
||||
"cd.throttle.perPeer_invalid",
|
||||
{
|
||||
context: { perPeerBytesPerSec: config.perPeerBytesPerSec },
|
||||
}
|
||||
);
|
||||
}
|
||||
if (
|
||||
!Number.isFinite(config.globalBytesPerSec) ||
|
||||
config.globalBytesPerSec <= 0
|
||||
) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID,
|
||||
"cd.throttle.global_invalid",
|
||||
{
|
||||
context: { globalBytesPerSec: config.globalBytesPerSec },
|
||||
}
|
||||
);
|
||||
}
|
||||
this.perPeerRate = config.perPeerBytesPerSec;
|
||||
this.perPeerBurst = config.perPeerBurstBytes ?? this.perPeerRate;
|
||||
this.globalRate = config.globalBytesPerSec;
|
||||
this.globalBurst = config.globalBurstBytes ?? this.globalRate;
|
||||
if (this.perPeerBurst <= 0 || this.globalBurst <= 0) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID,
|
||||
"cd.throttle.burst_invalid",
|
||||
{
|
||||
context: {
|
||||
perPeerBurst: this.perPeerBurst,
|
||||
globalBurst: this.globalBurst,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
this.clock = config.clock ?? (() => Date.now());
|
||||
this.bus = config.bus;
|
||||
this.sleep = config.sleep ?? defaultSleep;
|
||||
this.global = { tokens: this.globalBurst, lastRefill: this.clock() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reserve `bytes` for a chunk dispatch to `peerId`. Resolves to the
|
||||
* tagged `BulkDispatch` envelope. Sleeps internally if either bucket is
|
||||
* insufficient. A single-chunk request larger than either bucket's burst
|
||||
* capacity throws `THROTTLE_RATE_INVALID` rather than blocking forever.
|
||||
*/
|
||||
public async admit(
|
||||
peerId: string,
|
||||
chunkHash: string,
|
||||
bytes: number
|
||||
): Promise<BulkDispatch> {
|
||||
if (bytes < 0 || !Number.isFinite(bytes)) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID,
|
||||
"cd.throttle.bytes_invalid",
|
||||
{ context: { bytes } }
|
||||
);
|
||||
}
|
||||
if (bytes > this.perPeerBurst || bytes > this.globalBurst) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID,
|
||||
"cd.throttle.chunk_exceeds_burst",
|
||||
{
|
||||
context: {
|
||||
bytes,
|
||||
perPeerBurst: this.perPeerBurst,
|
||||
globalBurst: this.globalBurst,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
const start = this.clock();
|
||||
let totalWaitedMs = 0;
|
||||
const peerBucket = this.getPeerBucket(peerId);
|
||||
// Loop until both buckets simultaneously have enough tokens. We refill
|
||||
// before each check so a single sleep generally suffices.
|
||||
while (true) {
|
||||
this.refill(peerBucket, this.perPeerRate, this.perPeerBurst);
|
||||
this.refill(this.global, this.globalRate, this.globalBurst);
|
||||
const peerShort = bytes - peerBucket.tokens;
|
||||
const globalShort = bytes - this.global.tokens;
|
||||
if (peerShort <= 0 && globalShort <= 0) {
|
||||
peerBucket.tokens -= bytes;
|
||||
this.global.tokens -= bytes;
|
||||
break;
|
||||
}
|
||||
const peerWaitMs =
|
||||
peerShort > 0 ? (peerShort / this.perPeerRate) * 1000 : 0;
|
||||
const globalWaitMs =
|
||||
globalShort > 0 ? (globalShort / this.globalRate) * 1000 : 0;
|
||||
const reason: "per-peer" | "global" =
|
||||
peerWaitMs >= globalWaitMs ? "per-peer" : "global";
|
||||
const delayMs = Math.max(peerWaitMs, globalWaitMs);
|
||||
this.bus?.emit("cd:throttled", {
|
||||
peerId,
|
||||
chunkHash,
|
||||
bytes,
|
||||
delayMs,
|
||||
reason,
|
||||
observedAt: this.clock(),
|
||||
});
|
||||
totalWaitedMs += delayMs;
|
||||
await this.sleep(delayMs);
|
||||
}
|
||||
const admittedAt = this.clock();
|
||||
return {
|
||||
priority: "bulk",
|
||||
peerId,
|
||||
chunkHash,
|
||||
bytes,
|
||||
admittedAt,
|
||||
waitedMs: Math.max(0, admittedAt - start) || totalWaitedMs,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-blocking probe — admits the dispatch only if both buckets have
|
||||
* sufficient tokens right now. Returns `undefined` on insufficient
|
||||
* capacity. No `cd:throttled` emission.
|
||||
*/
|
||||
public tryAdmit(
|
||||
peerId: string,
|
||||
chunkHash: string,
|
||||
bytes: number
|
||||
): BulkDispatch | undefined {
|
||||
if (bytes < 0 || !Number.isFinite(bytes)) {
|
||||
throw new NetworkCdError(
|
||||
NETWORK_CD_CODES.THROTTLE_RATE_INVALID,
|
||||
"cd.throttle.bytes_invalid",
|
||||
{ context: { bytes } }
|
||||
);
|
||||
}
|
||||
const peerBucket = this.getPeerBucket(peerId);
|
||||
this.refill(peerBucket, this.perPeerRate, this.perPeerBurst);
|
||||
this.refill(this.global, this.globalRate, this.globalBurst);
|
||||
if (peerBucket.tokens < bytes || this.global.tokens < bytes) {
|
||||
return undefined;
|
||||
}
|
||||
peerBucket.tokens -= bytes;
|
||||
this.global.tokens -= bytes;
|
||||
const now = this.clock();
|
||||
return {
|
||||
priority: "bulk",
|
||||
peerId,
|
||||
chunkHash,
|
||||
bytes,
|
||||
admittedAt: now,
|
||||
waitedMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
public availableForPeer(peerId: string): number {
|
||||
const bucket = this.getPeerBucket(peerId);
|
||||
this.refill(bucket, this.perPeerRate, this.perPeerBurst);
|
||||
return Math.min(bucket.tokens, this.globalAvailable());
|
||||
}
|
||||
|
||||
public globalAvailable(): number {
|
||||
this.refill(this.global, this.globalRate, this.globalBurst);
|
||||
return this.global.tokens;
|
||||
}
|
||||
|
||||
private getPeerBucket(peerId: string): Bucket {
|
||||
const existing = this.perPeer.get(peerId);
|
||||
if (existing !== undefined) return existing;
|
||||
const fresh: Bucket = {
|
||||
tokens: this.perPeerBurst,
|
||||
lastRefill: this.clock(),
|
||||
};
|
||||
this.perPeer.set(peerId, fresh);
|
||||
return fresh;
|
||||
}
|
||||
|
||||
private refill(bucket: Bucket, rate: number, burst: number): void {
|
||||
const now = this.clock();
|
||||
const elapsedMs = Math.max(0, now - bucket.lastRefill);
|
||||
if (elapsedMs === 0) return;
|
||||
const refillTokens = (elapsedMs / 1000) * rate;
|
||||
bucket.tokens = Math.min(burst, bucket.tokens + refillTokens);
|
||||
bucket.lastRefill = now;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
# chunk.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Wire-stable shapes for every record that flows through the content-delivery subsystem: the per-file `ChunkManifest` that announces a packed file (small enough to broadcast on the `NetworkBus`), the per-chunk `ChunkAdvertisement` that a peer emits when it stores a new chunk, the merged `ChunkIndexEntry` the local `ChunkIndex` keeps, the `ChunkRequest` envelope a peer sends when it needs bytes, and the `ChunkBytes` envelope the responding peer ships back with its identity attached so `IntegrityVerifier` can blacklist on hash mismatch.
|
||||
|
||||
These types are the public contract every subsystem in `@suaveplan/network-plugin-content-delivery` agrees on, so they live in a shared `types/` directory rather than next to any one consumer.
|
||||
|
||||
## Features
|
||||
|
||||
- `ChunkManifest.hashAlgorithm` is a free-form string (e.g. `"sha-256"` / `"blake3"`) so the verifier can swap algorithms without a wire-format change.
|
||||
- `ChunkAdvertisement.advertisedAt` is the holder's local timestamp and is the gossip merge key (later advertisements replace earlier ones for the same `(chunkHash, holderId)`).
|
||||
- `ChunkIndexEntry.holders` is `readonly string[]` so the index can hand out shared references without defensive copies — callers that mutate the array are doing the wrong thing.
|
||||
- `ChunkRequest.requesterId` lets the responder credit / debit per-peer bandwidth budgets and rate-limit greedy peers.
|
||||
- `ChunkBytes.senderId` is required so the integrity verifier can blacklist the sender without an out-of-band correlation.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type {
|
||||
ChunkManifest,
|
||||
ChunkAdvertisement,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const manifest: ChunkManifest = {
|
||||
fileId: "f-1",
|
||||
totalChunks: 4,
|
||||
chunkHashes: ["aaa", "bbb", "ccc", "ddd"],
|
||||
hashAlgorithm: "sha-256",
|
||||
packerVersion: "1.0.0",
|
||||
};
|
||||
|
||||
const adv: ChunkAdvertisement = {
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
holderId: "peer-1",
|
||||
advertisedAt: Date.now(),
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type {
|
||||
ChunkBytes,
|
||||
ChunkRequest,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const request: ChunkRequest = {
|
||||
chunkHash: "aaa",
|
||||
fileId: "f-1",
|
||||
chunkIndex: 0,
|
||||
requesterId: "peer-2",
|
||||
};
|
||||
|
||||
const response: ChunkBytes = {
|
||||
chunkHash: "aaa",
|
||||
bytes: new Uint8Array([1, 2, 3]),
|
||||
senderId: "peer-1",
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `ChunkManifest` — `{ fileId; totalChunks; chunkHashes; hashAlgorithm; packerVersion }`.
|
||||
- `ChunkAdvertisement` — `{ chunkHash; fileId; chunkIndex; holderId; advertisedAt }`.
|
||||
- `ChunkIndexEntry` — `{ chunkHash; fileId; chunkIndex; holders; redundancyTarget; lastUpdated }`.
|
||||
- `ChunkRequest` — `{ chunkHash; fileId; chunkIndex; requesterId }`.
|
||||
- `ChunkBytes` — `{ chunkHash; bytes; senderId }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- All shapes are pure data — no methods, no behavior. Logic lives in the consuming subsystems (`ChunkIndex`, `RedundancyManager`, etc.).
|
||||
- `chunkIndex` (the array index inside the manifest) is duplicated on advertisements and entries for fast bookkeeping; consumers should always treat `chunkHash` as the identity key and `chunkIndex` as a positional hint that may go stale across re-packs.
|
||||
@@ -0,0 +1,37 @@
|
||||
export interface ChunkManifest {
|
||||
readonly fileId: string;
|
||||
readonly totalChunks: number;
|
||||
readonly chunkHashes: readonly string[];
|
||||
readonly hashAlgorithm: string;
|
||||
readonly packerVersion: string;
|
||||
}
|
||||
|
||||
export interface ChunkAdvertisement {
|
||||
readonly chunkHash: string;
|
||||
readonly fileId: string;
|
||||
readonly chunkIndex: number;
|
||||
readonly holderId: string;
|
||||
readonly advertisedAt: number;
|
||||
}
|
||||
|
||||
export interface ChunkIndexEntry {
|
||||
readonly chunkHash: string;
|
||||
readonly fileId: string;
|
||||
readonly chunkIndex: number;
|
||||
readonly holders: readonly string[];
|
||||
readonly redundancyTarget: number;
|
||||
readonly lastUpdated: number;
|
||||
}
|
||||
|
||||
export interface ChunkRequest {
|
||||
readonly chunkHash: string;
|
||||
readonly fileId: string;
|
||||
readonly chunkIndex: number;
|
||||
readonly requesterId: string;
|
||||
}
|
||||
|
||||
export interface ChunkBytes {
|
||||
readonly chunkHash: string;
|
||||
readonly bytes: Uint8Array;
|
||||
readonly senderId: string;
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
# redundancy.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Captures the inputs and outputs of `RedundancyManager`. The four-class taxonomy (`critical` / `shared` / `user-content` / `telemetry`) maps directly to the redundancy table in the proposal: critical executable chunks need 5 copies to survive multiple peer departures, shared assets need 3 copies, user-generated content needs 2, and telemetry needs only 1 because the server has the canonical record.
|
||||
|
||||
The `ReplicationTask` shape is what the redundancy manager *emits* when it determines a chunk needs more copies — it does NOT execute the replication itself. The actual fetch/place is the responsibility of `drive-mesh` (per the boundary documented in the proposal). This separation keeps redundancy policy and chunk-fetch scheduling decoupled.
|
||||
|
||||
## Features
|
||||
|
||||
- `RedundancyConfig.defaultClass` lets callers route untagged chunks (e.g. ad-hoc gossip blobs) to a sensible default rather than failing.
|
||||
- `RedundancyConfig.perClass` is a `Readonly<Record<...>>` so the configuration is frozen at construction time; mutating it is a runtime contract violation.
|
||||
- `ReplicationTask.recommendedCandidates` is a hint, not a directive — the consuming `drive-mesh` scheduler may reorder based on local latency / bandwidth state.
|
||||
- `ReplicationReason` discriminates `"below-target"` (initial under-replication on first advertise), `"holder-departed"` (peer left the mesh and we lost a copy), and `"manual"` (caller-driven re-replication).
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type {
|
||||
RedundancyConfig,
|
||||
RedundancyClass,
|
||||
} from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
const config: RedundancyConfig = {
|
||||
defaultClass: "shared",
|
||||
perClass: { critical: 5, shared: 3, "user-content": 2, telemetry: 1 },
|
||||
};
|
||||
|
||||
const klass: RedundancyClass = "critical";
|
||||
console.log(config.perClass[klass]); // 5
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { ReplicationTask } from "@suaveplan/network-plugin-content-delivery";
|
||||
|
||||
function applyTask(task: ReplicationTask): void {
|
||||
if (task.reason === "holder-departed") {
|
||||
metrics.increment("cd.replication.churn");
|
||||
}
|
||||
for (const candidate of task.recommendedCandidates) {
|
||||
driveMesh.scheduleFetch(task.chunkHash, candidate);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `RedundancyClass` — `"critical" | "shared" | "user-content" | "telemetry"`.
|
||||
- `RedundancyConfig` — `{ defaultClass; perClass: Readonly<Record<RedundancyClass, number>> }`.
|
||||
- `ReplicationTask` — `{ chunkHash; fileId; chunkIndex; currentHolders; targetCount; recommendedCandidates; reason }`.
|
||||
- `ReplicationReason` — `"below-target" | "holder-departed" | "manual"`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The four-class enum is intentionally closed: adding a new class is a code change, not a configuration change. This keeps the redundancy table auditable in source.
|
||||
- `targetCount` is materialised on the task (rather than re-derived from `RedundancyConfig`) so a downstream consumer can act on the task without re-resolving configuration; this matters when the manager and consumer run in different processes.
|
||||
@@ -0,0 +1,22 @@
|
||||
export type RedundancyClass =
|
||||
| "critical"
|
||||
| "shared"
|
||||
| "user-content"
|
||||
| "telemetry";
|
||||
|
||||
export interface RedundancyConfig {
|
||||
readonly defaultClass: RedundancyClass;
|
||||
readonly perClass: Readonly<Record<RedundancyClass, number>>;
|
||||
}
|
||||
|
||||
export interface ReplicationTask {
|
||||
readonly chunkHash: string;
|
||||
readonly fileId: string;
|
||||
readonly chunkIndex: number;
|
||||
readonly currentHolders: readonly string[];
|
||||
readonly targetCount: number;
|
||||
readonly recommendedCandidates: readonly string[];
|
||||
readonly reason: ReplicationReason;
|
||||
}
|
||||
|
||||
export type ReplicationReason = "below-target" | "holder-departed" | "manual";
|
||||
@@ -0,0 +1,22 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import * as cd from "../src/index.js";
|
||||
|
||||
describe("@suaveplan/network-plugin-content-delivery public surface", () => {
|
||||
it("exports every documented class", () => {
|
||||
expect(cd.ChunkIndex).toBeDefined();
|
||||
expect(cd.ChunkRequestRouter).toBeDefined();
|
||||
expect(cd.RedundancyManager).toBeDefined();
|
||||
expect(cd.IntegrityVerifier).toBeDefined();
|
||||
expect(cd.ContentDeliveryPlugin).toBeDefined();
|
||||
});
|
||||
|
||||
it("exports the error registry", () => {
|
||||
expect(cd.NETWORK_CD_CODES).toBeDefined();
|
||||
expect(cd.NetworkCdError).toBeDefined();
|
||||
});
|
||||
|
||||
it("exports helper values", () => {
|
||||
expect(cd.DEFAULT_REDUNDANCY).toBeDefined();
|
||||
expect(typeof cd.defaultEncode).toBe("function");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,7 @@
|
||||
import { webcrypto } from "node:crypto";
|
||||
|
||||
type WritableGlobal = { crypto?: Crypto };
|
||||
const g = globalThis as WritableGlobal;
|
||||
if (g.crypto === undefined) {
|
||||
g.crypto = webcrypto as unknown as Crypto;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": ".",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts", "tests/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"],
|
||||
"references": [{ "path": "./tsconfig.lib.json" }]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/network-plugin-content-delivery",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/network",
|
||||
"@suaveplan/types",
|
||||
"tslib",
|
||||
"zod",
|
||||
"@opentelemetry/api",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: [
|
||||
"src/__benchmarks__/**",
|
||||
"benchmarks/**",
|
||||
"e2e/**",
|
||||
"dist/**",
|
||||
"node_modules/**",
|
||||
],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/**/*.schemas.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 5000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user