Initial Commit
This commit is contained in:
135
packages/persistence/storage/storage-http/README.md
Normal file
135
packages/persistence/storage/storage-http/README.md
Normal file
@@ -0,0 +1,135 @@
|
||||
# @suaveplan/storage-http - Universal HTTP/CDN AbstractStorage adapter for @suaveplan/storage: GET/HEAD/Range reads against any baseUrl + key origin, ETag cache, AbortSignal everywhere, strict key sanitisation, native Web Streams, transient-status retry with Retry-After, optional writes via pluggable HttpUploader, full OTel surface.
|
||||
|
||||
Universal HTTP/CDN-backed `AbstractStorage` adapter for `@suaveplan/storage`. Reads via `GET`/`HEAD`/`Range` against any `<baseUrl><key>` origin, with optional writes through a pluggable `HttpUploader`. Threads `AbortSignal` everywhere, retries transient outcomes honouring `Retry-After`, supports conditional reads via an ETag cache, and ships native `ReadableStream` streaming — all in a single tier-pure package that runs unmodified under Node 18+, Bun, Deno, browsers, and Cloudflare Workers.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [constants](./src/constants/constants.md) — Centralises the tunable defaults used across `@suaveplan/storage-http`: key-length cap, retry attempts/backoff, ETag-ca…
|
||||
- [constants](./src/constants/constants.md) — Centralises the tunable defaults used across `@suaveplan/storage-http`: key-length cap, retry attempts/backoff, ETag-ca…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — Declares the canonical `STORAGE_HTTP` error namespace used by every module in this package. The codes are produced via…
|
||||
- [error-codes](./src/error-codes/error-codes.md) — Declares the canonical `STORAGE_HTTP` error namespace used by every module in this package. The codes are produced via…
|
||||
- [etag-cache](./src/etag-cache/etag-cache.md) — In-memory LRU keyed by sanitised storage key, valued by `{ etag, body, lastModified }`. Backs `HttpStorage`'s condition…
|
||||
- [etag-cache](./src/etag-cache/etag-cache.md) — In-memory LRU keyed by sanitised storage key, valued by `{ etag, body, lastModified }`. Backs `HttpStorage`'s condition…
|
||||
- [fetch-adapter](./src/fetch-adapter/fetch-adapter.md) — Wraps `globalThis.fetch` with the contract `HttpStorage` actually consumes. Translates raw network rejections (DNS fail…
|
||||
- [fetch-adapter](./src/fetch-adapter/fetch-adapter.md) — Wraps `globalThis.fetch` with the contract `HttpStorage` actually consumes. Translates raw network rejections (DNS fail…
|
||||
- [http-storage](./src/http-storage/http-storage.md) — `HttpStorage` is the universal `AbstractStorage` adapter that maps every storage operation onto an HTTP request against…
|
||||
- [http-storage.helpers](./src/http-storage/http-storage.helpers.md) — Internal helpers used by `HttpStorage`. Splits boundary concerns (option parsing, URL composition, header resolution, a…
|
||||
- [http-storage](./src/http-storage/http-storage.md) — `HttpStorage` is the universal `AbstractStorage` adapter that maps every storage operation onto an HTTP request against…
|
||||
- [http-storage.ops](./src/http-storage/http-storage.ops.md) — Reusable read-side operations used by `HttpStorage`: full-body `GET` with ETag-cache support, `HEAD` for metadata, `Ran…
|
||||
- [http-storage.streams](./src/http-storage/http-storage.streams.md) — Builds the `ReadableStream` and `WritableStream` instances `HttpStorage` returns from `createReadStream` / `createWrite…
|
||||
- [key-sanitizer](./src/key-sanitizer/key-sanitizer.md) — Validates and normalises caller-supplied storage keys before they reach any `fetch` call, and provides the default per-…
|
||||
- [key-sanitizer](./src/key-sanitizer/key-sanitizer.md) — Validates and normalises caller-supplied storage keys before they reach any `fetch` call, and provides the default per-…
|
||||
- [otel](./src/otel/otel.md) — Owns the OpenTelemetry instrument set for `@suaveplan/storage-http`: counters (`reads_total`, `writes_total`, `deletes_…
|
||||
- [otel](./src/otel/otel.md) — Owns the OpenTelemetry instrument set for `@suaveplan/storage-http`: counters (`reads_total`, `writes_total`, `deletes_…
|
||||
- [range](./src/range/range.md) — Encodes HTTP `Range` semantics for `HttpStorage.getPartial`. Builds the request header from `(offset, length)` and inte…
|
||||
- [range](./src/range/range.md) — Encodes HTTP `Range` semantics for `HttpStorage.getPartial`. Builds the request header from `(offset, length)` and inte…
|
||||
- [registry](./src/registry/registry.md) — Module-augments `@suaveplan/storage/factory`'s `StorageProviderRegistry` and `StorageProviderConfig` to add the `http`…
|
||||
- [registry](./src/registry/registry.md) — Module-augments `@suaveplan/storage/factory`'s `StorageProviderRegistry` and `StorageProviderConfig` to add the `http`…
|
||||
- [retry-helper](./src/retry-helper/retry-helper.md) — Coordinates retry logic for HTTP requests issued by `HttpStorage`. Distinguishes transient outcomes (HTTP `408`/`425`/`…
|
||||
- [retry-helper](./src/retry-helper/retry-helper.md) — Coordinates retry logic for HTTP requests issued by `HttpStorage`. Distinguishes transient outcomes (HTTP `408`/`425`/`…
|
||||
- [uploader](./src/uploader/uploader.md) — Defines the pluggable write strategy for `HttpStorage` and ships the only generic uploader implementation: `presignedPu…
|
||||
- [uploader](./src/uploader/uploader.md) — Defines the pluggable write strategy for `HttpStorage` and ships the only generic uploader implementation: `presignedPu…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/storage-http
|
||||
```
|
||||
|
||||
## Quick start
|
||||
|
||||
```ts
|
||||
import { HttpStorage } from "@suaveplan/storage-http";
|
||||
|
||||
const storage = new HttpStorage({
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
});
|
||||
|
||||
const bytes = await storage.get("a/b/test.doc");
|
||||
```
|
||||
|
||||
Writes require an injected `HttpUploader`:
|
||||
|
||||
```ts
|
||||
import { HttpStorage, presignedPutUploader } from "@suaveplan/storage-http";
|
||||
|
||||
const storage = new HttpStorage({
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
uploader: presignedPutUploader({
|
||||
signer: async (key) => ({ url: await signMyProvider("PUT", key, 60) }),
|
||||
}),
|
||||
});
|
||||
|
||||
await storage.set("a/b/test.doc", new Uint8Array([1, 2, 3]));
|
||||
```
|
||||
|
||||
`list`, `clear`, and `getSize` always throw `STORAGE_HTTP.READ_ONLY` because HTTP origins do not expose a uniform directory protocol.
|
||||
|
||||
## Composition recipe — disk-backed ETag cache
|
||||
|
||||
For cross-process cache survival, layer `@suaveplan/storage-file`'s `NamespacedStorage` underneath:
|
||||
|
||||
```ts
|
||||
import { HttpStorage } from "@suaveplan/storage-http";
|
||||
import { NodeFsStorage } from "@suaveplan/storage-file";
|
||||
|
||||
const local = new NodeFsStorage({ rootDir: "/var/cache/cdn-mirror" });
|
||||
const remote = new HttpStorage({
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
etagCache: { capacity: 1024 },
|
||||
});
|
||||
|
||||
async function readCached(key: string): Promise<Uint8Array | undefined> {
|
||||
const cached = await local.get(key);
|
||||
if (cached !== undefined) return cached;
|
||||
const remoteBody = await remote.get(key);
|
||||
if (remoteBody !== undefined) await local.set(key, remoteBody);
|
||||
return remoteBody;
|
||||
}
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- [error-codes](./src/error-codes.md) — `STORAGE_HTTP_CODES` registry + `StorageHttpError`.
|
||||
- [constants](./src/constants/constants.md) — tunable defaults.
|
||||
- [key-sanitizer](./src/key-sanitizer/key-sanitizer.md) — strict key validation + per-segment encoder.
|
||||
- [fetch-adapter](./src/fetch-adapter/fetch-adapter.md) — `defaultFetch()` wrapping `globalThis.fetch`.
|
||||
- [range](./src/range/range.md) — `Range` header + 206/200/416 disambiguation.
|
||||
- [etag-cache](./src/etag-cache/etag-cache.md) — in-memory LRU for conditional GETs.
|
||||
- [uploader](./src/uploader/uploader.md) — `HttpUploader` contract + `presignedPutUploader` helper.
|
||||
- [retry-helper](./src/retry-helper/retry-helper.md) — transient classifier + `Retry-After` honour.
|
||||
- [otel](./src/otel/otel.md) — meter, tracer, counters, histograms.
|
||||
- [http-storage](./src/http-storage/http-storage.md) — the `HttpStorage` class itself.
|
||||
- [registry](./src/registry/registry.md) — module-augmentation slot for `createStorage({ type: "http", ... })`.
|
||||
|
||||
## Features
|
||||
|
||||
- **Universal purity** — no `node:*` or DOM-only imports. Runs unmodified everywhere Node 18+ runs.
|
||||
- **Full `AbstractStorage` surface** — `get`, `set`, `delete`, `exists`, `getMetadata`, `getWithMetadata`, `getItemSize`, `getPartial`, `createReadStream`, `createWriteStream`, `setPartial`. `list`/`clear`/`getSize` throw `READ_ONLY`.
|
||||
- **Native streaming** — `createReadStream` returns `Response.body` directly (no client-side buffering).
|
||||
- **Range reads with fallback metric** — 200-when-Range-was-asked is sliced client-side and counted under `storage_http.range_fallback_total`.
|
||||
- **ETag-cached conditional reads** — `If-None-Match` on hit, `304` returns cached body.
|
||||
- **AbortSignal everywhere** — pre-aborted signal short-circuits before fetch; mid-flight abort propagates to `fetch.init.signal`.
|
||||
- **Retry on transient outcomes** — `408`/`425`/`429`/`5xx` and network rejection retried with exponential backoff or `Retry-After`.
|
||||
- **Pluggable writes** — `HttpUploader` slot keeps the core read-only-by-default while permitting any write strategy.
|
||||
- **OTel built-in** — duration / IO bytes histograms with `status_class` attribute; counters per outcome; spans per call.
|
||||
- **100 / 100 / 100 / 100 Istanbul coverage** — see `bunx turbo test:coverage --filter=@suaveplan/storage-http`.
|
||||
|
||||
## Capabilities matrix
|
||||
|
||||
```ts
|
||||
{
|
||||
partialReads: true,
|
||||
partialWrites: false,
|
||||
nativeStreams: true,
|
||||
atomicWrites: false,
|
||||
contentAddressable: false,
|
||||
}
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT — see `LICENSE` at the repo root.
|
||||
|
||||
166
packages/persistence/storage/storage-http/package.json
Normal file
166
packages/persistence/storage/storage-http/package.json
Normal file
@@ -0,0 +1,166 @@
|
||||
{
|
||||
"name": "@suaveplan/storage-http",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Universal HTTP/CDN AbstractStorage adapter for @suaveplan/storage: GET/HEAD/Range reads against any baseUrl + key origin, ETag cache, AbortSignal everywhere, strict key sanitisation, native Web Streams, transient-status retry with Retry-After, optional writes via pluggable HttpUploader, full OTel surface.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./http-storage": {
|
||||
"import": {
|
||||
"types": "./dist/http-storage/index.d.ts",
|
||||
"default": "./dist/http-storage/http-storage.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/http-storage/index.d.ts",
|
||||
"default": "./dist/http-storage/http-storage.js"
|
||||
}
|
||||
},
|
||||
"./key-sanitizer": {
|
||||
"import": {
|
||||
"types": "./dist/key-sanitizer/index.d.ts",
|
||||
"default": "./dist/key-sanitizer/key-sanitizer.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/key-sanitizer/index.d.ts",
|
||||
"default": "./dist/key-sanitizer/key-sanitizer.js"
|
||||
}
|
||||
},
|
||||
"./fetch-adapter": {
|
||||
"import": {
|
||||
"types": "./dist/fetch-adapter/index.d.ts",
|
||||
"default": "./dist/fetch-adapter/fetch-adapter.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/fetch-adapter/index.d.ts",
|
||||
"default": "./dist/fetch-adapter/fetch-adapter.js"
|
||||
}
|
||||
},
|
||||
"./range": {
|
||||
"import": {
|
||||
"types": "./dist/range/index.d.ts",
|
||||
"default": "./dist/range/range.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/range/index.d.ts",
|
||||
"default": "./dist/range/range.js"
|
||||
}
|
||||
},
|
||||
"./etag-cache": {
|
||||
"import": {
|
||||
"types": "./dist/etag-cache/index.d.ts",
|
||||
"default": "./dist/etag-cache/etag-cache.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/etag-cache/index.d.ts",
|
||||
"default": "./dist/etag-cache/etag-cache.js"
|
||||
}
|
||||
},
|
||||
"./uploader": {
|
||||
"import": {
|
||||
"types": "./dist/uploader/index.d.ts",
|
||||
"default": "./dist/uploader/uploader.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/uploader/index.d.ts",
|
||||
"default": "./dist/uploader/uploader.js"
|
||||
}
|
||||
},
|
||||
"./retry-helper": {
|
||||
"import": {
|
||||
"types": "./dist/retry-helper/index.d.ts",
|
||||
"default": "./dist/retry-helper/retry-helper.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/retry-helper/index.d.ts",
|
||||
"default": "./dist/retry-helper/retry-helper.js"
|
||||
}
|
||||
},
|
||||
"./otel": {
|
||||
"import": {
|
||||
"types": "./dist/otel/index.d.ts",
|
||||
"default": "./dist/otel/otel.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/otel/index.d.ts",
|
||||
"default": "./dist/otel/otel.js"
|
||||
}
|
||||
},
|
||||
"./registry": {
|
||||
"import": {
|
||||
"types": "./dist/registry/index.d.ts",
|
||||
"default": "./dist/registry/registry.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/registry/index.d.ts",
|
||||
"default": "./dist/registry/registry.js"
|
||||
}
|
||||
},
|
||||
"./types": {
|
||||
"import": {
|
||||
"types": "./dist/types/index.d.ts",
|
||||
"default": "./dist/types/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/types/index.d.ts",
|
||||
"default": "./dist/types/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/retry": "workspace:^",
|
||||
"@suaveplan/storage": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"fast-check": "catalog:testing",
|
||||
"mitata": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
# constants
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Centralises the tunable defaults used across `@suaveplan/storage-http`: key-length cap, retry attempts/backoff, ETag-cache capacity, and the transient-status set. Every other module reads from this file rather than embedding magic numbers, so operators have one knob to turn when tuning behaviour for a specific origin.
|
||||
|
||||
## Features
|
||||
|
||||
- `MAX_KEY_LENGTH = 2048` — conservative intersection of major CDN path limits.
|
||||
- `DEFAULT_RETRY_MAX_ATTEMPTS = 3`, `DEFAULT_RETRY_BASE_DELAY_MS = 100`, `DEFAULT_RETRY_MAX_DELAY_MS = 30_000` — used when callers don't override.
|
||||
- `RETRY_AFTER_MAX_DELAY_MS = 60_000` — hard cap on honouring `Retry-After` to protect against malicious origins that try to stall callers indefinitely.
|
||||
- `DEFAULT_ETAG_CACHE_CAPACITY = 256` — LRU bound for the in-memory ETag cache.
|
||||
- `TRANSIENT_HTTP_STATUSES` — frozen `Set` of non-`5xx` transient codes (`408`, `425`, `429`).
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { MAX_KEY_LENGTH } from "@suaveplan/storage-http";
|
||||
|
||||
if (key.length > MAX_KEY_LENGTH) {
|
||||
throw new Error(`Key too long: ${key.length}`);
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import {
|
||||
DEFAULT_RETRY_MAX_ATTEMPTS,
|
||||
RETRY_AFTER_MAX_DELAY_MS,
|
||||
TRANSIENT_HTTP_STATUSES,
|
||||
} from "@suaveplan/storage-http";
|
||||
|
||||
function classify(status: number): "transient" | "fatal" {
|
||||
if (status >= 500 || TRANSIENT_HTTP_STATUSES.has(status)) return "transient";
|
||||
return "fatal";
|
||||
}
|
||||
|
||||
const retryAfter = Math.min(parsedSeconds * 1000, RETRY_AFTER_MAX_DELAY_MS);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `MAX_KEY_LENGTH: number` — character limit for sanitised keys.
|
||||
- `DEFAULT_RETRY_MAX_ATTEMPTS: number` — default retry attempts.
|
||||
- `DEFAULT_RETRY_BASE_DELAY_MS: number` — initial exponential-backoff delay.
|
||||
- `DEFAULT_RETRY_MAX_DELAY_MS: number` — exponential-backoff ceiling.
|
||||
- `RETRY_AFTER_MAX_DELAY_MS: number` — server-supplied `Retry-After` ceiling.
|
||||
- `DEFAULT_ETAG_CACHE_CAPACITY: number` — LRU capacity for ETag cache.
|
||||
- `TRANSIENT_HTTP_STATUSES: ReadonlySet<number>` — non-`5xx` transient codes.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `RETRY_AFTER_MAX_DELAY_MS` is **not** the maximum backoff for the adapter as a whole — `DEFAULT_RETRY_MAX_DELAY_MS` controls exponential-backoff. The 60s ceiling specifically caps the value honoured from the server's `Retry-After` header.
|
||||
- `5xx` is intentionally NOT in the `TRANSIENT_HTTP_STATUSES` set — the retry classifier checks the range directly (`status >= 500 && status <= 599`) so callers reading the set don't have to enumerate 100 entries.
|
||||
- `MAX_KEY_LENGTH` is per-key, not per-segment; segment URL-encoding may inflate the wire length beyond this number, but that is the encoder's concern, not the sanitiser's.
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
DEFAULT_ETAG_CACHE_CAPACITY,
|
||||
DEFAULT_RETRY_BASE_DELAY_MS,
|
||||
DEFAULT_RETRY_MAX_ATTEMPTS,
|
||||
DEFAULT_RETRY_MAX_DELAY_MS,
|
||||
MAX_KEY_LENGTH,
|
||||
RETRY_AFTER_MAX_DELAY_MS,
|
||||
TRANSIENT_HTTP_STATUSES,
|
||||
} from "./constants.js";
|
||||
|
||||
describe("constants", () => {
|
||||
it("expose the documented defaults", { timeout: 5000 }, () => {
|
||||
expect(MAX_KEY_LENGTH).toBe(2048);
|
||||
expect(DEFAULT_RETRY_MAX_ATTEMPTS).toBeGreaterThan(0);
|
||||
expect(DEFAULT_RETRY_BASE_DELAY_MS).toBeGreaterThan(0);
|
||||
expect(DEFAULT_RETRY_MAX_DELAY_MS).toBeGreaterThan(
|
||||
DEFAULT_RETRY_BASE_DELAY_MS
|
||||
);
|
||||
expect(RETRY_AFTER_MAX_DELAY_MS).toBe(60_000);
|
||||
expect(DEFAULT_ETAG_CACHE_CAPACITY).toBeGreaterThan(0);
|
||||
expect(TRANSIENT_HTTP_STATUSES.has(408)).toBe(true);
|
||||
expect(TRANSIENT_HTTP_STATUSES.has(429)).toBe(true);
|
||||
expect(TRANSIENT_HTTP_STATUSES.has(500)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Maximum permitted key length in characters. Keeps URL composition within
|
||||
* documented limits for every major CDN — Cloudflare/CloudFront/Fastly all
|
||||
* impose path limits around 2k, BunnyCDN around 1.5k; this is the conservative
|
||||
* intersection. Rejected with `STORAGE_HTTP.KEY_TOO_LONG`.
|
||||
*/
|
||||
export const MAX_KEY_LENGTH = 2048;
|
||||
|
||||
/** Default retry attempts when `options.retry.maxAttempts` is unset. */
|
||||
export const DEFAULT_RETRY_MAX_ATTEMPTS = 3;
|
||||
/** Default exponential-backoff initial delay. */
|
||||
export const DEFAULT_RETRY_BASE_DELAY_MS = 100;
|
||||
/** Default exponential-backoff cap. */
|
||||
export const DEFAULT_RETRY_MAX_DELAY_MS = 30_000;
|
||||
/** Hard cap on `Retry-After` honouring — protects against malicious origins. */
|
||||
export const RETRY_AFTER_MAX_DELAY_MS = 60_000;
|
||||
/** Default ETag cache LRU capacity. */
|
||||
export const DEFAULT_ETAG_CACHE_CAPACITY = 256;
|
||||
|
||||
/**
|
||||
* HTTP status codes the adapter classifies as transient. `5xx` is handled as a
|
||||
* range check separately; `408`, `425`, `429` are the documented non-5xx
|
||||
* transient set.
|
||||
*/
|
||||
export const TRANSIENT_HTTP_STATUSES: ReadonlySet<number> = new Set([
|
||||
408, 425, 429,
|
||||
]);
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
DEFAULT_ETAG_CACHE_CAPACITY,
|
||||
DEFAULT_RETRY_BASE_DELAY_MS,
|
||||
DEFAULT_RETRY_MAX_ATTEMPTS,
|
||||
DEFAULT_RETRY_MAX_DELAY_MS,
|
||||
MAX_KEY_LENGTH,
|
||||
RETRY_AFTER_MAX_DELAY_MS,
|
||||
TRANSIENT_HTTP_STATUSES,
|
||||
} from "./constants.js";
|
||||
@@ -0,0 +1,71 @@
|
||||
# Error codes
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the canonical `STORAGE_HTTP` error namespace used by every module in this package. The codes are produced via `@suaveplan/error.defineCodes` so the resulting string values are guaranteed-unique within the workspace (`"STORAGE_HTTP.NOT_FOUND"`, `"STORAGE_HTTP.UPSTREAM_ERROR"`, …). The companion `StorageHttpError` class extends `SuaveplanError` and adds three diagnostic discriminants — `key`, `httpStatus`, and `url` — so consumers can correlate failures with the offending request without parsing message text.
|
||||
|
||||
## Features
|
||||
|
||||
- 10 stable error codes covering every HTTP failure mode the adapter raises.
|
||||
- `StorageHttpError` subclass with `key`, `httpStatus`, and `url` carry-fields.
|
||||
- Cause-chain preservation via `SuaveplanError`'s `cause` option.
|
||||
- Type-safe `StorageHttpErrorCode` discriminant inferred from the `defineCodes` tuple.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { STORAGE_HTTP_CODES, StorageHttpError } from "@suaveplan/storage-http";
|
||||
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND,
|
||||
"Object not found",
|
||||
{ key: "a/b.bin", httpStatus: 404, url: "https://cdn.example.com/a/b.bin" },
|
||||
);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { STORAGE_HTTP_CODES, StorageHttpError } from "@suaveplan/storage-http";
|
||||
|
||||
try {
|
||||
await storage.get("missing");
|
||||
} catch (err) {
|
||||
if (err instanceof StorageHttpError && err.code === STORAGE_HTTP_CODES.UPSTREAM_ERROR) {
|
||||
// Retry classifier may have already exhausted attempts — surface to the operator.
|
||||
operatorLog.error({ url: err.url, status: err.httpStatus }, "Upstream failed");
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `STORAGE_HTTP_CODES`
|
||||
|
||||
Frozen record produced by `defineCodes("STORAGE_HTTP", [...] as const)`. The string values are namespaced — `STORAGE_HTTP_CODES.NOT_FOUND === "STORAGE_HTTP.NOT_FOUND"`.
|
||||
|
||||
Codes: `NOT_FOUND`, `UNAUTHORIZED`, `FORBIDDEN`, `NETWORK_ERROR`, `INVALID_KEY`, `KEY_TOO_LONG`, `INVALID_OPTIONS`, `READ_ONLY`, `ABORTED`, `UPSTREAM_ERROR`.
|
||||
|
||||
### `StorageHttpErrorCode`
|
||||
|
||||
Union of the namespaced code string values.
|
||||
|
||||
### `class StorageHttpError extends SuaveplanError<StorageHttpErrorCode>`
|
||||
|
||||
- `constructor(code, message, options?)` — `options` accepts `cause`, `details`, `key`, `httpStatus`, `url`.
|
||||
- `key: string | undefined` — sanitised key on which the failure occurred.
|
||||
- `httpStatus: number | undefined` — HTTP status when the failure originated from an upstream response.
|
||||
- `url: string | undefined` — fully-resolved URL the adapter attempted.
|
||||
- `name = "StorageHttpError"`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `STORAGE_HTTP.NOT_FOUND` is raised only when `getPartial` requests a range past EOF (`416 Range Not Satisfiable`). A plain `GET 404` returns `undefined` per `AbstractStorage` contract; it does **not** throw.
|
||||
- `STORAGE_HTTP.UPSTREAM_ERROR` is the bucket for non-transient `4xx`/`5xx` after retry exhaustion. It is distinct from `NETWORK_ERROR` (fetch reject — DNS/TLS/abort-without-signal) so retry classifiers and operator dashboards can tell them apart.
|
||||
- `STORAGE_HTTP.READ_ONLY` covers `set`, `delete`, `list`, `clear`, and `createWriteStream` when no `HttpUploader` is wired (or when the op is unconditionally unsupported, e.g. `list`).
|
||||
- The `key` carry-field stores the **sanitised** key, never the raw caller input — preserving the boundary cleanly.
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { STORAGE_HTTP_CODES, StorageHttpError } from "./error-codes.js";
|
||||
|
||||
describe("STORAGE_HTTP_CODES", () => {
|
||||
it("contains every documented code with the STORAGE_HTTP. prefix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const expected = [
|
||||
"NOT_FOUND",
|
||||
"UNAUTHORIZED",
|
||||
"FORBIDDEN",
|
||||
"NETWORK_ERROR",
|
||||
"INVALID_KEY",
|
||||
"KEY_TOO_LONG",
|
||||
"INVALID_OPTIONS",
|
||||
"READ_ONLY",
|
||||
"ABORTED",
|
||||
"UPSTREAM_ERROR",
|
||||
] as const;
|
||||
for (const key of expected) {
|
||||
expect(STORAGE_HTTP_CODES[key]).toBe(`STORAGE_HTTP.${key}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("StorageHttpError", () => {
|
||||
it("carries key/httpStatus/url discriminants", { timeout: 5000 }, () => {
|
||||
const err = new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND,
|
||||
"missing",
|
||||
{
|
||||
key: "a/b.bin",
|
||||
httpStatus: 404,
|
||||
url: "https://example.com/a/b.bin",
|
||||
}
|
||||
);
|
||||
expect(err.key).toBe("a/b.bin");
|
||||
expect(err.httpStatus).toBe(404);
|
||||
expect(err.url).toBe("https://example.com/a/b.bin");
|
||||
expect(err.name).toBe("StorageHttpError");
|
||||
expect(err.code).toBe(STORAGE_HTTP_CODES.NOT_FOUND);
|
||||
});
|
||||
|
||||
it("leaves discriminants undefined when options are omitted", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const err = new StorageHttpError(STORAGE_HTTP_CODES.ABORTED, "aborted");
|
||||
expect(err.key).toBeUndefined();
|
||||
expect(err.httpStatus).toBeUndefined();
|
||||
expect(err.url).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import {
|
||||
defineCodes,
|
||||
SuaveplanError,
|
||||
type SuaveplanErrorOptions,
|
||||
} from "@suaveplan/error";
|
||||
|
||||
export const STORAGE_HTTP_CODES = defineCodes("STORAGE_HTTP", [
|
||||
"NOT_FOUND",
|
||||
"UNAUTHORIZED",
|
||||
"FORBIDDEN",
|
||||
"NETWORK_ERROR",
|
||||
"INVALID_KEY",
|
||||
"KEY_TOO_LONG",
|
||||
"INVALID_OPTIONS",
|
||||
"READ_ONLY",
|
||||
"ABORTED",
|
||||
"UPSTREAM_ERROR",
|
||||
] as const);
|
||||
|
||||
export type StorageHttpErrorCode =
|
||||
(typeof STORAGE_HTTP_CODES)[keyof typeof STORAGE_HTTP_CODES];
|
||||
|
||||
export interface StorageHttpErrorOptions extends SuaveplanErrorOptions {
|
||||
key?: string;
|
||||
httpStatus?: number;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structured error for HTTP/CDN storage operations.
|
||||
*
|
||||
* Carries optional `key`, `httpStatus`, and `url` discriminants so callers can
|
||||
* correlate failures with the offending request without parsing message text.
|
||||
*/
|
||||
export class StorageHttpError extends SuaveplanError<StorageHttpErrorCode> {
|
||||
public readonly key: string | undefined;
|
||||
public readonly httpStatus: number | undefined;
|
||||
public readonly url: string | undefined;
|
||||
|
||||
public constructor(
|
||||
code: StorageHttpErrorCode,
|
||||
message: string,
|
||||
options?: StorageHttpErrorOptions
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.key = options?.key;
|
||||
this.httpStatus = options?.httpStatus;
|
||||
this.url = options?.url;
|
||||
this.name = "StorageHttpError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export type {
|
||||
StorageHttpErrorCode,
|
||||
StorageHttpErrorOptions,
|
||||
} from "./error-codes.js";
|
||||
export {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "./error-codes.js";
|
||||
@@ -0,0 +1,75 @@
|
||||
# etag-cache
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
In-memory LRU keyed by sanitised storage key, valued by `{ etag, body, lastModified }`. Backs `HttpStorage`'s conditional-GET path so reads of unchanged objects round-trip a single `304 Not Modified` instead of the full body. Capacity defaults to `DEFAULT_ETAG_CACHE_CAPACITY` (256) and is per-instance — cross-process caching is the caller's concern (typically composed via `storage-file` underneath).
|
||||
|
||||
## Features
|
||||
|
||||
- LRU eviction using `Map` insertion-order semantics — no external dependency.
|
||||
- `get(key)` is a side-effecting accessor that moves the hit to the MRU position.
|
||||
- `set(key, entry)` evicts the LRU entry when at capacity.
|
||||
- `clear()` and `delete(key)` for explicit invalidation.
|
||||
- `size` and `capacity` getters for tests and operator observability.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { EtagCache } from "@suaveplan/storage-http";
|
||||
|
||||
const cache = new EtagCache({ capacity: 128 });
|
||||
cache.set("a/b.bin", { etag: '"v1"', body: bytes, lastModified: undefined });
|
||||
const hit = cache.get("a/b.bin");
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { EtagCache } from "@suaveplan/storage-http";
|
||||
|
||||
const cache = new EtagCache();
|
||||
|
||||
async function conditionalGet(url: string, key: string): Promise<Uint8Array> {
|
||||
const cached = cache.get(key);
|
||||
const headers: Record<string, string> = {};
|
||||
if (cached !== undefined) headers["If-None-Match"] = cached.etag;
|
||||
const response = await fetch(url, { headers });
|
||||
if (response.status === 304 && cached !== undefined) return cached.body;
|
||||
const body = new Uint8Array(await response.arrayBuffer());
|
||||
const etag = response.headers.get("etag");
|
||||
if (etag !== null) {
|
||||
cache.set(key, { etag, body, lastModified: response.headers.get("last-modified") ?? undefined });
|
||||
}
|
||||
return body;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `class EtagCache`
|
||||
|
||||
- `constructor(options?: EtagCacheOptions)` — `options.capacity` defaults to `DEFAULT_ETAG_CACHE_CAPACITY`; values below `1` are clamped to `1`.
|
||||
- `size: number` — current entry count.
|
||||
- `capacity: number` — bound supplied at construction.
|
||||
- `get(key: string): EtagCacheEntry | undefined` — moves hit to MRU; returns `undefined` on miss.
|
||||
- `has(key: string): boolean` — non-side-effecting existence check.
|
||||
- `set(key: string, entry: EtagCacheEntry): void` — insert / replace; evicts LRU when full.
|
||||
- `delete(key: string): boolean` — returns `true` when an entry was removed.
|
||||
- `clear(): void` — drops every entry.
|
||||
|
||||
### `interface EtagCacheEntry`
|
||||
|
||||
- `etag: string` — verbatim from `ETag` response header (caller decides whether to strip weak prefix `W/`).
|
||||
- `body: Uint8Array` — cached body.
|
||||
- `lastModified: string | undefined` — verbatim `Last-Modified` header when present.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- LRU via insertion-order `Map` mutation is the cheapest correct implementation in JS without a doubly-linked list. The `get` hit path is two `Map` mutations (delete + set) but stays O(log n) due to the underlying hashtable.
|
||||
- `capacity` is clamped at construction; a caller passing `0` ends up with a 1-entry cache (degenerate but defined). Passing a negative number is a programmer error but is silently corrected to `1` rather than throwing — the cache is best-effort by design.
|
||||
- Cache invalidation on writes is the caller's responsibility — when `HttpStorage.set` succeeds, it calls `delete(key)` so the next read re-fetches. The cache itself has no awareness of writes.
|
||||
- Body storage uses the `Uint8Array` reference verbatim; callers that need defensive copies must slice before inserting.
|
||||
@@ -0,0 +1,105 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { EtagCache } from "./etag-cache.js";
|
||||
|
||||
const body1 = new Uint8Array([1, 2, 3]);
|
||||
const body2 = new Uint8Array([4, 5, 6]);
|
||||
|
||||
describe("EtagCache", () => {
|
||||
it("304 Not Modified returns cached body without retransmission", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const cache = new EtagCache();
|
||||
cache.set("k", { etag: '"v1"', body: body1, lastModified: undefined });
|
||||
const fetchSpy = async (
|
||||
input: string | URL | Request,
|
||||
init?: RequestInit
|
||||
): Promise<Response> => {
|
||||
const headers = init?.headers as Record<string, string>;
|
||||
expect(headers["If-None-Match"]).toBe('"v1"');
|
||||
return new Response(null, { status: 304 });
|
||||
};
|
||||
const response = await fetchSpy("https://example.com/k", {
|
||||
headers: { "If-None-Match": cache.get("k")?.etag ?? "" },
|
||||
});
|
||||
if (response.status === 304) {
|
||||
const hit = cache.get("k");
|
||||
expect(hit?.body).toBe(body1);
|
||||
}
|
||||
});
|
||||
|
||||
it("stores entries and returns them on get", { timeout: 5000 }, () => {
|
||||
const cache = new EtagCache();
|
||||
cache.set("k", { etag: '"v1"', body: body1, lastModified: undefined });
|
||||
const hit = cache.get("k");
|
||||
expect(hit?.etag).toBe('"v1"');
|
||||
expect(hit?.body).toBe(body1);
|
||||
});
|
||||
|
||||
it("get returns undefined on miss", { timeout: 5000 }, () => {
|
||||
const cache = new EtagCache();
|
||||
expect(cache.get("missing")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("has() is non-side-effecting", { timeout: 5000 }, () => {
|
||||
const cache = new EtagCache();
|
||||
cache.set("k", { etag: '"v"', body: body1, lastModified: undefined });
|
||||
expect(cache.has("k")).toBe(true);
|
||||
expect(cache.has("nope")).toBe(false);
|
||||
});
|
||||
|
||||
it("delete removes entries and reports prior existence", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cache = new EtagCache();
|
||||
cache.set("k", { etag: '"v"', body: body1, lastModified: undefined });
|
||||
expect(cache.delete("k")).toBe(true);
|
||||
expect(cache.delete("k")).toBe(false);
|
||||
expect(cache.size).toBe(0);
|
||||
});
|
||||
|
||||
it("clear drops every entry", { timeout: 5000 }, () => {
|
||||
const cache = new EtagCache();
|
||||
cache.set("a", { etag: '"a"', body: body1, lastModified: undefined });
|
||||
cache.set("b", { etag: '"b"', body: body2, lastModified: undefined });
|
||||
cache.clear();
|
||||
expect(cache.size).toBe(0);
|
||||
});
|
||||
|
||||
it("evicts the LRU entry when capacity is reached", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cache = new EtagCache({ capacity: 2 });
|
||||
cache.set("a", { etag: '"a"', body: body1, lastModified: undefined });
|
||||
cache.set("b", { etag: '"b"', body: body2, lastModified: undefined });
|
||||
// Touch a so b becomes LRU.
|
||||
cache.get("a");
|
||||
cache.set("c", {
|
||||
etag: '"c"',
|
||||
body: new Uint8Array([7]),
|
||||
lastModified: undefined,
|
||||
});
|
||||
expect(cache.has("a")).toBe(true);
|
||||
expect(cache.has("b")).toBe(false);
|
||||
expect(cache.has("c")).toBe(true);
|
||||
});
|
||||
|
||||
it("replaces existing entries without growing size", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cache = new EtagCache({ capacity: 2 });
|
||||
cache.set("k", { etag: '"v1"', body: body1, lastModified: undefined });
|
||||
cache.set("k", { etag: '"v2"', body: body2, lastModified: undefined });
|
||||
expect(cache.get("k")?.etag).toBe('"v2"');
|
||||
expect(cache.size).toBe(1);
|
||||
});
|
||||
|
||||
it("clamps non-positive capacity to 1", { timeout: 5000 }, () => {
|
||||
const cache = new EtagCache({ capacity: 0 });
|
||||
expect(cache.capacity).toBe(1);
|
||||
});
|
||||
|
||||
it("uses default capacity when none is supplied", { timeout: 5000 }, () => {
|
||||
const cache = new EtagCache();
|
||||
expect(cache.capacity).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
import { DEFAULT_ETAG_CACHE_CAPACITY } from "../constants/index.js";
|
||||
import type {
|
||||
EtagCacheEntry,
|
||||
EtagCacheOptions,
|
||||
} from "../types/etag-cache.types.js";
|
||||
|
||||
/**
|
||||
* In-memory LRU cache of `(key → {etag, body, lastModified})`. Backs the
|
||||
* conditional-GET path of `HttpStorage`: on `get`, if the cache has an entry
|
||||
* for the key the adapter attaches `If-None-Match: <etag>`; on `304 Not
|
||||
* Modified` the cached body is returned without retransmission; on `200 OK`
|
||||
* the entry is replaced.
|
||||
*
|
||||
* The implementation relies on the insertion-order guarantee of `Map` to
|
||||
* implement LRU: on hit, the entry is re-inserted (moving it to the
|
||||
* most-recently-used position); on miss with the cache full, the oldest
|
||||
* entry (first key) is evicted.
|
||||
*/
|
||||
export class EtagCache {
|
||||
readonly #capacity: number;
|
||||
readonly #entries = new Map<string, EtagCacheEntry>();
|
||||
|
||||
public constructor(options: EtagCacheOptions = {}) {
|
||||
const requested = options.capacity ?? DEFAULT_ETAG_CACHE_CAPACITY;
|
||||
this.#capacity = Math.max(1, requested);
|
||||
}
|
||||
|
||||
/** Current entry count — primarily for tests. */
|
||||
public get size(): number {
|
||||
return this.#entries.size;
|
||||
}
|
||||
|
||||
/** Cache capacity (lower bound 1, upper bound caller-supplied). */
|
||||
public get capacity(): number {
|
||||
return this.#capacity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the cached entry for `key` and moves it to the
|
||||
* most-recently-used position. Returns `undefined` on miss.
|
||||
*/
|
||||
public get(key: string): EtagCacheEntry | undefined {
|
||||
const entry = this.#entries.get(key);
|
||||
if (entry === undefined) return undefined;
|
||||
this.#entries.delete(key);
|
||||
this.#entries.set(key, entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** True when the cache holds an entry for `key`. */
|
||||
public has(key: string): boolean {
|
||||
return this.#entries.has(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert or replace the entry for `key`. Evicts the least-recently-used
|
||||
* entry when at capacity.
|
||||
*/
|
||||
public set(key: string, entry: EtagCacheEntry): void {
|
||||
if (this.#entries.has(key)) {
|
||||
this.#entries.delete(key);
|
||||
} else if (this.#entries.size >= this.#capacity) {
|
||||
// `size >= capacity >= 1` implies `keys().next().value` is a string,
|
||||
// so the `done: false` branch is statically guaranteed.
|
||||
const oldest = this.#entries.keys().next().value as string;
|
||||
this.#entries.delete(oldest);
|
||||
}
|
||||
this.#entries.set(key, entry);
|
||||
}
|
||||
|
||||
/** Remove `key` from the cache. Returns `true` when an entry existed. */
|
||||
public delete(key: string): boolean {
|
||||
return this.#entries.delete(key);
|
||||
}
|
||||
|
||||
/** Drop every entry. */
|
||||
public clear(): void {
|
||||
this.#entries.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { EtagCache } from "./etag-cache.js";
|
||||
@@ -0,0 +1,63 @@
|
||||
# fetch-adapter
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Wraps `globalThis.fetch` with the contract `HttpStorage` actually consumes. Translates raw network rejections (DNS failure, TLS error, connection reset, abort-without-signal) into a structured `StorageHttpError` carrying code `STORAGE_HTTP.NETWORK_ERROR` and the offending URL. `AbortError` is re-raised unchanged so the per-method wrapper in `HttpStorage` can map signal-driven cancellation to `STORAGE_HTTP.ABORTED`.
|
||||
|
||||
## Features
|
||||
|
||||
- Single function `defaultFetch()` returns a `FetchLike` bound to the global `fetch`.
|
||||
- `NETWORK_ERROR` mapping covers every kind of `fetch` reject that isn't an abort.
|
||||
- URL is extracted from `string`, `URL`, or `Request` inputs and attached to the error for diagnostics.
|
||||
- `isAbortError(err)` exposed for callers that need to distinguish abort from other failures without re-implementing the `DOMException`-shaped check.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { defaultFetch } from "@suaveplan/storage-http";
|
||||
|
||||
const fetch = defaultFetch();
|
||||
const response = await fetch("https://cdn.example.com/file.bin");
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { defaultFetch, isAbortError, StorageHttpError, STORAGE_HTTP_CODES } from "@suaveplan/storage-http";
|
||||
|
||||
const fetch = defaultFetch();
|
||||
try {
|
||||
const response = await fetch("https://cdn.example.com/file.bin", { signal });
|
||||
if (!response.ok) handleStatus(response.status);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) return; // caller-driven cancellation
|
||||
if (err instanceof StorageHttpError && err.code === STORAGE_HTTP_CODES.NETWORK_ERROR) {
|
||||
operatorLog.error({ url: err.url, cause: err.cause }, "Origin unreachable");
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `defaultFetch(): FetchLike`
|
||||
|
||||
Returns a `FetchLike` (`(input, init?) => Promise<Response>`) that:
|
||||
|
||||
- Forwards to `globalThis.fetch` unchanged on success.
|
||||
- Re-raises `AbortError` unchanged for the caller's wrapper.
|
||||
- Maps every other `fetch` reject to `StorageHttpError(STORAGE_HTTP.NETWORK_ERROR)` with the resolved URL and the original error as `cause`.
|
||||
|
||||
### `isAbortError(err: unknown): boolean`
|
||||
|
||||
Returns `true` when `err` is a non-null object with `name === "AbortError"`. Runtime-neutral — works for both `DOMException` (browsers, Workers) and Node 18+'s native `AbortError`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The adapter is intentionally thin. Status-code mapping (404 → `undefined`, 401 → `UNAUTHORIZED`, etc.) belongs to `HttpStorage`'s per-method wrappers, not here, so this module stays usable for callers that want raw status access (e.g. a custom listing implementation).
|
||||
- URL extraction handles all three documented `fetch` input shapes. A pathological input that is none of them returns `undefined` URL on the error — the message still names the failure type.
|
||||
- `globalThis.fetch` is required (Node 18+, Bun, Deno, modern browsers, Workers). The package does not ship a polyfill — that would violate tier purity.
|
||||
@@ -0,0 +1,175 @@
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { defaultFetch, isAbortError } from "./fetch-adapter.js";
|
||||
|
||||
describe("defaultFetch", () => {
|
||||
it("forwards a string URL to globalThis.fetch", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockResolvedValueOnce(new Response(new Uint8Array([1])));
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
const response = await fetchFn("https://example.com/x");
|
||||
expect(response.ok).toBe(true);
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("maps network rejection to STORAGE_HTTP.NETWORK_ERROR with URL preserved", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce(new Error("ECONNREFUSED"));
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
await fetchFn("https://example.com/x");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(StorageHttpError);
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR
|
||||
);
|
||||
expect((err as StorageHttpError).url).toBe("https://example.com/x");
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("re-raises AbortError unchanged", { timeout: 5000 }, async () => {
|
||||
const abort = Object.assign(new Error("aborted"), {
|
||||
name: "AbortError",
|
||||
});
|
||||
const spy = vi.spyOn(globalThis, "fetch").mockRejectedValueOnce(abort);
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
await fetchFn("https://example.com/x");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBe(abort);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves URL from a URL object", { timeout: 5000 }, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce(new Error("boom"));
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
try {
|
||||
await fetchFn(new URL("https://example.com/y"));
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).url).toBe(
|
||||
"https://example.com/y"
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves URL from a Request-like input", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce(new Error("boom"));
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
try {
|
||||
await fetchFn({ url: "https://example.com/z" } as Request);
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).url).toBe(
|
||||
"https://example.com/z"
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to no URL when input is unrecognised shape", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce(new Error("boom"));
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
try {
|
||||
await fetchFn({} as Request);
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).url).toBeUndefined();
|
||||
}
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("uses String(err) when the reject value is not an Error", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce("boom-string");
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
try {
|
||||
await fetchFn("https://example.com/x");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).message).toContain(
|
||||
"boom-string"
|
||||
);
|
||||
expect((err as StorageHttpError).cause).toBeUndefined();
|
||||
}
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("maps non-Error rejection on an input with no URL", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce("boom-no-url");
|
||||
try {
|
||||
const fetchFn = defaultFetch();
|
||||
try {
|
||||
await fetchFn({} as Request);
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).url).toBeUndefined();
|
||||
expect((err as StorageHttpError).cause).toBeUndefined();
|
||||
}
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAbortError", () => {
|
||||
it("returns true for AbortError-shaped objects", { timeout: 5000 }, () => {
|
||||
expect(isAbortError({ name: "AbortError" })).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for null and primitives", { timeout: 5000 }, () => {
|
||||
expect(isAbortError(null)).toBe(false);
|
||||
expect(isAbortError("x")).toBe(false);
|
||||
expect(isAbortError(42)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for Errors with a different name", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(isAbortError(new Error("boom"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import type { FetchLike } from "../types/fetch.types.js";
|
||||
|
||||
/**
|
||||
* Returns a `FetchLike` bound to `globalThis.fetch`. Network rejections
|
||||
* (DNS, TLS, abort-without-signal, connection reset) are mapped to
|
||||
* `StorageHttpError(NETWORK_ERROR)`; `AbortError` (signal-driven) is
|
||||
* re-raised so the caller's wrapper can map it to `ABORTED`.
|
||||
*/
|
||||
export function defaultFetch(): FetchLike {
|
||||
return async (input, init) => {
|
||||
try {
|
||||
return await globalThis.fetch(
|
||||
input as Parameters<typeof globalThis.fetch>[0],
|
||||
init
|
||||
);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) {
|
||||
throw err;
|
||||
}
|
||||
const url = resolveUrl(input);
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR,
|
||||
`fetch failed: ${message}`,
|
||||
url !== undefined
|
||||
? { url, cause: err instanceof Error ? err : undefined }
|
||||
: { cause: err instanceof Error ? err : undefined }
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when `err` looks like a `DOMException` (or any `Error`) with
|
||||
* `name === "AbortError"` — the runtime-neutral way to detect a signal-
|
||||
* cancelled fetch without importing DOM types.
|
||||
*/
|
||||
export function isAbortError(err: unknown): boolean {
|
||||
if (err === null || typeof err !== "object") return false;
|
||||
const name = (err as { name?: unknown }).name;
|
||||
return name === "AbortError";
|
||||
}
|
||||
|
||||
function resolveUrl(input: string | URL | Request): string | undefined {
|
||||
if (typeof input === "string") return input;
|
||||
if (input instanceof URL) return input.toString();
|
||||
if (typeof (input as { url?: unknown }).url === "string") {
|
||||
return (input as { url: string }).url;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
defaultFetch,
|
||||
isAbortError,
|
||||
} from "./fetch-adapter.js";
|
||||
@@ -0,0 +1,82 @@
|
||||
# http-storage.helpers
|
||||
|
||||
## Purpose
|
||||
|
||||
Internal helpers used by `HttpStorage`. Splits boundary concerns (option parsing, URL composition, header resolution, abort guards, status → error mapping) out of the public class so the latter stays close to 250 lines and the helpers can be unit-tested in isolation.
|
||||
|
||||
## Features
|
||||
|
||||
- `resolveHttpStorageConfig(options)` — parses options through `HttpStorageOptionsSchema`, throws `STORAGE_HTTP.INVALID_OPTIONS` on validation failure, and pre-resolves `fetch` / `keyEncoder` defaults.
|
||||
- `composeUrl(config, sanitisedKey)` — applies `keyEncoder` and `new URL(encoded, baseUrl)` in one call.
|
||||
- `resolveHeaders(resolver, key, op)` — static record or async per-op callback.
|
||||
- `throwIfAborted(signal)` — pre-fetch abort guard producing `STORAGE_HTTP.ABORTED`.
|
||||
- `errorForStatus(status, url, key)` — maps `401`/`403`/`404`/`410`/other to the canonical `StorageHttpError`.
|
||||
- `outcomeFromError(err)` — translates any thrown error to the `{ outcome, statusClass }` pair the OTel wrapper records.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { resolveHttpStorageConfig, composeUrl } from "@suaveplan/storage-http";
|
||||
|
||||
const config = resolveHttpStorageConfig({ baseUrl: "https://cdn.example.com/start/here/" });
|
||||
const url = composeUrl(config, "a/b/file.bin");
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { errorForStatus, outcomeFromError, resolveHeaders, throwIfAborted } from "@suaveplan/storage-http";
|
||||
|
||||
async function customGet(config, key, signal) {
|
||||
throwIfAborted(signal);
|
||||
const headers = await resolveHeaders(config.headers, key, "get");
|
||||
const response = await config.fetch(composeUrl(config, key), { headers, signal });
|
||||
if (!response.ok) {
|
||||
const err = errorForStatus(response.status, composeUrl(config, key), key);
|
||||
const { outcome, statusClass } = outcomeFromError(err);
|
||||
recordOp({ op: "get", outcome, statusClass, durationMs: 0 });
|
||||
throw err;
|
||||
}
|
||||
return new Uint8Array(await response.arrayBuffer());
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `resolveHttpStorageConfig(options): ResolvedHttpStorageConfig`
|
||||
|
||||
Throws `StorageHttpError(STORAGE_HTTP.INVALID_OPTIONS)` with one message per Zod issue. Returns `{ baseUrl, headers, fetch, keyEncoder, options }`.
|
||||
|
||||
### `composeUrl(config, sanitisedKey): string`
|
||||
|
||||
Encodes the sanitised key via `config.keyEncoder`, resolves against `config.baseUrl`.
|
||||
|
||||
### `resolveHeaders(resolver, key, op): Promise<Record<string, string>>`
|
||||
|
||||
- `undefined` → `{}`.
|
||||
- Static record → shallow copy.
|
||||
- Function → awaited, then shallow-copied.
|
||||
|
||||
### `throwIfAborted(signal?): void`
|
||||
|
||||
Throws `StorageHttpError(STORAGE_HTTP.ABORTED)` when `signal?.aborted === true`.
|
||||
|
||||
### `errorForStatus(status, url, key): StorageHttpError`
|
||||
|
||||
| status | code |
|
||||
|---|---|
|
||||
| 401 | `UNAUTHORIZED` |
|
||||
| 403 | `FORBIDDEN` |
|
||||
| 404, 410 | `NOT_FOUND` |
|
||||
| other | `UPSTREAM_ERROR` |
|
||||
|
||||
### `outcomeFromError(err): { outcome, statusClass }`
|
||||
|
||||
`StorageHttpError` → maps by `code`; anything else → `{ outcome: "upstream_error", statusClass: "none" }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The error-status table intentionally collapses `404` and `410` to `NOT_FOUND`. `410 Gone` is semantically distinct, but storage callers treat both as "no value" — distinguishing them would force every caller to handle a second code path with no behavioural difference.
|
||||
- `outcomeFromError` returns the wider `outcome` enum used by `recordOp`. Adding a new error code requires extending the switch — every code MUST map deterministically so the OTel counters stay correct.
|
||||
- `resolveHeaders` defensively copies its inputs so a caller mutating their static record between calls doesn't leak into in-flight requests.
|
||||
- `composeUrl` relies on `new URL(rel, base)` enforcing the trailing-slash invariant. Because the schema rejects non-`/`-terminated `baseUrl`, the function cannot silently truncate a path segment.
|
||||
@@ -0,0 +1,148 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import {
|
||||
composeUrl,
|
||||
errorForStatus,
|
||||
outcomeFromError,
|
||||
resolveHeaders,
|
||||
resolveHttpStorageConfig,
|
||||
throwIfAborted,
|
||||
} from "./http-storage.helpers.js";
|
||||
|
||||
const baseUrl = "https://cdn.example.com/start/here/";
|
||||
|
||||
describe("resolveHttpStorageConfig", () => {
|
||||
it("returns defaults for omitted optional fields", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const config = resolveHttpStorageConfig({ baseUrl });
|
||||
expect(config.baseUrl).toBe(baseUrl);
|
||||
expect(typeof config.fetch).toBe("function");
|
||||
expect(typeof config.keyEncoder).toBe("function");
|
||||
});
|
||||
|
||||
it("uses caller-supplied fetch and keyEncoder", { timeout: 5000 }, () => {
|
||||
const fetchFn = async () => new Response();
|
||||
const encoder = (k: string) => k.toUpperCase();
|
||||
const config = resolveHttpStorageConfig({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
keyEncoder: encoder,
|
||||
});
|
||||
expect(config.fetch).toBe(fetchFn);
|
||||
expect(config.keyEncoder).toBe(encoder);
|
||||
});
|
||||
});
|
||||
|
||||
describe("composeUrl", () => {
|
||||
it("joins baseUrl with the encoded key", { timeout: 5000 }, () => {
|
||||
const config = resolveHttpStorageConfig({ baseUrl });
|
||||
expect(composeUrl(config, "a/b/c.bin")).toBe(
|
||||
"https://cdn.example.com/start/here/a/b/c.bin"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveHeaders", () => {
|
||||
it("returns an empty record when resolver is undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const out = await resolveHeaders(undefined, "k", "get");
|
||||
expect(out).toEqual({});
|
||||
});
|
||||
|
||||
it("returns a defensive copy of a static record", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const source = { x: "1" };
|
||||
const out = await resolveHeaders(source, "k", "get");
|
||||
out.x = "2";
|
||||
expect(source.x).toBe("1");
|
||||
});
|
||||
|
||||
it("invokes the function form with key and op", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fn = async (k: string, op: string) => ({ key: k, op });
|
||||
const out = await resolveHeaders(fn, "alpha", "head");
|
||||
expect(out).toEqual({ key: "alpha", op: "head" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("throwIfAborted", () => {
|
||||
it("throws when the signal is already aborted", { timeout: 5000 }, () => {
|
||||
const c = new AbortController();
|
||||
c.abort();
|
||||
expect(() => throwIfAborted(c.signal)).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("is a no-op when signal is undefined or not aborted", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() => throwIfAborted(undefined)).not.toThrow();
|
||||
const c = new AbortController();
|
||||
expect(() => throwIfAborted(c.signal)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("errorForStatus", () => {
|
||||
it("maps 401 → UNAUTHORIZED", { timeout: 5000 }, () => {
|
||||
const err = errorForStatus(401, "https://x", "k");
|
||||
expect(err.code).toBe(STORAGE_HTTP_CODES.UNAUTHORIZED);
|
||||
});
|
||||
|
||||
it("maps 403 → FORBIDDEN", { timeout: 5000 }, () => {
|
||||
expect(errorForStatus(403, "https://x", "k").code).toBe(
|
||||
STORAGE_HTTP_CODES.FORBIDDEN
|
||||
);
|
||||
});
|
||||
|
||||
it("maps 404 and 410 → NOT_FOUND", { timeout: 5000 }, () => {
|
||||
expect(errorForStatus(404, "https://x", "k").code).toBe(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND
|
||||
);
|
||||
expect(errorForStatus(410, "https://x", "k").code).toBe(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND
|
||||
);
|
||||
});
|
||||
|
||||
it("maps other codes → UPSTREAM_ERROR", { timeout: 5000 }, () => {
|
||||
expect(errorForStatus(418, "https://x", "k").code).toBe(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("outcomeFromError", () => {
|
||||
it("maps each StorageHttpError code to the documented outcome", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const cases: Array<[string, string]> = [
|
||||
[STORAGE_HTTP_CODES.ABORTED, "aborted"],
|
||||
[STORAGE_HTTP_CODES.NOT_FOUND, "not_found"],
|
||||
[STORAGE_HTTP_CODES.UNAUTHORIZED, "unauthorized"],
|
||||
[STORAGE_HTTP_CODES.FORBIDDEN, "forbidden"],
|
||||
[STORAGE_HTTP_CODES.NETWORK_ERROR, "network_error"],
|
||||
[STORAGE_HTTP_CODES.READ_ONLY, "read_only"],
|
||||
[STORAGE_HTTP_CODES.UPSTREAM_ERROR, "upstream_error"],
|
||||
[STORAGE_HTTP_CODES.INVALID_OPTIONS, "upstream_error"],
|
||||
];
|
||||
for (const [code, expected] of cases) {
|
||||
const result = outcomeFromError(
|
||||
new StorageHttpError(code as never, "x")
|
||||
);
|
||||
expect(result.outcome).toBe(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("maps non-StorageHttpError to upstream_error/none", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = outcomeFromError(new Error("boom"));
|
||||
expect(result.outcome).toBe("upstream_error");
|
||||
expect(result.statusClass).toBe("none");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,192 @@
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { defaultFetch } from "../fetch-adapter/fetch-adapter.js";
|
||||
import { defaultKeyEncoder } from "../key-sanitizer/key-sanitizer.js";
|
||||
import { type StatusClass, statusClassOf } from "../otel/otel.js";
|
||||
import type { RetryWithBackoffOptions } from "../retry-helper/retry-helper.js";
|
||||
import type { FetchLike } from "../types/fetch.types.js";
|
||||
import { HttpStorageOptionsSchema } from "../types/http-storage.schemas.js";
|
||||
import type {
|
||||
HttpHeadersResolver,
|
||||
HttpStorageOp,
|
||||
HttpStorageOptions,
|
||||
KeyEncoder,
|
||||
} from "../types/http-storage.types.js";
|
||||
|
||||
export interface ResolvedHttpStorageConfig {
|
||||
baseUrl: string;
|
||||
headers: HttpHeadersResolver | undefined;
|
||||
fetch: FetchLike;
|
||||
keyEncoder: KeyEncoder;
|
||||
options: HttpStorageOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the caller-supplied options through Zod and pre-resolve the
|
||||
* `fetch` / `keyEncoder` defaults.
|
||||
*/
|
||||
export function resolveHttpStorageConfig(
|
||||
options: HttpStorageOptions
|
||||
): ResolvedHttpStorageConfig {
|
||||
const parsed = HttpStorageOptionsSchema.safeParse(options);
|
||||
if (!parsed.success) {
|
||||
const message = parsed.error.issues
|
||||
.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
|
||||
.join("; ");
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_OPTIONS,
|
||||
`invalid HttpStorage options: ${message}`
|
||||
);
|
||||
}
|
||||
return {
|
||||
baseUrl: options.baseUrl,
|
||||
headers: options.headers,
|
||||
fetch: options.fetch ?? defaultFetch(),
|
||||
keyEncoder: options.keyEncoder ?? defaultKeyEncoder,
|
||||
options,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose `new URL(encodedKey, baseUrl)` with caller-supplied encoding rules.
|
||||
*/
|
||||
export function composeUrl(
|
||||
config: ResolvedHttpStorageConfig,
|
||||
sanitisedKey: string
|
||||
): string {
|
||||
const encoded = config.keyEncoder(sanitisedKey);
|
||||
return new URL(encoded, config.baseUrl).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the `headers` callback for one request. Returns an empty record
|
||||
* when no headers were configured.
|
||||
*/
|
||||
export async function resolveHeaders(
|
||||
resolver: HttpHeadersResolver | undefined,
|
||||
key: string,
|
||||
op: HttpStorageOp
|
||||
): Promise<Record<string, string>> {
|
||||
if (resolver === undefined) return {};
|
||||
if (typeof resolver === "function") {
|
||||
const value = await resolver(key, op);
|
||||
return { ...value };
|
||||
}
|
||||
return { ...resolver };
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw `STORAGE_HTTP.ABORTED` immediately when `signal` is already aborted.
|
||||
*/
|
||||
export function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted === true) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.ABORTED,
|
||||
"Operation aborted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the retry option bag from storage options + caller signal. The
|
||||
* adapter passes this to `retryWithBackoff` on every wrapped operation.
|
||||
*/
|
||||
export function buildRetryOptions(
|
||||
options: HttpStorageOptions,
|
||||
signal: AbortSignal | undefined
|
||||
): RetryWithBackoffOptions {
|
||||
const opts: RetryWithBackoffOptions = {};
|
||||
if (options.retry !== undefined) opts.retry = options.retry;
|
||||
if (options.timers !== undefined) opts.timers = options.timers;
|
||||
if (signal !== undefined) opts.signal = signal;
|
||||
return opts;
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate an upstream HTTP status to the canonical storage-http error code.
|
||||
* The caller passes the URL/key so the resulting error carries diagnostics.
|
||||
*/
|
||||
export function errorForStatus(
|
||||
status: number,
|
||||
url: string,
|
||||
key: string
|
||||
): StorageHttpError {
|
||||
const opts = { url, key, httpStatus: status };
|
||||
if (status === 401) {
|
||||
return new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.UNAUTHORIZED,
|
||||
`unauthorised (401) for ${url}`,
|
||||
opts
|
||||
);
|
||||
}
|
||||
if (status === 403) {
|
||||
return new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.FORBIDDEN,
|
||||
`forbidden (403) for ${url}`,
|
||||
opts
|
||||
);
|
||||
}
|
||||
if (status === 404 || status === 410) {
|
||||
return new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND,
|
||||
`not found (${status}) for ${url}`,
|
||||
opts
|
||||
);
|
||||
}
|
||||
return new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR,
|
||||
`upstream error (${status}) for ${url}`,
|
||||
opts
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Map any thrown error to a storage-http outcome attribute. Used by the
|
||||
* per-method OTel wrapper.
|
||||
*/
|
||||
export function outcomeFromError(err: unknown): {
|
||||
outcome:
|
||||
| "ok"
|
||||
| "not_found"
|
||||
| "unauthorized"
|
||||
| "forbidden"
|
||||
| "network_error"
|
||||
| "upstream_error"
|
||||
| "aborted"
|
||||
| "read_only";
|
||||
statusClass: StatusClass;
|
||||
} {
|
||||
if (err instanceof StorageHttpError) {
|
||||
switch (err.code) {
|
||||
case STORAGE_HTTP_CODES.ABORTED:
|
||||
return { outcome: "aborted", statusClass: "none" };
|
||||
case STORAGE_HTTP_CODES.NOT_FOUND:
|
||||
return {
|
||||
outcome: "not_found",
|
||||
statusClass: statusClassOf(err.httpStatus),
|
||||
};
|
||||
case STORAGE_HTTP_CODES.UNAUTHORIZED:
|
||||
return {
|
||||
outcome: "unauthorized",
|
||||
statusClass: statusClassOf(err.httpStatus),
|
||||
};
|
||||
case STORAGE_HTTP_CODES.FORBIDDEN:
|
||||
return {
|
||||
outcome: "forbidden",
|
||||
statusClass: statusClassOf(err.httpStatus),
|
||||
};
|
||||
case STORAGE_HTTP_CODES.NETWORK_ERROR:
|
||||
return { outcome: "network_error", statusClass: "none" };
|
||||
case STORAGE_HTTP_CODES.READ_ONLY:
|
||||
return { outcome: "read_only", statusClass: "none" };
|
||||
default:
|
||||
return {
|
||||
outcome: "upstream_error",
|
||||
statusClass: statusClassOf(err.httpStatus),
|
||||
};
|
||||
}
|
||||
}
|
||||
return { outcome: "upstream_error", statusClass: "none" };
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
# http-storage
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
`HttpStorage` is the universal `AbstractStorage` adapter that maps every storage operation onto an HTTP request against a configured `baseUrl + key`. It covers the full read surface (`GET`, `HEAD`, `Range`, native streaming, ETag-cached conditional reads) and gates write operations behind an injectable `HttpUploader` — without one, `set`, `delete`, `setPartial`, and `createWriteStream` throw `STORAGE_HTTP.READ_ONLY`. `list`, `clear`, and `getSize` always throw `READ_ONLY` because HTTP origins do not expose a uniform directory protocol.
|
||||
|
||||
The class is the natural binding for static-site CDNs (Cloudflare, Fastly), prefix-readable S3 buckets, public BunnyCDN Pull Zones, GitHub Pages, Caddy file servers, and similar origins. Provider-specific adapters (Rackspace, BunnyCDN, R2) layer on top via the `HttpUploader` slot.
|
||||
|
||||
## Features
|
||||
|
||||
- Full `AbstractStorage` shape — `get`, `set` (with uploader), `exists`, `getMetadata`, `getWithMetadata`, `getItemSize`, `getPartial`, `createReadStream`, `createWriteStream`, `delete`, `setPartial`.
|
||||
- `capabilities` declares `partialReads: true`, `nativeStreams: true`, `atomicWrites: false`, `partialWrites: false`, `contentAddressable: false`.
|
||||
- AbortSignal threaded into every fetch; pre-aborted signal short-circuits before any network call.
|
||||
- Optional ETag-cached conditional reads — `If-None-Match` attached automatically, `304` returns cached body.
|
||||
- HTTP `Range` reads with `206`/`200`/`416` disambiguation and a counter for misconfigured origins.
|
||||
- Native `ReadableStream` wrapping `Response.body` — no client-side buffering.
|
||||
- Retry on transient outcomes (`408`/`425`/`429`/`5xx` and network rejection) honouring `Retry-After`.
|
||||
- Full OTel surface — duration / IO bytes histograms; per-op outcome counters; spans per call.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { HttpStorage } from "@suaveplan/storage-http";
|
||||
|
||||
const storage = new HttpStorage({
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
});
|
||||
const bytes = await storage.get("a/b/test.doc");
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { HttpStorage, presignedPutUploader } from "@suaveplan/storage-http";
|
||||
|
||||
const storage = new HttpStorage({
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
headers: (key, op) =>
|
||||
op === "set"
|
||||
? { Authorization: `Bearer ${signedToken()}`, "Content-Type": "application/octet-stream" }
|
||||
: { Authorization: `Bearer ${readToken()}` },
|
||||
uploader: presignedPutUploader({ signer: presignFor(key, "PUT") }),
|
||||
etagCache: { capacity: 1024 },
|
||||
retry: { maxAttempts: 5, baseDelayMs: 200, maxDelayMs: 10_000, honourRetryAfter: true },
|
||||
});
|
||||
|
||||
const controller = new AbortController();
|
||||
const stream = storage.createReadStream("video.mp4", controller.signal);
|
||||
for await (const chunk of stream) processChunk(chunk);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `class HttpStorage extends AbstractStorage`
|
||||
|
||||
- `constructor(options: HttpStorageOptions)` — validates `options` via `HttpStorageOptionsSchema`; throws `STORAGE_HTTP.INVALID_OPTIONS` on failure (notably missing trailing slash on `baseUrl`).
|
||||
- `capabilities: StorageCapabilities` — frozen per the matrix above.
|
||||
|
||||
Read methods:
|
||||
|
||||
- `get(key, signal?): Promise<Uint8Array | undefined>` — `404`/`410` → `undefined`; `4xx` → mapped error; transient → retried.
|
||||
- `exists(key, signal?): Promise<boolean>` — HEAD-backed.
|
||||
- `getMetadata(key, signal?): Promise<StorageMetadata | null>` — parses `Content-Length`, `Last-Modified`.
|
||||
- `getWithMetadata`, `getItemSize` — composition over `get` + `getMetadata`.
|
||||
- `getPartial(key, offset, length, signal?): Promise<Uint8Array | null>` — Range with 206/200/416 handling.
|
||||
- `createReadStream(key, signal?): ReadableStream<Uint8Array>` — wraps `Response.body`.
|
||||
|
||||
Write methods (require `uploader`):
|
||||
|
||||
- `set(key, value, options?, signal?): Promise<void>` — calls `uploader.put`; invalidates ETag cache entry.
|
||||
- `delete(key, signal?): Promise<boolean>` — requires `uploader.delete`; throws `READ_ONLY` otherwise.
|
||||
- `setPartial` — read-modify-write via `defaultSetPartial`.
|
||||
- `createWriteStream(key, signal?): WritableStream<Uint8Array>` — buffers, calls `uploader.put` on `close`.
|
||||
|
||||
Always-`READ_ONLY`:
|
||||
|
||||
- `list`, `clear`, `getSize` — HTTP origins do not standardise these.
|
||||
|
||||
### `createHttpStorage(options): HttpStorage`
|
||||
|
||||
Convenience factory mirroring `createNodeFsStorage`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `getWithMetadata` issues two requests (`GET` + `HEAD`). Operators that prefer one round-trip should call `get` and synthesise metadata from the response headers (the ETag cache stores `lastModified` for exactly this reason).
|
||||
- `setPartial` only proceeds when an uploader is wired; it then delegates to `AbstractStorage.defaultSetPartial` which is O(N) memory per call. Backends with native range writes belong in a different adapter.
|
||||
- `createWriteStream` collects to memory before calling `uploader.put`. HTTP origins generally don't support truly chunked PUTs without provider-specific multipart logic, and the spec scoped that out.
|
||||
- ETag cache invalidation is unconditional on `set` / `delete` / `createWriteStream.close` — the cache must never hold stale bodies after a write.
|
||||
- All non-`ok` HEAD responses go through `upstreamError` so the `STORAGE_HTTP.UPSTREAM_ERROR` code is consistently raised on the metadata path.
|
||||
- The `_options` parameter on `set` is currently unused — `StorageOptions.ttl` / `compress` / `encrypt` have no HTTP semantics, and `metadata` belongs in `headers` via the caller's resolver.
|
||||
@@ -0,0 +1,92 @@
|
||||
# http-storage.ops
|
||||
|
||||
## Purpose
|
||||
|
||||
Reusable read-side operations used by `HttpStorage`: full-body `GET` with ETag-cache support, `HEAD` for metadata, `Range`-based partial reads, and streaming `GET`. Each helper takes a `ReadContext` (config + cache + key + signal + retry options) and returns the result; the public class methods compose them with the OTel wrapper and signal guards.
|
||||
|
||||
Split out so tests can exercise each path in isolation without setting up a full `HttpStorage` instance.
|
||||
|
||||
## Features
|
||||
|
||||
- `performGet(ctx)` — full-body GET, ETag-cache aware (sends `If-None-Match` on hit, returns cached body on `304`), retries transient outcomes, maps non-`ok` status to `errorForStatus`.
|
||||
- `performHead(ctx)` — HEAD with retry.
|
||||
- `performGetPartial(ctx, offset, length)` — issues Range, delegates 200/206/416 disambiguation to `consumeRangeResponse`, increments range-fallback counter on `200`.
|
||||
- `performGetStream(ctx)` — single GET without ETag-cache, returns the raw `Response` so the caller can pump `body.getReader()`.
|
||||
- `fetchWithSignal(ctx, url, method, headers)` — low-level wrapper that attaches the signal to `init`.
|
||||
- `parseMetadata(response)` — extracts `StorageMetadata` from HEAD response headers.
|
||||
- `readOnlyError(method, key?)` — canonical `STORAGE_HTTP.READ_ONLY` builder.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { performGet } from "@suaveplan/storage-http";
|
||||
|
||||
const ctx = { config, cache: undefined, key: "file.bin", signal, retryOptions: {} };
|
||||
const body = await performGet(ctx);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import {
|
||||
composeUrl,
|
||||
fetchWithSignal,
|
||||
performGetPartial,
|
||||
performHead,
|
||||
parseMetadata,
|
||||
readOnlyError,
|
||||
} from "@suaveplan/storage-http";
|
||||
|
||||
async function existsAndMeta(ctx): Promise<StorageMetadata | null> {
|
||||
const response = await performHead(ctx);
|
||||
if (response.status === 404) return null;
|
||||
return parseMetadata(response);
|
||||
}
|
||||
|
||||
async function partialReader(ctx, offset: number, length: number) {
|
||||
const bytes = await performGetPartial(ctx, offset, length);
|
||||
if (bytes === null) throw readOnlyError("getPartial", ctx.key);
|
||||
return bytes;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `interface ReadContext`
|
||||
|
||||
`{ config, cache, key, signal, retryOptions }` — the bundle every read helper consumes.
|
||||
|
||||
### `performGet(ctx): Promise<Uint8Array | undefined>`
|
||||
|
||||
Returns `undefined` on `404`/`410`, the cached body on `304`, the response bytes otherwise. Updates the cache on `200` when an `ETag` is present.
|
||||
|
||||
### `performHead(ctx): Promise<Response>`
|
||||
|
||||
Returns the raw response. Callers inspect `status` and headers themselves.
|
||||
|
||||
### `performGetPartial(ctx, offset, length): Promise<Uint8Array | null>`
|
||||
|
||||
Returns `null` on `404`/`410`, throws on `401`/`403`/non-2xx without Range fallback, otherwise returns the slice via `consumeRangeResponse`.
|
||||
|
||||
### `performGetStream(ctx): Promise<Response>`
|
||||
|
||||
Throws `errorForStatus` on non-`ok` responses; caller is responsible for consuming `body`.
|
||||
|
||||
### `fetchWithSignal(ctx, url, method, headers): Promise<Response>`
|
||||
|
||||
Thin wrapper that calls `config.fetch(url, init)` with the signal attached when present.
|
||||
|
||||
### `parseMetadata(response): StorageMetadata`
|
||||
|
||||
Extracts `Content-Length` (defaulting `size` to 0 when absent or unparsed), `Last-Modified` (defaulting to `Date.now()` when missing). Returns `createdAt === modifiedAt` because HTTP HEAD does not surface creation time.
|
||||
|
||||
### `readOnlyError(method, key?): StorageHttpError`
|
||||
|
||||
Builds the canonical `STORAGE_HTTP.READ_ONLY` error with a documented message.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `performGet` is the only helper that touches the ETag cache; `performGetPartial` and `performGetStream` deliberately bypass it because partial reads and streams complicate the caching contract (you would have to cache slices, not the whole body).
|
||||
- The 401/403 short-circuit in `performGetPartial` is deliberate — non-`ok` non-range statuses should surface as auth errors before `consumeRangeResponse` complains about the body not being a partial.
|
||||
- `parseMetadata` returns a usable shape even when the origin omits `Content-Length` (S3 with chunked transfer, some edge caches). Callers that need exact size must call `getItemSize` against an origin that does set the header.
|
||||
- `readOnlyError` carries the sanitised key when known so the `key` field on the resulting error matches the key the caller would see in success cases.
|
||||
@@ -0,0 +1,140 @@
|
||||
import type { StorageMetadata } from "@suaveplan/storage/types";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import type { EtagCache } from "../etag-cache/etag-cache.js";
|
||||
import { recordRangeFallback } from "../otel/otel.js";
|
||||
import { buildRangeHeader, consumeRangeResponse } from "../range/range.js";
|
||||
import {
|
||||
type RetryWithBackoffOptions,
|
||||
retryWithBackoff,
|
||||
} from "../retry-helper/retry-helper.js";
|
||||
import type { EtagCacheEntry } from "../types/etag-cache.types.js";
|
||||
import {
|
||||
composeUrl,
|
||||
errorForStatus,
|
||||
type ResolvedHttpStorageConfig,
|
||||
resolveHeaders,
|
||||
} from "./http-storage.helpers.js";
|
||||
|
||||
export interface ReadContext {
|
||||
config: ResolvedHttpStorageConfig;
|
||||
cache: EtagCache | undefined;
|
||||
key: string;
|
||||
signal: AbortSignal | undefined;
|
||||
retryOptions: RetryWithBackoffOptions;
|
||||
}
|
||||
|
||||
/** Issue a GET, with ETag-cache support when the cache is wired. */
|
||||
export async function performGet(
|
||||
ctx: ReadContext
|
||||
): Promise<Uint8Array | undefined> {
|
||||
const url = composeUrl(ctx.config, ctx.key);
|
||||
const cached = ctx.cache?.get(ctx.key);
|
||||
const headers = await resolveHeaders(ctx.config.headers, ctx.key, "get");
|
||||
if (cached !== undefined) headers["If-None-Match"] = cached.etag;
|
||||
const response = await retryWithBackoff(
|
||||
() => fetchWithSignal(ctx, url, "GET", headers),
|
||||
ctx.retryOptions
|
||||
);
|
||||
if (response.status === 304 && cached !== undefined) return cached.body;
|
||||
if (response.status === 404 || response.status === 410) return undefined;
|
||||
if (!response.ok) throw errorForStatus(response.status, url, ctx.key);
|
||||
const body = new Uint8Array(await response.arrayBuffer());
|
||||
const etag = response.headers.get("etag");
|
||||
if (etag !== null) {
|
||||
const entry: EtagCacheEntry = {
|
||||
etag,
|
||||
body,
|
||||
lastModified: response.headers.get("last-modified") ?? undefined,
|
||||
};
|
||||
ctx.cache?.set(ctx.key, entry);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
/** Issue a HEAD; returns the response so callers can read headers. */
|
||||
export async function performHead(ctx: ReadContext): Promise<Response> {
|
||||
const url = composeUrl(ctx.config, ctx.key);
|
||||
const headers = await resolveHeaders(ctx.config.headers, ctx.key, "head");
|
||||
return retryWithBackoff(
|
||||
() => fetchWithSignal(ctx, url, "HEAD", headers),
|
||||
ctx.retryOptions
|
||||
);
|
||||
}
|
||||
|
||||
/** Parse a HEAD response into `StorageMetadata`. */
|
||||
export function parseMetadata(response: Response): StorageMetadata {
|
||||
const contentLength = response.headers.get("content-length");
|
||||
const lastModified = response.headers.get("last-modified");
|
||||
const modifiedMs =
|
||||
lastModified !== null && !Number.isNaN(Date.parse(lastModified))
|
||||
? Date.parse(lastModified)
|
||||
: Date.now();
|
||||
const size =
|
||||
contentLength !== null ? Number.parseInt(contentLength, 10) : -1;
|
||||
const metadata: StorageMetadata = {
|
||||
createdAt: modifiedMs,
|
||||
modifiedAt: modifiedMs,
|
||||
size: size >= 0 ? size : 0,
|
||||
};
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/** Range read with 200/206/416 disambiguation. */
|
||||
export async function performGetPartial(
|
||||
ctx: ReadContext,
|
||||
offset: number,
|
||||
length: number
|
||||
): Promise<Uint8Array | null> {
|
||||
const url = composeUrl(ctx.config, ctx.key);
|
||||
const headers = await resolveHeaders(
|
||||
ctx.config.headers,
|
||||
ctx.key,
|
||||
"getPartial"
|
||||
);
|
||||
headers["Range"] = buildRangeHeader(offset, length);
|
||||
const response = await retryWithBackoff(
|
||||
() => fetchWithSignal(ctx, url, "GET", headers),
|
||||
ctx.retryOptions
|
||||
);
|
||||
if (response.status === 404 || response.status === 410) return null;
|
||||
if (response.status === 401 || response.status === 403) {
|
||||
throw errorForStatus(response.status, url, ctx.key);
|
||||
}
|
||||
const outcome = await consumeRangeResponse(response, offset, length, url);
|
||||
if (outcome.fellBack) recordRangeFallback();
|
||||
return outcome.bytes;
|
||||
}
|
||||
|
||||
/** GET issued for streaming — returns the raw response so the caller can consume `.body`. */
|
||||
export async function performGetStream(ctx: ReadContext): Promise<Response> {
|
||||
const url = composeUrl(ctx.config, ctx.key);
|
||||
const headers = await resolveHeaders(ctx.config.headers, ctx.key, "get");
|
||||
const response = await fetchWithSignal(ctx, url, "GET", headers);
|
||||
if (!response.ok) throw errorForStatus(response.status, url, ctx.key);
|
||||
return response;
|
||||
}
|
||||
|
||||
/** Read the response body fully as a stream; identical to `performGetStream` but does not throw on non-200. */
|
||||
export async function fetchWithSignal(
|
||||
ctx: ReadContext,
|
||||
url: string,
|
||||
method: string,
|
||||
headers: Record<string, string>
|
||||
): Promise<Response> {
|
||||
const init: RequestInit = { method, headers };
|
||||
if (ctx.signal !== undefined) init.signal = ctx.signal;
|
||||
return ctx.config.fetch(url, init);
|
||||
}
|
||||
|
||||
/** Build the `STORAGE_HTTP.READ_ONLY` error with a documented message. */
|
||||
export function readOnlyError(method: string, key?: string): StorageHttpError {
|
||||
const message =
|
||||
key !== undefined
|
||||
? `${method} not supported in read-only mode (no uploader configured) for key '${key}'`
|
||||
: `${method} not supported in read-only mode`;
|
||||
const opts = key !== undefined ? { key } : {};
|
||||
return new StorageHttpError(STORAGE_HTTP_CODES.READ_ONLY, message, opts);
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
import { readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { AbstractStorage } from "@suaveplan/storage/abstract";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import type { FetchLike } from "../types/fetch.types.js";
|
||||
import type { HttpUploader } from "../types/uploader.types.js";
|
||||
import { HttpStorage } from "./http-storage.js";
|
||||
|
||||
const baseUrl = "https://cdn.example.com/start/here/";
|
||||
|
||||
describe("HttpStorage spec conformance", () => {
|
||||
it("declares the documented capabilities matrix", { timeout: 5000 }, () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
expect(storage.capabilities).toEqual({
|
||||
partialReads: true,
|
||||
partialWrites: false,
|
||||
nativeStreams: true,
|
||||
atomicWrites: false,
|
||||
contentAddressable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("instance is structurally an AbstractStorage", { timeout: 5000 }, () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
expect(storage).toBeInstanceOf(AbstractStorage);
|
||||
});
|
||||
|
||||
it("every STORAGE_HTTP code is reachable from a real call site", {
|
||||
timeout: 15000,
|
||||
}, async () => {
|
||||
const reached = new Set<string>();
|
||||
const collect = async (fn: () => Promise<unknown>): Promise<void> => {
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
if (err instanceof StorageHttpError) reached.add(err.code);
|
||||
}
|
||||
};
|
||||
|
||||
// INVALID_OPTIONS — bad baseUrl.
|
||||
await collect(async () => new HttpStorage({ baseUrl: "no-slash" }));
|
||||
// INVALID_KEY — traversal.
|
||||
await collect(async () => {
|
||||
const s = new HttpStorage({ baseUrl });
|
||||
await s.get("../etc/passwd");
|
||||
});
|
||||
// KEY_TOO_LONG — overlong.
|
||||
await collect(async () => {
|
||||
const s = new HttpStorage({ baseUrl });
|
||||
await s.get("a".repeat(2049));
|
||||
});
|
||||
// READ_ONLY — list.
|
||||
await collect(async () => new HttpStorage({ baseUrl }).list());
|
||||
// ABORTED — pre-aborted signal.
|
||||
await collect(async () => {
|
||||
const c = new AbortController();
|
||||
c.abort();
|
||||
await new HttpStorage({ baseUrl }).get("k", c.signal);
|
||||
});
|
||||
// NOT_FOUND — 416 from Range.
|
||||
const not404Fetch: FetchLike = async () =>
|
||||
new Response(null, { status: 416 });
|
||||
await collect(async () => {
|
||||
const s = new HttpStorage({ baseUrl, fetch: not404Fetch });
|
||||
await s.getPartial("k", 100, 5);
|
||||
});
|
||||
// UNAUTHORIZED.
|
||||
await collect(async () => {
|
||||
const s = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: async () => new Response(null, { status: 401 }),
|
||||
});
|
||||
await s.get("k");
|
||||
});
|
||||
// FORBIDDEN.
|
||||
await collect(async () => {
|
||||
const s = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: async () => new Response(null, { status: 403 }),
|
||||
});
|
||||
await s.get("k");
|
||||
});
|
||||
// UPSTREAM_ERROR — 500 after exhaustion.
|
||||
await collect(async () => {
|
||||
const s = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: async () => new Response(null, { status: 500 }),
|
||||
retry: { maxAttempts: 1, honourRetryAfter: false },
|
||||
});
|
||||
await s.get("k");
|
||||
});
|
||||
// NETWORK_ERROR — fetch reject.
|
||||
await collect(async () => {
|
||||
const uploader: HttpUploader = {
|
||||
put: async () => {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR,
|
||||
"down"
|
||||
);
|
||||
},
|
||||
};
|
||||
const s = new HttpStorage({ baseUrl, uploader });
|
||||
await s.set("k", new Uint8Array([1]));
|
||||
});
|
||||
|
||||
expect(reached.has(STORAGE_HTTP_CODES.INVALID_OPTIONS)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.INVALID_KEY)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.KEY_TOO_LONG)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.READ_ONLY)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.ABORTED)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.NOT_FOUND)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.UNAUTHORIZED)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.FORBIDDEN)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.UPSTREAM_ERROR)).toBe(true);
|
||||
expect(reached.has(STORAGE_HTTP_CODES.NETWORK_ERROR)).toBe(true);
|
||||
});
|
||||
|
||||
it("src tree contains no node: or bun: imports", { timeout: 5000 }, () => {
|
||||
const srcDir = join(__dirname, "..");
|
||||
const offenders: string[] = [];
|
||||
walk(srcDir, (file) => {
|
||||
if (
|
||||
!file.endsWith(".ts") ||
|
||||
file.endsWith(".test.ts") ||
|
||||
file.endsWith(".spec-conformance.test.ts")
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const text = readFileSync(file, "utf8");
|
||||
if (/from\s+['"](node:|bun:)/.test(text)) offenders.push(file);
|
||||
});
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
function walk(dir: string, visit: (file: string) => void): void {
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
const s = statSync(full);
|
||||
if (s.isDirectory()) walk(full, visit);
|
||||
else visit(full);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
# http-storage.streams
|
||||
|
||||
## Purpose
|
||||
|
||||
Builds the `ReadableStream` and `WritableStream` instances `HttpStorage` returns from `createReadStream` / `createWriteStream`. Splits the stream plumbing out of the class body so each side can be unit-tested without instantiating the public adapter.
|
||||
|
||||
`buildReadStream` wraps `Response.body` directly — no client-side buffering — so callers see the upstream chunk boundaries. `buildWriteStream` buffers chunks until `close` and then dispatches `uploader.put` with the concatenated body; without an uploader it throws `STORAGE_HTTP.READ_ONLY` synchronously at construction.
|
||||
|
||||
## Features
|
||||
|
||||
- Native streaming: `buildReadStream` returns `Response.body` chunks 1:1 to the consumer.
|
||||
- Cancellation: errors mid-stream surface via `controller.error`; aborts propagate through the underlying fetch's signal.
|
||||
- Synchronous READ_ONLY enforcement: `buildWriteStream` throws before returning the stream so callers don't observe a half-built object.
|
||||
- `abort()` handler drops buffered chunks promptly so memory does not linger after cancellation.
|
||||
- `upstreamError` helper used by `HttpStorage` for non-`ok` HEAD/GET responses that aren't already mapped by `errorForStatus`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { buildReadStream, buildWriteStream } from "@suaveplan/storage-http";
|
||||
|
||||
const stream = buildReadStream(ctx, signal);
|
||||
for await (const chunk of stream) process(chunk);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { buildReadStream, buildWriteStream, upstreamError } from "@suaveplan/storage-http";
|
||||
|
||||
const write = buildWriteStream({
|
||||
uploader,
|
||||
key,
|
||||
resolveHeaders: () => resolveHeaders(headersResolver, key, "set"),
|
||||
signal,
|
||||
onWritten: (bytes) => emitOtel(bytes),
|
||||
});
|
||||
const writer = write.getWriter();
|
||||
await writer.write(chunkA);
|
||||
await writer.write(chunkB);
|
||||
await writer.close();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `buildReadStream(ctx: ReadContext, signal?: AbortSignal): ReadableStream<Uint8Array>`
|
||||
|
||||
Starts the underlying `GET` on first read. Errors propagate through `controller.error`. `Response.body === null` closes the stream cleanly without enqueueing.
|
||||
|
||||
### `interface WriteStreamArgs`
|
||||
|
||||
`{ uploader, key, resolveHeaders, signal, onWritten }` — `resolveHeaders` is invoked once at `close` time so the request gets the latest token from the caller's resolver.
|
||||
|
||||
### `buildWriteStream(args: WriteStreamArgs): WritableStream<Uint8Array>`
|
||||
|
||||
Throws `STORAGE_HTTP.READ_ONLY` immediately when `args.uploader` is undefined. On `close`, concatenates buffered chunks, calls `uploader.put`, then invokes `onWritten(bytes)` for OTel emission and ETag-cache invalidation.
|
||||
|
||||
### `upstreamError(method, url, key, status): StorageHttpError`
|
||||
|
||||
`STORAGE_HTTP.UPSTREAM_ERROR` with the method/url/key/status carried as discriminants.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Buffered-then-flush is the right tradeoff for HTTP `PUT`: real chunked uploads require provider-specific multipart logic (S3 / R2 / GCS) which is out of scope for the core adapter. Provider packages may override `createWriteStream`.
|
||||
- `abort()` clears `chunks` rather than reusing the array — clearing length forces garbage collection of every enqueued buffer once the writer releases its reference.
|
||||
- `releaseLock()` on the response body reader runs in `finally` so a mid-stream error doesn't leak the lock.
|
||||
- `onWritten` is invoked **after** `uploader.put` resolves so the cache invalidation never runs against an in-flight write.
|
||||
@@ -0,0 +1,105 @@
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import type { HttpUploader } from "../types/uploader.types.js";
|
||||
import { throwIfAborted } from "./http-storage.helpers.js";
|
||||
import {
|
||||
performGetStream,
|
||||
type ReadContext,
|
||||
readOnlyError,
|
||||
} from "./http-storage.ops.js";
|
||||
|
||||
/**
|
||||
* Build a `ReadableStream` that, on first read, issues the GET and pumps
|
||||
* the response body chunks to the consumer. Errors mid-stream surface via
|
||||
* `controller.error`.
|
||||
*/
|
||||
export function buildReadStream(
|
||||
ctx: ReadContext,
|
||||
signal: AbortSignal | undefined
|
||||
): ReadableStream<Uint8Array> {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start: async (controller) => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const response = await performGetStream(ctx);
|
||||
const body = response.body;
|
||||
if (body === null) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
const reader = body.getReader();
|
||||
try {
|
||||
while (true) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
controller.enqueue(chunk.value);
|
||||
}
|
||||
controller.close();
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
} catch (err) {
|
||||
controller.error(err);
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export interface WriteStreamArgs {
|
||||
uploader: HttpUploader | undefined;
|
||||
key: string;
|
||||
resolveHeaders: () => Promise<Record<string, string>>;
|
||||
signal: AbortSignal | undefined;
|
||||
onWritten: (bytes: number) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `WritableStream` that buffers chunks until `close`, then calls
|
||||
* `uploader.put` with the concatenated body. Throws `READ_ONLY` synchronously
|
||||
* when no uploader is wired.
|
||||
*/
|
||||
export function buildWriteStream(
|
||||
args: WriteStreamArgs
|
||||
): WritableStream<Uint8Array> {
|
||||
const { uploader } = args;
|
||||
if (uploader === undefined)
|
||||
throw readOnlyError("createWriteStream", args.key);
|
||||
const chunks: Uint8Array[] = [];
|
||||
return new WritableStream<Uint8Array>({
|
||||
write: (chunk) => {
|
||||
chunks.push(chunk);
|
||||
},
|
||||
close: async () => {
|
||||
let total = 0;
|
||||
for (const c of chunks) total += c.byteLength;
|
||||
const body = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
body.set(c, offset);
|
||||
offset += c.byteLength;
|
||||
}
|
||||
const headers = await args.resolveHeaders();
|
||||
await uploader.put(args.key, body, headers, args.signal);
|
||||
args.onWritten(body.byteLength);
|
||||
},
|
||||
abort: () => {
|
||||
chunks.length = 0;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Surface an unexpected upstream status as `STORAGE_HTTP.UPSTREAM_ERROR`. */
|
||||
export function upstreamError(
|
||||
method: string,
|
||||
url: string,
|
||||
key: string,
|
||||
status: number
|
||||
): StorageHttpError {
|
||||
return new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR,
|
||||
`${method} upstream ${status}`,
|
||||
{ url, key, httpStatus: status }
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { setStorageHttpMeterProvider } from "../otel/otel.js";
|
||||
import type { FetchLike } from "../types/fetch.types.js";
|
||||
import { createHttpStorage, HttpStorage } from "./http-storage.js";
|
||||
|
||||
const baseUrl = "https://cdn.example.com/start/here/";
|
||||
|
||||
function fetchOf(...responses: Response[]): FetchLike & { calls: number } {
|
||||
let i = 0;
|
||||
const fn = (async () =>
|
||||
responses[i++] ?? new Response(null, { status: 200 })) as FetchLike & {
|
||||
calls: number;
|
||||
};
|
||||
Object.defineProperty(fn, "calls", { get: () => i });
|
||||
return fn;
|
||||
}
|
||||
|
||||
describe("HttpStorage construction", () => {
|
||||
beforeEach(() => setStorageHttpMeterProvider(undefined));
|
||||
afterEach(() => setStorageHttpMeterProvider(undefined));
|
||||
|
||||
it("implements every AbstractStorage abstract method", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
expect(typeof storage.get).toBe("function");
|
||||
expect(typeof storage.set).toBe("function");
|
||||
expect(typeof storage.delete).toBe("function");
|
||||
expect(typeof storage.exists).toBe("function");
|
||||
expect(typeof storage.list).toBe("function");
|
||||
expect(typeof storage.clear).toBe("function");
|
||||
expect(typeof storage.getWithMetadata).toBe("function");
|
||||
expect(typeof storage.getMetadata).toBe("function");
|
||||
expect(typeof storage.getSize).toBe("function");
|
||||
expect(typeof storage.getPartial).toBe("function");
|
||||
expect(typeof storage.setPartial).toBe("function");
|
||||
expect(typeof storage.getItemSize).toBe("function");
|
||||
expect(typeof storage.createReadStream).toBe("function");
|
||||
expect(typeof storage.createWriteStream).toBe("function");
|
||||
});
|
||||
|
||||
it("rejects baseUrl missing trailing slash with INVALID_OPTIONS", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
new HttpStorage({ baseUrl: "https://cdn.example.com/start/here" });
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.INVALID_OPTIONS
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-http(s) baseUrl with INVALID_OPTIONS", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
new HttpStorage({ baseUrl: "ftp://example.com/" });
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.INVALID_OPTIONS
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts the documented option shape", { timeout: 5000 }, () => {
|
||||
const storage = createHttpStorage({
|
||||
baseUrl,
|
||||
headers: { Authorization: "Bearer x" },
|
||||
etagCache: { capacity: 16 },
|
||||
});
|
||||
expect(storage).toBeInstanceOf(HttpStorage);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HttpStorage read operations", () => {
|
||||
it("GET round-trips body bytes from baseUrl + key", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(new Uint8Array([1, 2, 3])));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const body = await storage.get("a/b/test.doc");
|
||||
expect(body).toBeInstanceOf(Uint8Array);
|
||||
expect(Array.from(body as Uint8Array)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("GET returns undefined on 404 without throwing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 404 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.get("missing")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("GET returns undefined on 410 Gone", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 410 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.get("gone")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("GET surfaces 401 as UNAUTHORIZED", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 401 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
try {
|
||||
await storage.get("x");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.UNAUTHORIZED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("HEAD 200 yields exists=true and parsed metadata", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const headers = new Headers({
|
||||
"content-length": "42",
|
||||
"content-type": "text/plain",
|
||||
"last-modified": new Date(2026, 0, 1).toUTCString(),
|
||||
etag: '"abc"',
|
||||
});
|
||||
const fetchFn = fetchOf(
|
||||
new Response(null, { status: 200, headers }),
|
||||
new Response(null, { status: 200, headers })
|
||||
);
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.exists("k")).toBe(true);
|
||||
const meta = await storage.getMetadata("k");
|
||||
expect(meta?.size).toBe(42);
|
||||
});
|
||||
|
||||
it("HEAD 404 yields exists=false without throwing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 404 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.exists("missing")).toBe(false);
|
||||
});
|
||||
|
||||
it("HEAD 500 surfaces UPSTREAM_ERROR", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(null, { status: 500 }),
|
||||
new Response(null, { status: 500 }),
|
||||
new Response(null, { status: 500 })
|
||||
);
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
retry: { maxAttempts: 1, honourRetryAfter: false },
|
||||
});
|
||||
try {
|
||||
await storage.exists("x");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("getMetadata returns null on 404", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 404 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.getMetadata("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("getMetadata throws UPSTREAM_ERROR on 5xx after exhaustion", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 500 }));
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
retry: { maxAttempts: 1, honourRetryAfter: false },
|
||||
});
|
||||
try {
|
||||
await storage.getMetadata("x");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("getMetadata defaults size to 0 when Content-Length absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 200 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const meta = await storage.getMetadata("k");
|
||||
expect(meta?.size).toBe(0);
|
||||
});
|
||||
|
||||
it("getWithMetadata composes get + getMetadata", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const headers = new Headers({ "content-length": "3" });
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([1, 2, 3])),
|
||||
new Response(null, { status: 200, headers })
|
||||
);
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const result = await storage.getWithMetadata("k");
|
||||
expect(result?.metadata.size).toBe(3);
|
||||
expect(Array.from(result?.value ?? new Uint8Array())).toEqual([
|
||||
1, 2, 3,
|
||||
]);
|
||||
});
|
||||
|
||||
it("getWithMetadata returns null when get is undefined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 404 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.getWithMetadata("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("getWithMetadata synthesises metadata when HEAD returns null", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([9])),
|
||||
new Response(null, { status: 404 })
|
||||
);
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const result = await storage.getWithMetadata("k");
|
||||
expect(result?.value.byteLength).toBe(1);
|
||||
expect(result?.metadata.size).toBe(1);
|
||||
});
|
||||
|
||||
it("getItemSize returns the parsed size or null", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const headers = new Headers({ "content-length": "10" });
|
||||
const fetchFn = fetchOf(new Response(null, { status: 200, headers }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.getItemSize("k")).toBe(10);
|
||||
const storage2 = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchOf(new Response(null, { status: 404 })),
|
||||
});
|
||||
expect(await storage2.getItemSize("missing")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("HttpStorage getPartial", () => {
|
||||
it("206 returns partial bytes verbatim", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([2, 3]), { status: 206 })
|
||||
);
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const bytes = await storage.getPartial("k", 1, 2);
|
||||
expect(Array.from(bytes as Uint8Array)).toEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("returns null on 404", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 404 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.getPartial("missing", 0, 1)).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces 401 distinctly from range disambiguation", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 401 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
try {
|
||||
await storage.getPartial("k", 0, 1);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.UNAUTHORIZED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("200 fallback returns sliced bytes", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([10, 20, 30, 40]), { status: 200 })
|
||||
);
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const bytes = await storage.getPartial("k", 1, 2);
|
||||
expect(Array.from(bytes as Uint8Array)).toEqual([20, 30]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HttpStorage streams and abort", () => {
|
||||
it("createReadStream returns Response.body without buffering", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const chunks = [
|
||||
new Uint8Array([1]),
|
||||
new Uint8Array([2]),
|
||||
new Uint8Array([3]),
|
||||
];
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (const c of chunks) controller.enqueue(c);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
const fetchFn = fetchOf(new Response(body, { status: 200 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const out: number[] = [];
|
||||
const reader = storage.createReadStream("k").getReader();
|
||||
while (true) {
|
||||
const r = await reader.read();
|
||||
if (r.done) break;
|
||||
out.push(...r.value);
|
||||
}
|
||||
expect(out).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("createReadStream closes cleanly when body is null", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 200 }));
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const reader = storage.createReadStream("k").getReader();
|
||||
const r = await reader.read();
|
||||
expect(r.done).toBe(true);
|
||||
});
|
||||
|
||||
it("createReadStream errors when GET status is non-ok", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(new Response(null, { status: 500 }));
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const reader = storage.createReadStream("k").getReader();
|
||||
try {
|
||||
await reader.read();
|
||||
expect.fail("expected error");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(StorageHttpError);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects with STORAGE_HTTP.ABORTED when signal is pre-aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const fetchFn = vi.fn();
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn as FetchLike,
|
||||
});
|
||||
try {
|
||||
await storage.get("k", controller.signal);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.ABORTED
|
||||
);
|
||||
expect(fetchFn).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
it("propagates mid-flight abort into the underlying fetch", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const observed: AbortSignal[] = [];
|
||||
const fetchFn: FetchLike = async (_input, init) => {
|
||||
if (init?.signal !== undefined && init.signal !== null) {
|
||||
observed.push(init.signal);
|
||||
}
|
||||
return new Response(new Uint8Array([1]));
|
||||
};
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
const controller = new AbortController();
|
||||
await storage.get("k", controller.signal);
|
||||
expect(observed[0]).toBe(controller.signal);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,296 @@
|
||||
import { AbstractStorage } from "@suaveplan/storage/abstract";
|
||||
import type {
|
||||
StorageCapabilities,
|
||||
StorageListOptions,
|
||||
StorageListResult,
|
||||
StorageMetadata,
|
||||
StorageOptions,
|
||||
StorageResult,
|
||||
} from "@suaveplan/storage/types";
|
||||
import { EtagCache } from "../etag-cache/etag-cache.js";
|
||||
import { sanitizeKey } from "../key-sanitizer/key-sanitizer.js";
|
||||
import { recordOp, type StorageHttpOp } from "../otel/otel.js";
|
||||
import type { HttpStorageOptions } from "../types/http-storage.types.js";
|
||||
import {
|
||||
buildRetryOptions,
|
||||
composeUrl,
|
||||
outcomeFromError,
|
||||
type ResolvedHttpStorageConfig,
|
||||
resolveHeaders,
|
||||
resolveHttpStorageConfig,
|
||||
throwIfAborted,
|
||||
} from "./http-storage.helpers.js";
|
||||
import {
|
||||
parseMetadata,
|
||||
performGet,
|
||||
performGetPartial,
|
||||
performHead,
|
||||
type ReadContext,
|
||||
readOnlyError,
|
||||
} from "./http-storage.ops.js";
|
||||
import {
|
||||
buildReadStream,
|
||||
buildWriteStream,
|
||||
upstreamError,
|
||||
} from "./http-storage.streams.js";
|
||||
|
||||
/**
|
||||
* Universal HTTP/CDN-backed `AbstractStorage`. Reads via `GET`/`HEAD`/`Range`
|
||||
* against `<baseUrl><encodedKey>`, with optional writes through an injected
|
||||
* `HttpUploader`. See `README.md` for the full feature matrix.
|
||||
*/
|
||||
export class HttpStorage extends AbstractStorage {
|
||||
readonly #config: ResolvedHttpStorageConfig;
|
||||
readonly #cache: EtagCache | undefined;
|
||||
|
||||
public override readonly capabilities: StorageCapabilities = {
|
||||
partialReads: true,
|
||||
partialWrites: false,
|
||||
nativeStreams: true,
|
||||
atomicWrites: false,
|
||||
contentAddressable: false,
|
||||
};
|
||||
|
||||
public constructor(options: HttpStorageOptions) {
|
||||
super();
|
||||
this.#config = resolveHttpStorageConfig(options);
|
||||
this.#cache =
|
||||
options.etagCache !== undefined
|
||||
? new EtagCache(options.etagCache)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
#readContext(key: string, signal: AbortSignal | undefined): ReadContext {
|
||||
return {
|
||||
config: this.#config,
|
||||
cache: this.#cache,
|
||||
key,
|
||||
signal,
|
||||
retryOptions: buildRetryOptions(this.#config.options, signal),
|
||||
};
|
||||
}
|
||||
|
||||
async #wrap<T>(
|
||||
op: StorageHttpOp,
|
||||
signal: AbortSignal | undefined,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const value = await fn();
|
||||
recordOp({
|
||||
op,
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: Date.now() - start,
|
||||
});
|
||||
return value;
|
||||
} catch (err) {
|
||||
const { outcome, statusClass } = outcomeFromError(err);
|
||||
recordOp({
|
||||
op,
|
||||
outcome,
|
||||
statusClass,
|
||||
durationMs: Date.now() - start,
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
public override async get(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<Uint8Array | undefined> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("get", signal, () =>
|
||||
performGet(this.#readContext(safe, signal))
|
||||
);
|
||||
}
|
||||
|
||||
public override async exists(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<boolean> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("exists", signal, async () => {
|
||||
const response = await performHead(this.#readContext(safe, signal));
|
||||
if (response.status === 404 || response.status === 410)
|
||||
return false;
|
||||
if (!response.ok) {
|
||||
throw upstreamError(
|
||||
"exists",
|
||||
composeUrl(this.#config, safe),
|
||||
safe,
|
||||
response.status
|
||||
);
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
public override async getMetadata(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageMetadata | null> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("getMetadata", signal, async () => {
|
||||
const response = await performHead(this.#readContext(safe, signal));
|
||||
if (response.status === 404 || response.status === 410) return null;
|
||||
if (!response.ok) {
|
||||
throw upstreamError(
|
||||
"getMetadata",
|
||||
composeUrl(this.#config, safe),
|
||||
safe,
|
||||
response.status
|
||||
);
|
||||
}
|
||||
return parseMetadata(response);
|
||||
});
|
||||
}
|
||||
|
||||
public override async getWithMetadata(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageResult<Uint8Array> | null> {
|
||||
const safe = sanitizeKey(key);
|
||||
const value = await this.get(safe, signal);
|
||||
if (value === undefined) return null;
|
||||
const metadata = (await this.getMetadata(safe, signal)) ?? {
|
||||
createdAt: Date.now(),
|
||||
modifiedAt: Date.now(),
|
||||
size: value.byteLength,
|
||||
};
|
||||
return { value, metadata };
|
||||
}
|
||||
|
||||
public override async getItemSize(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<number | null> {
|
||||
const meta = await this.getMetadata(key, signal);
|
||||
return meta?.size ?? null;
|
||||
}
|
||||
|
||||
public override async getPartial(
|
||||
key: string,
|
||||
offset: number,
|
||||
length: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<Uint8Array | null> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("getPartial", signal, () =>
|
||||
performGetPartial(this.#readContext(safe, signal), offset, length)
|
||||
);
|
||||
}
|
||||
|
||||
public override createReadStream(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): ReadableStream<Uint8Array> {
|
||||
const safe = sanitizeKey(key);
|
||||
return buildReadStream(this.#readContext(safe, signal), signal);
|
||||
}
|
||||
|
||||
public override async set(
|
||||
key: string,
|
||||
value: Uint8Array,
|
||||
_options?: StorageOptions,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const safe = sanitizeKey(key);
|
||||
await this.validateValueSize(safe, value, signal);
|
||||
const uploader = this.#config.options.uploader;
|
||||
if (uploader === undefined) throw readOnlyError("set", safe);
|
||||
await this.#wrap("set", signal, async () => {
|
||||
const headers = await resolveHeaders(
|
||||
this.#config.headers,
|
||||
safe,
|
||||
"set"
|
||||
);
|
||||
await uploader.put(safe, value, headers, signal);
|
||||
recordOp({
|
||||
op: "set",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 0,
|
||||
bytes: value.byteLength,
|
||||
direction: "write",
|
||||
});
|
||||
this.#cache?.delete(safe);
|
||||
});
|
||||
}
|
||||
|
||||
public override async delete(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<boolean> {
|
||||
const safe = sanitizeKey(key);
|
||||
const uploader = this.#config.options.uploader;
|
||||
if (uploader?.delete === undefined) throw readOnlyError("delete", safe);
|
||||
return this.#wrap("delete", signal, async () => {
|
||||
await uploader.delete?.(safe, signal);
|
||||
this.#cache?.delete(safe);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
public override list(
|
||||
_options?: StorageListOptions,
|
||||
_signal?: AbortSignal
|
||||
): Promise<StorageListResult> {
|
||||
return Promise.reject(readOnlyError("list"));
|
||||
}
|
||||
|
||||
public override clear(_signal?: AbortSignal): Promise<void> {
|
||||
return Promise.reject(readOnlyError("clear"));
|
||||
}
|
||||
|
||||
public override getSize(_signal?: AbortSignal): Promise<number> {
|
||||
return Promise.reject(readOnlyError("getSize"));
|
||||
}
|
||||
|
||||
public override async setPartial(
|
||||
key: string,
|
||||
offset: number,
|
||||
bytes: Uint8Array,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
if (this.#config.options.uploader === undefined) {
|
||||
throw readOnlyError("setPartial", sanitizeKey(key));
|
||||
}
|
||||
await this.defaultSetPartial(key, offset, bytes, signal);
|
||||
}
|
||||
|
||||
public override createWriteStream(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): WritableStream<Uint8Array> {
|
||||
const safe = sanitizeKey(key);
|
||||
return buildWriteStream({
|
||||
uploader: this.#config.options.uploader,
|
||||
key: safe,
|
||||
resolveHeaders: () =>
|
||||
resolveHeaders(this.#config.headers, safe, "set"),
|
||||
signal,
|
||||
onWritten: (bytes) => this.#onWritten(safe, bytes),
|
||||
});
|
||||
}
|
||||
|
||||
#onWritten(key: string, bytes: number): void {
|
||||
recordOp({
|
||||
op: "createWriteStream",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 0,
|
||||
bytes,
|
||||
direction: "write",
|
||||
});
|
||||
this.#cache?.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
/** Convenience factory mirroring `createNodeFsStorage`. */
|
||||
export function createHttpStorage(options: HttpStorageOptions): HttpStorage {
|
||||
return new HttpStorage(options);
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import type { FetchLike } from "../types/fetch.types.js";
|
||||
import type { HttpUploader } from "../types/uploader.types.js";
|
||||
import { HttpStorage } from "./http-storage.js";
|
||||
|
||||
const baseUrl = "https://cdn.example.com/start/here/";
|
||||
|
||||
function fetchOf(...responses: Response[]): FetchLike & { calls: number } {
|
||||
let i = 0;
|
||||
const fn = (async () =>
|
||||
responses[i++] ?? new Response(null, { status: 200 })) as FetchLike & {
|
||||
calls: number;
|
||||
};
|
||||
Object.defineProperty(fn, "calls", { get: () => i });
|
||||
return fn;
|
||||
}
|
||||
|
||||
function controlledTimers(): { timers: TimersLike; flushNext(): void } {
|
||||
const pending: { fn: () => void; id: number }[] = [];
|
||||
let nextId = 1;
|
||||
return {
|
||||
timers: {
|
||||
setTimer: (handler) => {
|
||||
const id = nextId++;
|
||||
pending.push({ fn: handler, id });
|
||||
return id;
|
||||
},
|
||||
clearTimer: (id) => {
|
||||
const idx = pending.findIndex((p) => p.id === id);
|
||||
if (idx >= 0) pending.splice(idx, 1);
|
||||
},
|
||||
},
|
||||
flushNext(): void {
|
||||
const next = pending.shift();
|
||||
if (next !== undefined) next.fn();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("HttpStorage write operations", () => {
|
||||
it("set invokes injected uploader.put exactly once", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const put = vi.fn(async () => undefined);
|
||||
const uploader: HttpUploader = { put };
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
uploader,
|
||||
headers: { "x-flow": "test" },
|
||||
});
|
||||
await storage.set("a/b.bin", new Uint8Array([1, 2, 3]));
|
||||
expect(put).toHaveBeenCalledTimes(1);
|
||||
const [key, body, headers] = put.mock.calls[0] as [
|
||||
string,
|
||||
Uint8Array,
|
||||
Record<string, string>,
|
||||
];
|
||||
expect(key).toBe("a/b.bin");
|
||||
expect(Array.from(body)).toEqual([1, 2, 3]);
|
||||
expect(headers["x-flow"]).toBe("test");
|
||||
});
|
||||
|
||||
it("set without uploader throws READ_ONLY", { timeout: 5000 }, async () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
try {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("delete invokes uploader.delete when defined", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const del = vi.fn(async () => undefined);
|
||||
const uploader: HttpUploader = {
|
||||
put: async () => undefined,
|
||||
delete: del,
|
||||
};
|
||||
const storage = new HttpStorage({ baseUrl, uploader });
|
||||
expect(await storage.delete("k")).toBe(true);
|
||||
expect(del).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("delete throws READ_ONLY when uploader lacks delete", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const uploader: HttpUploader = { put: async () => undefined };
|
||||
const storage = new HttpStorage({ baseUrl, uploader });
|
||||
try {
|
||||
await storage.delete("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("delete throws READ_ONLY when no uploader at all", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
try {
|
||||
await storage.delete("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("list always throws READ_ONLY", { timeout: 5000 }, async () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
try {
|
||||
await storage.list();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("clear always throws READ_ONLY", { timeout: 5000 }, async () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
try {
|
||||
await storage.clear();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("getSize always throws READ_ONLY", { timeout: 5000 }, async () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
try {
|
||||
await storage.getSize();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("setPartial throws READ_ONLY without uploader", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
try {
|
||||
await storage.setPartial("k", 0, new Uint8Array([1]));
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.READ_ONLY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("setPartial falls through to defaultSetPartial with uploader", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const stored = new Map<string, Uint8Array>();
|
||||
const fetchFn: FetchLike = async (input) => {
|
||||
const url = typeof input === "string" ? input : input.toString();
|
||||
const key = url.slice(baseUrl.length);
|
||||
const bytes = stored.get(key);
|
||||
return bytes !== undefined
|
||||
? new Response(bytes, { status: 200 })
|
||||
: new Response(null, { status: 404 });
|
||||
};
|
||||
const uploader: HttpUploader = {
|
||||
put: async (key, body) => {
|
||||
stored.set(key, body as Uint8Array);
|
||||
},
|
||||
};
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn, uploader });
|
||||
await storage.set("k", new Uint8Array([1, 2, 3, 4]));
|
||||
await storage.setPartial("k", 1, new Uint8Array([9, 9]));
|
||||
const out = stored.get("k");
|
||||
expect(Array.from(out as Uint8Array)).toEqual([1, 9, 9, 4]);
|
||||
});
|
||||
|
||||
it("createWriteStream throws READ_ONLY without uploader", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const storage = new HttpStorage({ baseUrl });
|
||||
expect(() => storage.createWriteStream("k")).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("createWriteStream buffers chunks and dispatches put on close", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const put = vi.fn(async () => undefined);
|
||||
const uploader: HttpUploader = { put };
|
||||
const storage = new HttpStorage({ baseUrl, uploader });
|
||||
const ws = storage.createWriteStream("k");
|
||||
const writer = ws.getWriter();
|
||||
await writer.write(new Uint8Array([1, 2]));
|
||||
await writer.write(new Uint8Array([3]));
|
||||
await writer.close();
|
||||
expect(put).toHaveBeenCalledTimes(1);
|
||||
const [, body] = put.mock.calls[0] as [string, Uint8Array];
|
||||
expect(Array.from(body)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("createWriteStream abort drops buffered chunks", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const put = vi.fn(async () => undefined);
|
||||
const uploader: HttpUploader = { put };
|
||||
const storage = new HttpStorage({ baseUrl, uploader });
|
||||
const ws = storage.createWriteStream("k");
|
||||
const writer = ws.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
await writer.abort(new Error("nope"));
|
||||
expect(put).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("HttpStorage retry and headers", () => {
|
||||
it("503 with Retry-After is retried then 200 is returned", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(null, {
|
||||
status: 503,
|
||||
headers: { "retry-after": "0" },
|
||||
}),
|
||||
new Response(new Uint8Array([5]))
|
||||
);
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1 },
|
||||
});
|
||||
const body = await storage.get("k");
|
||||
expect(Array.from(body as Uint8Array)).toEqual([5]);
|
||||
expect(fetchFn.calls).toBe(2);
|
||||
});
|
||||
|
||||
it("invokes async headers resolver per request", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const calls: string[] = [];
|
||||
const fetchFn: FetchLike = async (_input, init) => {
|
||||
const headers = init?.headers as Record<string, string>;
|
||||
calls.push(headers["x-token"] ?? "missing");
|
||||
return new Response(new Uint8Array([1]));
|
||||
};
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
headers: async (key) => ({ "x-token": `t-${key}` }),
|
||||
});
|
||||
await storage.get("a");
|
||||
await storage.get("b");
|
||||
expect(calls).toEqual(["t-a", "t-b"]);
|
||||
});
|
||||
|
||||
it("uses ETag cache to send If-None-Match and accept 304", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const body = new Uint8Array([7, 7, 7]);
|
||||
const headers = new Headers({ etag: '"v1"' });
|
||||
let stage = 0;
|
||||
const fetchFn: FetchLike = async (_input, init) => {
|
||||
stage++;
|
||||
if (stage === 1)
|
||||
return new Response(body, { status: 200, headers });
|
||||
const ifNoneMatch = (init?.headers as Record<string, string>)[
|
||||
"If-None-Match"
|
||||
];
|
||||
expect(ifNoneMatch).toBe('"v1"');
|
||||
return new Response(null, { status: 304 });
|
||||
};
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
etagCache: { capacity: 4 },
|
||||
});
|
||||
const first = await storage.get("k");
|
||||
const second = await storage.get("k");
|
||||
expect(Array.from(first as Uint8Array)).toEqual([7, 7, 7]);
|
||||
expect(Array.from(second as Uint8Array)).toEqual([7, 7, 7]);
|
||||
});
|
||||
|
||||
it("set invalidates the ETag cache entry", { timeout: 5000 }, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([1]), {
|
||||
status: 200,
|
||||
headers: { etag: '"v1"' },
|
||||
})
|
||||
);
|
||||
const put = vi.fn(async () => undefined);
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
uploader: { put },
|
||||
etagCache: { capacity: 4 },
|
||||
});
|
||||
await storage.get("k");
|
||||
await storage.set("k", new Uint8Array([9]));
|
||||
expect(put).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("createWriteStream + uploader invalidates ETag cache on close", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([1]), {
|
||||
status: 200,
|
||||
headers: { etag: '"v1"' },
|
||||
})
|
||||
);
|
||||
const put = vi.fn(async () => undefined);
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
uploader: { put },
|
||||
etagCache: { capacity: 4 },
|
||||
});
|
||||
await storage.get("k");
|
||||
const ws = storage.createWriteStream("k");
|
||||
const writer = ws.getWriter();
|
||||
await writer.write(new Uint8Array([9]));
|
||||
await writer.close();
|
||||
expect(put).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses default retry options when none are supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = fetchOf(
|
||||
new Response(new Uint8Array([1]), { status: 200 })
|
||||
);
|
||||
const storage = new HttpStorage({ baseUrl, fetch: fetchFn });
|
||||
expect(await storage.get("k")).toBeInstanceOf(Uint8Array);
|
||||
});
|
||||
|
||||
it("threads injected timers into the retry wait", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { timers } = controlledTimers();
|
||||
const fetchFn = fetchOf(new Response(new Uint8Array([42])));
|
||||
const storage = new HttpStorage({
|
||||
baseUrl,
|
||||
fetch: fetchFn,
|
||||
timers,
|
||||
retry: { maxAttempts: 1 },
|
||||
});
|
||||
const body = await storage.get("k");
|
||||
expect(Array.from(body as Uint8Array)).toEqual([42]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
export {
|
||||
composeUrl,
|
||||
errorForStatus,
|
||||
outcomeFromError,
|
||||
type ResolvedHttpStorageConfig,
|
||||
resolveHeaders,
|
||||
resolveHttpStorageConfig,
|
||||
throwIfAborted,
|
||||
} from "./http-storage.helpers.js";
|
||||
export {
|
||||
createHttpStorage,
|
||||
HttpStorage,
|
||||
} from "./http-storage.js";
|
||||
export {
|
||||
fetchWithSignal,
|
||||
parseMetadata,
|
||||
performGet,
|
||||
performGetPartial,
|
||||
performGetStream,
|
||||
performHead,
|
||||
type ReadContext,
|
||||
readOnlyError,
|
||||
} from "./http-storage.ops.js";
|
||||
export {
|
||||
buildReadStream,
|
||||
buildWriteStream,
|
||||
upstreamError,
|
||||
type WriteStreamArgs,
|
||||
} from "./http-storage.streams.js";
|
||||
101
packages/persistence/storage/storage-http/src/index.ts
Normal file
101
packages/persistence/storage/storage-http/src/index.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
// Side-effect import: ensures `defineCodes` registers the STORAGE_HTTP.*
|
||||
// registry even when subpath consumers never reach this file.
|
||||
import "./error-codes/index.js";
|
||||
|
||||
export {
|
||||
DEFAULT_ETAG_CACHE_CAPACITY,
|
||||
DEFAULT_RETRY_BASE_DELAY_MS,
|
||||
DEFAULT_RETRY_MAX_ATTEMPTS,
|
||||
DEFAULT_RETRY_MAX_DELAY_MS,
|
||||
MAX_KEY_LENGTH,
|
||||
RETRY_AFTER_MAX_DELAY_MS,
|
||||
TRANSIENT_HTTP_STATUSES,
|
||||
} from "./constants/index.js";
|
||||
export {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
type StorageHttpErrorCode,
|
||||
type StorageHttpErrorOptions,
|
||||
} from "./error-codes/index.js";
|
||||
export { EtagCache } from "./etag-cache/etag-cache.js";
|
||||
export {
|
||||
defaultFetch,
|
||||
isAbortError,
|
||||
} from "./fetch-adapter/fetch-adapter.js";
|
||||
export {
|
||||
buildReadStream,
|
||||
buildWriteStream,
|
||||
composeUrl,
|
||||
createHttpStorage,
|
||||
errorForStatus,
|
||||
fetchWithSignal,
|
||||
HttpStorage,
|
||||
outcomeFromError,
|
||||
parseMetadata,
|
||||
performGet,
|
||||
performGetPartial,
|
||||
performGetStream,
|
||||
performHead,
|
||||
type ReadContext,
|
||||
type ResolvedHttpStorageConfig,
|
||||
readOnlyError,
|
||||
resolveHeaders,
|
||||
resolveHttpStorageConfig,
|
||||
throwIfAborted,
|
||||
upstreamError,
|
||||
type WriteStreamArgs,
|
||||
} from "./http-storage/index.js";
|
||||
export {
|
||||
defaultKeyEncoder,
|
||||
sanitizeKey,
|
||||
} from "./key-sanitizer/key-sanitizer.js";
|
||||
export {
|
||||
getInstruments,
|
||||
type RecordOpInput,
|
||||
recordOp,
|
||||
recordRangeFallback,
|
||||
resetInstruments,
|
||||
type StatusClass,
|
||||
type StorageHttpInstruments,
|
||||
type StorageHttpOp,
|
||||
type StorageHttpOpOutcome,
|
||||
setStorageHttpMeterProvider,
|
||||
statusClassOf,
|
||||
} from "./otel/otel.js";
|
||||
export {
|
||||
buildRangeHeader,
|
||||
consumeRangeResponse,
|
||||
type RangeOutcome,
|
||||
} from "./range/range.js";
|
||||
export { registerHttpProvider } from "./registry/registry.js";
|
||||
export {
|
||||
type AttemptOutcome,
|
||||
isTransientError,
|
||||
isTransientStatus,
|
||||
parseRetryAfter,
|
||||
type RetryWithBackoffOptions,
|
||||
retryWithBackoff,
|
||||
} from "./retry-helper/retry-helper.js";
|
||||
export type {
|
||||
EtagCacheEntry,
|
||||
EtagCacheOptions,
|
||||
FetchLike,
|
||||
HttpHeadersResolver,
|
||||
HttpRetryOptions,
|
||||
HttpStorageOp,
|
||||
HttpStorageOptions,
|
||||
HttpUploader,
|
||||
KeyEncoder,
|
||||
PresignedUrlInfo,
|
||||
PresignedUrlSigner,
|
||||
TimersLike,
|
||||
} from "./types/index.js";
|
||||
export {
|
||||
EtagCacheOptionsSchema,
|
||||
HttpRetryOptionsSchema,
|
||||
HttpStorageOptionsSchema,
|
||||
} from "./types/index.js";
|
||||
export {
|
||||
type PresignedPutUploaderOptions,
|
||||
presignedPutUploader,
|
||||
} from "./uploader/uploader.js";
|
||||
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
defaultKeyEncoder,
|
||||
sanitizeKey,
|
||||
} from "./key-sanitizer.js";
|
||||
@@ -0,0 +1,72 @@
|
||||
# key-sanitizer
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Validates and normalises caller-supplied storage keys before they reach any `fetch` call, and provides the default per-segment URL encoder. Keys must be POSIX-form relative paths — no `..`, no leading `/`, no Windows drive prefixes, no control characters, no nulls, no values longer than `MAX_KEY_LENGTH` (2048 chars). Rejections throw `StorageHttpError` with code `STORAGE_HTTP.INVALID_KEY` or `STORAGE_HTTP.KEY_TOO_LONG`.
|
||||
|
||||
Sanitisation runs **before** every public method issues a fetch. This means a misbehaved caller can never observe network traffic for an invalid key, and the `key` carried on the resulting error is the original input for debugging.
|
||||
|
||||
## Features
|
||||
|
||||
- Strict validation: empty, traversal, absolute, drive-letter, null byte, control char, overlong.
|
||||
- Normalisation: Windows `\` → POSIX `/` (idempotent on already-POSIX input).
|
||||
- Distinct error code for length overflow (`KEY_TOO_LONG`) vs. content issues (`INVALID_KEY`).
|
||||
- Default key encoder applies `encodeURIComponent` per `/`-separated segment, preserving slashes.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { defaultKeyEncoder, sanitizeKey } from "@suaveplan/storage-http";
|
||||
|
||||
const safe = sanitizeKey("a/b/file name.txt");
|
||||
// → "a/b/file name.txt"
|
||||
const encoded = defaultKeyEncoder(safe);
|
||||
// → "a/b/file%20name.txt"
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { defaultKeyEncoder, sanitizeKey, StorageHttpError, STORAGE_HTTP_CODES } from "@suaveplan/storage-http";
|
||||
|
||||
function safeUrl(baseUrl: string, key: string): URL {
|
||||
let normalised: string;
|
||||
try {
|
||||
normalised = sanitizeKey(key);
|
||||
} catch (err) {
|
||||
if (err instanceof StorageHttpError && err.code === STORAGE_HTTP_CODES.KEY_TOO_LONG) {
|
||||
// Caller can retry with a hashed key.
|
||||
return safeUrl(baseUrl, hashKey(key));
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
return new URL(defaultKeyEncoder(normalised), baseUrl);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `sanitizeKey(key: string): string`
|
||||
|
||||
Throws `StorageHttpError` with code:
|
||||
|
||||
- `STORAGE_HTTP.INVALID_KEY` — empty, non-string, control char, leading `/`, drive prefix, or `..` segment.
|
||||
- `STORAGE_HTTP.KEY_TOO_LONG` — length > `MAX_KEY_LENGTH` (2048).
|
||||
|
||||
Returns the input with Windows `\` replaced by `/`.
|
||||
|
||||
### `defaultKeyEncoder(key: string): string`
|
||||
|
||||
Splits on `/`, runs `encodeURIComponent` per segment, re-joins. Preserves path hierarchy.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Path traversal is checked after `\` normalisation so `a\..\b` is rejected.
|
||||
- Drive prefix detection accepts `[A-Za-z]:[\\/]` — both `C:\` and `C:/` reject.
|
||||
- Control-char check scans every code unit; the early-exit loop is O(n) with one pass.
|
||||
- The sanitiser does not URL-encode — encoding is a separate concern handled by `defaultKeyEncoder` (or a caller-supplied alternative for CDNs whose signed-URL signatures require custom encoding rules).
|
||||
- Long-key error message includes a truncated prefix (`first 64 chars + ellipsis`) so the discriminant doesn't bloat logs.
|
||||
@@ -0,0 +1,88 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { defaultKeyEncoder, sanitizeKey } from "./key-sanitizer.js";
|
||||
|
||||
describe("sanitizeKey", () => {
|
||||
it("rejects path traversal segments", { timeout: 5000 }, () => {
|
||||
expect(() => sanitizeKey("../../etc/passwd")).toThrow(StorageHttpError);
|
||||
try {
|
||||
sanitizeKey("a/../b");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(StorageHttpError);
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects keys longer than 2048 chars with KEY_TOO_LONG", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
sanitizeKey("a".repeat(2049));
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.KEY_TOO_LONG
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects empty keys", { timeout: 5000 }, () => {
|
||||
expect(() => sanitizeKey("")).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("rejects non-string keys", { timeout: 5000 }, () => {
|
||||
expect(() => sanitizeKey(42 as unknown as string)).toThrow(
|
||||
StorageHttpError
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects control characters", { timeout: 5000 }, () => {
|
||||
expect(() => sanitizeKey("a\x00b")).toThrow(StorageHttpError);
|
||||
expect(() => sanitizeKey("a\x1Fb")).toThrow(StorageHttpError);
|
||||
expect(() => sanitizeKey("a\x7Fb")).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("rejects keys starting with /", { timeout: 5000 }, () => {
|
||||
expect(() => sanitizeKey("/abs")).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("rejects Windows drive prefixes", { timeout: 5000 }, () => {
|
||||
expect(() => sanitizeKey("C:\\a")).toThrow(StorageHttpError);
|
||||
expect(() => sanitizeKey("d:/b")).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("normalises Windows backslashes to POSIX slashes", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(sanitizeKey("a\\b\\c")).toBe("a/b/c");
|
||||
});
|
||||
|
||||
it("accepts a simple POSIX path", { timeout: 5000 }, () => {
|
||||
expect(sanitizeKey("a/b/c.txt")).toBe("a/b/c.txt");
|
||||
});
|
||||
});
|
||||
|
||||
describe("defaultKeyEncoder", () => {
|
||||
it("encodes spaces per segment, preserves slashes", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(defaultKeyEncoder("hello world/sub dir/file.txt")).toBe(
|
||||
"hello%20world/sub%20dir/file.txt"
|
||||
);
|
||||
});
|
||||
|
||||
it("encodes question marks and hashes inside a segment", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(defaultKeyEncoder("a?b/c#d")).toBe("a%3Fb/c%23d");
|
||||
});
|
||||
|
||||
it("encodes non-ASCII characters", { timeout: 5000 }, () => {
|
||||
expect(defaultKeyEncoder("café/menu")).toBe("caf%C3%A9/menu");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
import { MAX_KEY_LENGTH } from "../constants/index.js";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
|
||||
// Matches a leading drive-letter prefix on Windows (`C:\`, `D:/`).
|
||||
const WINDOWS_DRIVE_PREFIX = /^[A-Za-z]:[\\/]/;
|
||||
|
||||
function containsControlChar(value: string): boolean {
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const code = value.charCodeAt(i);
|
||||
if (code <= 0x1f || code === 0x7f) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and normalise a caller-supplied storage key. Returns the normalised
|
||||
* POSIX-separator form on success; throws `StorageHttpError(INVALID_KEY)` or
|
||||
* `StorageHttpError(KEY_TOO_LONG)` on rejection.
|
||||
*
|
||||
* Rejection rules:
|
||||
* - Empty / non-string.
|
||||
* - Length over `MAX_KEY_LENGTH` (2048) chars → `KEY_TOO_LONG`.
|
||||
* - Contains a `..` path segment (parent-directory traversal).
|
||||
* - Starts with `/` (POSIX absolute) or `<letter>:[\\/]` (Windows drive).
|
||||
* - Contains a null byte (`\x00`) or other control char (`0x00`–`0x1F`, `0x7F`).
|
||||
*
|
||||
* Normalisation: Windows `\` separators are replaced with POSIX `/`.
|
||||
*/
|
||||
export function sanitizeKey(key: string): string {
|
||||
if (typeof key !== "string") {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY,
|
||||
"key must be a string"
|
||||
);
|
||||
}
|
||||
if (key.length === 0) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY,
|
||||
"key must not be empty"
|
||||
);
|
||||
}
|
||||
if (key.length > MAX_KEY_LENGTH) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.KEY_TOO_LONG,
|
||||
`key length ${key.length} exceeds maximum ${MAX_KEY_LENGTH}`,
|
||||
{ key: `${key.slice(0, 64)}…` }
|
||||
);
|
||||
}
|
||||
if (containsControlChar(key)) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY,
|
||||
"key contains a control character",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
if (key.startsWith("/")) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY,
|
||||
"absolute paths are not allowed",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
if (WINDOWS_DRIVE_PREFIX.test(key)) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY,
|
||||
"drive-letter prefixed paths are not allowed",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
const normalised = key.replace(/\\/g, "/");
|
||||
for (const segment of normalised.split("/")) {
|
||||
if (segment === "..") {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_KEY,
|
||||
"path traversal segment (..) is not allowed",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
}
|
||||
return normalised;
|
||||
}
|
||||
|
||||
/**
|
||||
* Default key encoder — splits on `/`, runs `encodeURIComponent` on each
|
||||
* segment, then re-joins. Preserves the path hierarchy while encoding
|
||||
* everything that needs encoding within a segment (spaces, `?`, `#`, `&`,
|
||||
* non-ASCII).
|
||||
*/
|
||||
export function defaultKeyEncoder(key: string): string {
|
||||
return key.split("/").map(encodeURIComponent).join("/");
|
||||
}
|
||||
15
packages/persistence/storage/storage-http/src/otel/index.ts
Normal file
15
packages/persistence/storage/storage-http/src/otel/index.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
export type {
|
||||
RecordOpInput,
|
||||
StatusClass,
|
||||
StorageHttpInstruments,
|
||||
StorageHttpOp,
|
||||
StorageHttpOpOutcome,
|
||||
} from "./otel.js";
|
||||
export {
|
||||
getInstruments,
|
||||
recordOp,
|
||||
recordRangeFallback,
|
||||
resetInstruments,
|
||||
setStorageHttpMeterProvider,
|
||||
statusClassOf,
|
||||
} from "./otel.js";
|
||||
84
packages/persistence/storage/storage-http/src/otel/otel.md
Normal file
84
packages/persistence/storage/storage-http/src/otel/otel.md
Normal file
@@ -0,0 +1,84 @@
|
||||
# otel
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Owns the OpenTelemetry instrument set for `@suaveplan/storage-http`: counters (`reads_total`, `writes_total`, `deletes_total`, `errors_total`, `range_fallback_total`), histograms (`op_duration_ms`, `io_bytes`), and a tracer scoped to `@suaveplan/storage-http`. Every public operation in `HttpStorage` opens a span and calls `recordOp` so operators can observe per-status-class latency and per-outcome counts without instrumenting consumer code.
|
||||
|
||||
The module is intentionally pluggable — `setStorageHttpMeterProvider` lets tests bind to an `InMemoryMetricExporter` while production binds to whatever the host application has configured globally.
|
||||
|
||||
## Features
|
||||
|
||||
- Lazy instrument construction so consumers can install a custom MeterProvider before the first operation runs.
|
||||
- `setStorageHttpMeterProvider(provider)` override hook with auto-rebuild on next access.
|
||||
- `recordOp({ op, outcome, statusClass, durationMs, bytes?, direction? })` emits the full observation set for one operation.
|
||||
- `recordRangeFallback()` increments the `range_fallback_total` counter exactly once — surfaces misconfigured origins.
|
||||
- `statusClassOf(status)` maps numeric status to the `2xx`/`3xx`/`4xx`/`5xx`/`none` attribute used on histograms.
|
||||
- `resetInstruments()` for test isolation.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { recordOp } from "@suaveplan/storage-http";
|
||||
|
||||
recordOp({ op: "get", outcome: "ok", statusClass: "2xx", durationMs: 42, bytes: 1024, direction: "read" });
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { setStorageHttpMeterProvider, resetInstruments, recordOp } from "@suaveplan/storage-http";
|
||||
import { MeterProvider, InMemoryMetricExporter, PeriodicExportingMetricReader, AggregationTemporality } from "@opentelemetry/sdk-metrics";
|
||||
|
||||
const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE);
|
||||
const reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
setStorageHttpMeterProvider(provider);
|
||||
resetInstruments();
|
||||
recordOp({ op: "get", outcome: "ok", statusClass: "2xx", durationMs: 10 });
|
||||
await reader.collect();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `setStorageHttpMeterProvider(provider?: MeterProvider): void`
|
||||
|
||||
Override or restore the meter provider. Calling this invalidates the cached instrument set so the next `getInstruments()` rebinds.
|
||||
|
||||
### `getInstruments(force?: boolean): StorageHttpInstruments`
|
||||
|
||||
Returns the cached instrument bundle (`tracer`, `meter`, every counter and histogram). `force = true` rebuilds.
|
||||
|
||||
### `recordOp(input: RecordOpInput): void`
|
||||
|
||||
`input.outcome` decides which counter increments:
|
||||
|
||||
- `"ok"` + read-ish op → `reads_total`.
|
||||
- `"ok"` + `set`/`setPartial` → `writes_total`.
|
||||
- `"ok"` + `delete`/`clear` → `deletes_total`.
|
||||
- anything else → `errors_total`.
|
||||
|
||||
Always records `op_duration_ms{op, outcome, status_class}`. When `bytes` and `direction` are both provided, also records `io_bytes{op, direction}`.
|
||||
|
||||
### `recordRangeFallback(): void`
|
||||
|
||||
Increments `range_fallback_total{outcome="ok"}`.
|
||||
|
||||
### `statusClassOf(status: number | undefined): StatusClass`
|
||||
|
||||
`"2xx" | "3xx" | "4xx" | "5xx" | "none"`. `"none"` covers `undefined` and out-of-range inputs.
|
||||
|
||||
### `resetInstruments(): void`
|
||||
|
||||
Drops the cached bundle without changing the meter provider.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The cache is critical: instrument creation is not free and consumers call `recordOp` in the hot path. `setStorageHttpMeterProvider` is the one path that intentionally invalidates the cache, so test isolation is straightforward.
|
||||
- `status_class` is an attribute on `op_duration_ms` rather than a separate histogram so operators can break down latency by class without exploding cardinality.
|
||||
- `range_fallback_total` deliberately carries `outcome="ok"` only — the fallback path always succeeds (just less efficiently). If the underlying request fails, that surfaces via `errors_total` instead.
|
||||
- The `direction` attribute is only set for `set` / `setPartial` / `get` / `getPartial` — operations like `exists` or `delete` carry no bytes.
|
||||
- The tracer is acquired separately from the meter; no caching is required because `trace.getTracer` is idempotent.
|
||||
258
packages/persistence/storage/storage-http/src/otel/otel.test.ts
Normal file
258
packages/persistence/storage/storage-http/src/otel/otel.test.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import type {
|
||||
Counter,
|
||||
Histogram,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
getInstruments,
|
||||
recordOp,
|
||||
recordRangeFallback,
|
||||
resetInstruments,
|
||||
setStorageHttpMeterProvider,
|
||||
statusClassOf,
|
||||
} from "./otel.js";
|
||||
|
||||
interface RecordedHistogram {
|
||||
name: string;
|
||||
samples: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
interface RecordedCounter {
|
||||
name: string;
|
||||
additions: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
|
||||
class StubHistogram implements Histogram {
|
||||
public readonly entry: RecordedHistogram;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedHistogram[]
|
||||
) {
|
||||
this.entry = { name, samples: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public record(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.samples.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubCounter implements Counter {
|
||||
public readonly entry: RecordedCounter;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedCounter[]
|
||||
) {
|
||||
this.entry = { name, additions: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public add(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.additions.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeter implements Meter {
|
||||
public histograms: RecordedHistogram[] = [];
|
||||
public counters: RecordedCounter[] = [];
|
||||
public createHistogram(name: string): Histogram {
|
||||
return new StubHistogram(name, this.histograms);
|
||||
}
|
||||
public createCounter(name: string): Counter {
|
||||
return new StubCounter(name, this.counters);
|
||||
}
|
||||
public createObservableGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public addBatchObservableCallback(): void {}
|
||||
public removeBatchObservableCallback(): void {}
|
||||
public createGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeterProvider implements MeterProvider {
|
||||
public meters: StubMeter[] = [];
|
||||
public getMeter(): Meter {
|
||||
const meter = new StubMeter();
|
||||
this.meters.push(meter);
|
||||
return meter;
|
||||
}
|
||||
}
|
||||
|
||||
describe("storage-http otel", () => {
|
||||
let provider: StubMeterProvider;
|
||||
beforeEach(() => {
|
||||
provider = new StubMeterProvider();
|
||||
setStorageHttpMeterProvider(provider);
|
||||
});
|
||||
afterEach(() => {
|
||||
setStorageHttpMeterProvider(undefined);
|
||||
resetInstruments();
|
||||
});
|
||||
|
||||
function findCounter(name: string): RecordedCounter | undefined {
|
||||
return provider.meters[0]?.counters.find((c) => c.name === name);
|
||||
}
|
||||
function findHistogram(name: string): RecordedHistogram | undefined {
|
||||
return provider.meters[0]?.histograms.find((h) => h.name === name);
|
||||
}
|
||||
|
||||
it("successful GET emits read counter and duration histogram", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({
|
||||
op: "get",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 12,
|
||||
bytes: 100,
|
||||
direction: "read",
|
||||
});
|
||||
const reads = findCounter("storage_http.reads_total");
|
||||
expect(reads?.additions).toHaveLength(1);
|
||||
const dur = findHistogram("storage_http.op_duration_ms");
|
||||
expect(dur?.samples[0]?.attrs).toEqual({
|
||||
op: "get",
|
||||
outcome: "ok",
|
||||
status_class: "2xx",
|
||||
});
|
||||
expect(findHistogram("storage_http.io_bytes")?.samples[0]?.value).toBe(
|
||||
100
|
||||
);
|
||||
});
|
||||
|
||||
it("failed GET records error span and increments errors_total", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({
|
||||
op: "get",
|
||||
outcome: "upstream_error",
|
||||
statusClass: "5xx",
|
||||
durationMs: 5,
|
||||
});
|
||||
const errs = findCounter("storage_http.errors_total");
|
||||
expect(errs?.additions[0]?.attrs).toEqual({
|
||||
op: "get",
|
||||
outcome: "upstream_error",
|
||||
});
|
||||
});
|
||||
|
||||
it("increments writes_total for set/setPartial ok outcomes", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({
|
||||
op: "set",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 1,
|
||||
});
|
||||
recordOp({
|
||||
op: "setPartial",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 1,
|
||||
});
|
||||
expect(
|
||||
findCounter("storage_http.writes_total")?.additions
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("increments deletes_total for delete/clear ok outcomes", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({
|
||||
op: "delete",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 1,
|
||||
});
|
||||
recordOp({
|
||||
op: "clear",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 1,
|
||||
});
|
||||
expect(
|
||||
findCounter("storage_http.deletes_total")?.additions
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("omits io_bytes when bytes is undefined", { timeout: 5000 }, () => {
|
||||
recordOp({
|
||||
op: "exists",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 1,
|
||||
});
|
||||
expect(
|
||||
findHistogram("storage_http.io_bytes")?.samples ?? []
|
||||
).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("recordRangeFallback increments the dedicated counter", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordRangeFallback();
|
||||
expect(
|
||||
findCounter("storage_http.range_fallback_total")?.additions
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("returns the same instrument set on repeated calls", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const a = getInstruments();
|
||||
const b = getInstruments();
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("rebuilds instruments when force=true", { timeout: 5000 }, () => {
|
||||
const a = getInstruments();
|
||||
const b = getInstruments(true);
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
it("statusClassOf maps each range correctly", { timeout: 5000 }, () => {
|
||||
expect(statusClassOf(200)).toBe("2xx");
|
||||
expect(statusClassOf(304)).toBe("3xx");
|
||||
expect(statusClassOf(404)).toBe("4xx");
|
||||
expect(statusClassOf(503)).toBe("5xx");
|
||||
expect(statusClassOf(undefined)).toBe("none");
|
||||
expect(statusClassOf(700)).toBe("none");
|
||||
});
|
||||
|
||||
it("exists ok does not increment reads/writes/deletes", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({
|
||||
op: "exists",
|
||||
outcome: "ok",
|
||||
statusClass: "2xx",
|
||||
durationMs: 1,
|
||||
});
|
||||
expect(
|
||||
findCounter("storage_http.reads_total")?.additions ?? []
|
||||
).toHaveLength(0);
|
||||
expect(
|
||||
findCounter("storage_http.writes_total")?.additions ?? []
|
||||
).toHaveLength(0);
|
||||
expect(
|
||||
findCounter("storage_http.deletes_total")?.additions ?? []
|
||||
).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
169
packages/persistence/storage/storage-http/src/otel/otel.ts
Normal file
169
packages/persistence/storage/storage-http/src/otel/otel.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
import {
|
||||
type Counter,
|
||||
type Histogram,
|
||||
type Meter,
|
||||
type MeterProvider,
|
||||
metrics,
|
||||
type Tracer,
|
||||
trace,
|
||||
} from "@opentelemetry/api";
|
||||
|
||||
const METER_NAME = "@suaveplan/storage-http";
|
||||
const TRACER_NAME = "@suaveplan/storage-http";
|
||||
|
||||
let overrideMeterProvider: MeterProvider | undefined;
|
||||
|
||||
/**
|
||||
* Override the `MeterProvider` used to acquire the storage-http meter. Pass
|
||||
* `undefined` to fall back to the global provider. The instrument cache is
|
||||
* rebuilt automatically on the next access.
|
||||
*/
|
||||
export function setStorageHttpMeterProvider(
|
||||
provider: MeterProvider | undefined
|
||||
): void {
|
||||
overrideMeterProvider = provider;
|
||||
cachedInstruments = undefined;
|
||||
}
|
||||
|
||||
export type StorageHttpOp =
|
||||
| "get"
|
||||
| "set"
|
||||
| "delete"
|
||||
| "exists"
|
||||
| "list"
|
||||
| "clear"
|
||||
| "getMetadata"
|
||||
| "getWithMetadata"
|
||||
| "getSize"
|
||||
| "getPartial"
|
||||
| "setPartial"
|
||||
| "getItemSize"
|
||||
| "createReadStream"
|
||||
| "createWriteStream";
|
||||
|
||||
export type StorageHttpOpOutcome =
|
||||
| "ok"
|
||||
| "not_found"
|
||||
| "unauthorized"
|
||||
| "forbidden"
|
||||
| "network_error"
|
||||
| "upstream_error"
|
||||
| "aborted"
|
||||
| "read_only";
|
||||
|
||||
export type StatusClass = "2xx" | "3xx" | "4xx" | "5xx" | "none";
|
||||
|
||||
export interface StorageHttpInstruments {
|
||||
tracer: Tracer;
|
||||
meter: Meter;
|
||||
readsTotal: Counter;
|
||||
writesTotal: Counter;
|
||||
deletesTotal: Counter;
|
||||
errorsTotal: Counter;
|
||||
rangeFallbackTotal: Counter;
|
||||
opDurationMs: Histogram;
|
||||
ioBytes: Histogram;
|
||||
}
|
||||
|
||||
let cachedInstruments: StorageHttpInstruments | undefined;
|
||||
|
||||
/**
|
||||
* Lazily construct the OTel instrument set. Tests call `setStorageHttpMeterProvider`
|
||||
* BEFORE the first access so the instruments bind to a controlled provider.
|
||||
*/
|
||||
export function getInstruments(force = false): StorageHttpInstruments {
|
||||
if (cachedInstruments !== undefined && !force) return cachedInstruments;
|
||||
const tracer = trace.getTracer(TRACER_NAME);
|
||||
const meter = (overrideMeterProvider ?? metrics).getMeter(METER_NAME);
|
||||
cachedInstruments = {
|
||||
tracer,
|
||||
meter,
|
||||
readsTotal: meter.createCounter("storage_http.reads_total", {
|
||||
description: "Total successful read operations",
|
||||
}),
|
||||
writesTotal: meter.createCounter("storage_http.writes_total", {
|
||||
description: "Total successful write operations",
|
||||
}),
|
||||
deletesTotal: meter.createCounter("storage_http.deletes_total", {
|
||||
description: "Total successful delete operations",
|
||||
}),
|
||||
errorsTotal: meter.createCounter("storage_http.errors_total", {
|
||||
description: "Total operations that ended in error",
|
||||
}),
|
||||
rangeFallbackTotal: meter.createCounter(
|
||||
"storage_http.range_fallback_total",
|
||||
{
|
||||
description:
|
||||
"Count of Range requests where the origin returned 200 instead of 206",
|
||||
}
|
||||
),
|
||||
opDurationMs: meter.createHistogram("storage_http.op_duration_ms", {
|
||||
description: "Per-operation duration in milliseconds",
|
||||
unit: "ms",
|
||||
}),
|
||||
ioBytes: meter.createHistogram("storage_http.io_bytes", {
|
||||
description: "Bytes read or written per operation",
|
||||
unit: "By",
|
||||
}),
|
||||
};
|
||||
return cachedInstruments;
|
||||
}
|
||||
|
||||
export interface RecordOpInput {
|
||||
op: StorageHttpOp;
|
||||
outcome: StorageHttpOpOutcome;
|
||||
statusClass: StatusClass;
|
||||
durationMs: number;
|
||||
bytes?: number;
|
||||
direction?: "read" | "write";
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a complete observability record for one operation: increments the
|
||||
* appropriate counter, records a duration histogram observation, and (when
|
||||
* `bytes` is provided) records an `io_bytes` histogram observation.
|
||||
*/
|
||||
export function recordOp(input: RecordOpInput): void {
|
||||
const { op, outcome, statusClass, durationMs, bytes, direction } = input;
|
||||
const inst = getInstruments();
|
||||
inst.opDurationMs.record(durationMs, {
|
||||
op,
|
||||
outcome,
|
||||
status_class: statusClass,
|
||||
});
|
||||
if (outcome === "ok") {
|
||||
if (op === "get" || op === "getPartial" || op === "getWithMetadata") {
|
||||
inst.readsTotal.add(1, { op, outcome });
|
||||
} else if (op === "set" || op === "setPartial") {
|
||||
inst.writesTotal.add(1, { op, outcome });
|
||||
} else if (op === "delete" || op === "clear") {
|
||||
inst.deletesTotal.add(1, { op, outcome });
|
||||
}
|
||||
} else {
|
||||
inst.errorsTotal.add(1, { op, outcome });
|
||||
}
|
||||
if (bytes !== undefined && direction !== undefined) {
|
||||
inst.ioBytes.record(bytes, { op, direction });
|
||||
}
|
||||
}
|
||||
|
||||
/** Increment the range-fallback counter exactly once. */
|
||||
export function recordRangeFallback(): void {
|
||||
const inst = getInstruments();
|
||||
inst.rangeFallbackTotal.add(1, { outcome: "ok" });
|
||||
}
|
||||
|
||||
/** Map an HTTP status to a status-class attribute. */
|
||||
export function statusClassOf(status: number | undefined): StatusClass {
|
||||
if (status === undefined) return "none";
|
||||
if (status >= 200 && status < 300) return "2xx";
|
||||
if (status >= 300 && status < 400) return "3xx";
|
||||
if (status >= 400 && status < 500) return "4xx";
|
||||
if (status >= 500 && status < 600) return "5xx";
|
||||
return "none";
|
||||
}
|
||||
|
||||
/** Reset the cached instrument set. Tests use this between blocks. */
|
||||
export function resetInstruments(): void {
|
||||
cachedInstruments = undefined;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export type { RangeOutcome } from "./range.js";
|
||||
export {
|
||||
buildRangeHeader,
|
||||
consumeRangeResponse,
|
||||
} from "./range.js";
|
||||
63
packages/persistence/storage/storage-http/src/range/range.md
Normal file
63
packages/persistence/storage/storage-http/src/range/range.md
Normal file
@@ -0,0 +1,63 @@
|
||||
# range
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Encodes HTTP `Range` semantics for `HttpStorage.getPartial`. Builds the request header from `(offset, length)` and interprets the response in the three documented shapes: `206 Partial Content` (happy path — body returned verbatim), `200 OK` (origin ignored the Range — body sliced client-side and the fallback is flagged so the caller can emit a counter), and `416 Range Not Satisfiable` (request beyond EOF — mapped to `STORAGE_HTTP.NOT_FOUND`).
|
||||
|
||||
Without this module, every adapter consumer would have to re-implement the same disambiguation. Keeping it isolated also lets tests pin range behaviour without spinning up the full `HttpStorage`.
|
||||
|
||||
## Features
|
||||
|
||||
- `buildRangeHeader(offset, length)` — validates integer non-negativity / positivity, returns `bytes=<o>-<o+l-1>`.
|
||||
- `consumeRangeResponse(response, offset, length, url)` — distinguishes 206 / 200 / 416 / other, returns `{ bytes, fellBack }`.
|
||||
- Programmer-error inputs throw `STORAGE_HTTP.INVALID_OPTIONS` so they surface in development, not as cryptic upstream failures.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { buildRangeHeader, consumeRangeResponse } from "@suaveplan/storage-http";
|
||||
|
||||
const header = buildRangeHeader(0, 1024); // "bytes=0-1023"
|
||||
const response = await fetch(url, { headers: { Range: header } });
|
||||
const { bytes, fellBack } = await consumeRangeResponse(response, 0, 1024, url);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { buildRangeHeader, consumeRangeResponse } from "@suaveplan/storage-http";
|
||||
|
||||
async function readSlice(url: string, offset: number, length: number) {
|
||||
const response = await fetch(url, { headers: { Range: buildRangeHeader(offset, length) } });
|
||||
const { bytes, fellBack } = await consumeRangeResponse(response, offset, length, url);
|
||||
if (fellBack) operatorLog.warn({ url }, "origin ignored Range — full body returned");
|
||||
return bytes;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `buildRangeHeader(offset: number, length: number): string`
|
||||
|
||||
Returns `"bytes=<offset>-<offset+length-1>"`. Throws `StorageHttpError(INVALID_OPTIONS)` when `offset` is not a non-negative integer or `length` is not a positive integer.
|
||||
|
||||
### `interface RangeOutcome`
|
||||
|
||||
`{ bytes: Uint8Array; fellBack: boolean }` — `fellBack` is `true` only when the origin returned `200 OK` to a `Range` request and the slice was performed client-side.
|
||||
|
||||
### `consumeRangeResponse(response, offset, length, url): Promise<RangeOutcome>`
|
||||
|
||||
- `206` → returns the body verbatim with `fellBack = false`.
|
||||
- `200` → consumes the full body, slices `[offset, offset + length)`, returns with `fellBack = true`.
|
||||
- `416` → throws `StorageHttpError(NOT_FOUND)` with `httpStatus: 416`.
|
||||
- other → throws `StorageHttpError(UPSTREAM_ERROR)` with the observed status.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The `Content-Range` response header is intentionally NOT parsed here — it carries the total object size, which is useful for cache metadata but not required to decode the body. `HttpStorage` parses it in the metadata path when needed.
|
||||
- Client-side slicing on `200` uses `Math.min` clamps so out-of-bounds requests never throw `RangeError` on `slice`; the resulting array is simply empty if `offset >= body.byteLength`. The caller decides whether to treat that as "object shorter than expected" or pass it through.
|
||||
- `416` mapping to `NOT_FOUND` is a deliberate simplification: distinguishing "object missing" from "range past EOF on an existing object" requires a `HEAD` round-trip the caller already has cheaper alternatives for (e.g. `exists()` or attaching `If-Range` to disambiguate cache vs miss).
|
||||
@@ -0,0 +1,181 @@
|
||||
import type { Counter, Meter, MeterProvider } from "@opentelemetry/api";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { resetInstruments, setStorageHttpMeterProvider } from "../otel/otel.js";
|
||||
import { buildRangeHeader, consumeRangeResponse } from "./range.js";
|
||||
|
||||
interface RecordedCounter {
|
||||
name: string;
|
||||
additions: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
|
||||
class StubCounter implements Counter {
|
||||
public readonly entry: RecordedCounter;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedCounter[]
|
||||
) {
|
||||
this.entry = { name, additions: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public add(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.additions.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeter implements Meter {
|
||||
public counters: RecordedCounter[] = [];
|
||||
public createHistogram(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createCounter(name: string): Counter {
|
||||
return new StubCounter(name, this.counters);
|
||||
}
|
||||
public createObservableGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public addBatchObservableCallback(): void {}
|
||||
public removeBatchObservableCallback(): void {}
|
||||
public createGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeterProvider implements MeterProvider {
|
||||
public meters: StubMeter[] = [];
|
||||
public getMeter(): Meter {
|
||||
const meter = new StubMeter();
|
||||
this.meters.push(meter);
|
||||
return meter;
|
||||
}
|
||||
}
|
||||
|
||||
describe("buildRangeHeader", () => {
|
||||
it("returns the inclusive byte range form", { timeout: 5000 }, () => {
|
||||
expect(buildRangeHeader(0, 100)).toBe("bytes=0-99");
|
||||
expect(buildRangeHeader(50, 25)).toBe("bytes=50-74");
|
||||
});
|
||||
|
||||
it("rejects non-integer offset", { timeout: 5000 }, () => {
|
||||
expect(() => buildRangeHeader(1.5, 1)).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("rejects negative offset", { timeout: 5000 }, () => {
|
||||
expect(() => buildRangeHeader(-1, 1)).toThrow(StorageHttpError);
|
||||
});
|
||||
|
||||
it("rejects non-positive length", { timeout: 5000 }, () => {
|
||||
expect(() => buildRangeHeader(0, 0)).toThrow(StorageHttpError);
|
||||
expect(() => buildRangeHeader(0, -1)).toThrow(StorageHttpError);
|
||||
expect(() => buildRangeHeader(0, 1.5)).toThrow(StorageHttpError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("consumeRangeResponse", () => {
|
||||
let provider: StubMeterProvider;
|
||||
beforeEach(() => {
|
||||
provider = new StubMeterProvider();
|
||||
setStorageHttpMeterProvider(provider);
|
||||
});
|
||||
afterEach(() => {
|
||||
setStorageHttpMeterProvider(undefined);
|
||||
resetInstruments();
|
||||
});
|
||||
|
||||
it("206 Partial Content returns body verbatim", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const response = new Response(new Uint8Array([2, 3]), {
|
||||
status: 206,
|
||||
headers: { "content-range": "bytes 1-2/10" },
|
||||
});
|
||||
const outcome = await consumeRangeResponse(
|
||||
response,
|
||||
1,
|
||||
2,
|
||||
"https://example.com/x"
|
||||
);
|
||||
expect(outcome.fellBack).toBe(false);
|
||||
expect(Array.from(outcome.bytes)).toEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("200 fallback slices client-side and increments range_fallback_total", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const response = new Response(new Uint8Array([0, 1, 2, 3, 4]), {
|
||||
status: 200,
|
||||
});
|
||||
const outcome = await consumeRangeResponse(
|
||||
response,
|
||||
1,
|
||||
2,
|
||||
"https://example.com/x"
|
||||
);
|
||||
expect(outcome.fellBack).toBe(true);
|
||||
expect(Array.from(outcome.bytes)).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
it("416 maps to NOT_FOUND with httpStatus 416", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const response = new Response(null, { status: 416 });
|
||||
try {
|
||||
await consumeRangeResponse(
|
||||
response,
|
||||
100,
|
||||
10,
|
||||
"https://example.com/x"
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND
|
||||
);
|
||||
expect((err as StorageHttpError).httpStatus).toBe(416);
|
||||
}
|
||||
});
|
||||
|
||||
it("other status surfaces UPSTREAM_ERROR", { timeout: 5000 }, async () => {
|
||||
const response = new Response(null, { status: 500 });
|
||||
try {
|
||||
await consumeRangeResponse(response, 0, 1, "https://example.com/x");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("200 fallback clamps offset past EOF to empty result", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const response = new Response(new Uint8Array([1, 2]), { status: 200 });
|
||||
const outcome = await consumeRangeResponse(
|
||||
response,
|
||||
10,
|
||||
5,
|
||||
"https://example.com/x"
|
||||
);
|
||||
expect(outcome.fellBack).toBe(true);
|
||||
expect(outcome.bytes.byteLength).toBe(0);
|
||||
});
|
||||
});
|
||||
67
packages/persistence/storage/storage-http/src/range/range.ts
Normal file
67
packages/persistence/storage/storage-http/src/range/range.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
|
||||
/**
|
||||
* Build the inclusive `Range` request header value for a byte slice
|
||||
* `[offset, offset + length)`. Asserts `offset >= 0` and `length > 0` —
|
||||
* malformed inputs are programmer errors, never silently sanitised.
|
||||
*/
|
||||
export function buildRangeHeader(offset: number, length: number): string {
|
||||
if (!Number.isInteger(offset) || offset < 0) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_OPTIONS,
|
||||
`getPartial offset must be a non-negative integer (received ${offset})`
|
||||
);
|
||||
}
|
||||
if (!Number.isInteger(length) || length <= 0) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.INVALID_OPTIONS,
|
||||
`getPartial length must be a positive integer (received ${length})`
|
||||
);
|
||||
}
|
||||
const last = offset + length - 1;
|
||||
return `bytes=${offset}-${last}`;
|
||||
}
|
||||
|
||||
export interface RangeOutcome {
|
||||
bytes: Uint8Array;
|
||||
/** True when the origin ignored Range and returned `200` with the full body. */
|
||||
fellBack: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret a Range response. `206` → return body verbatim. `200` → slice
|
||||
* client-side and flag the fallback. `416` → throw `NOT_FOUND` (range past
|
||||
* EOF). Other statuses fall through to the caller's status-code handler.
|
||||
*/
|
||||
export async function consumeRangeResponse(
|
||||
response: Response,
|
||||
offset: number,
|
||||
length: number,
|
||||
url: string
|
||||
): Promise<RangeOutcome> {
|
||||
if (response.status === 206) {
|
||||
const body = new Uint8Array(await response.arrayBuffer());
|
||||
return { bytes: body, fellBack: false };
|
||||
}
|
||||
if (response.status === 200) {
|
||||
const body = new Uint8Array(await response.arrayBuffer());
|
||||
const end = Math.min(body.byteLength, offset + length);
|
||||
const start = Math.min(body.byteLength, offset);
|
||||
return { bytes: body.slice(start, end), fellBack: true };
|
||||
}
|
||||
if (response.status === 416) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NOT_FOUND,
|
||||
`range past end of object (offset=${offset}, length=${length})`,
|
||||
{ url, httpStatus: 416 }
|
||||
);
|
||||
}
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR,
|
||||
`unexpected status ${response.status} on Range request`,
|
||||
{ url, httpStatus: response.status }
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { registerHttpProvider } from "./registry.js";
|
||||
@@ -0,0 +1,77 @@
|
||||
# registry
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Module-augments `@suaveplan/storage/factory`'s `StorageProviderRegistry` and `StorageProviderConfig` to add the `http` slot, and exposes `registerHttpProvider()` to bind `HttpStorage` as the constructor for that slot. Consumers who prefer factory-style instantiation (`createStorage({ type: "http", config })`) call `registerHttpProvider()` once at boot and never reference the class directly.
|
||||
|
||||
Registration is idempotent — duplicate calls are no-ops, so wiring this into multiple bootstrappers (test fixtures + production startup) is safe.
|
||||
|
||||
## Features
|
||||
|
||||
- TypeScript module augmentation — `createStorage<"http">` now type-checks with `HttpStorageOptions`.
|
||||
- Idempotent `registerHttpProvider()` — safe to call from multiple bootstrap paths.
|
||||
- Zero runtime dependencies beyond `@suaveplan/storage/factory` and the local `HttpStorage` class.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { registerHttpProvider } from "@suaveplan/storage-http";
|
||||
import { createStorage } from "@suaveplan/storage/factory";
|
||||
|
||||
registerHttpProvider();
|
||||
const storage = createStorage({
|
||||
type: "http",
|
||||
config: { baseUrl: "https://cdn.example.com/start/here/" },
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { registerHttpProvider, presignedPutUploader } from "@suaveplan/storage-http";
|
||||
import { createStorage } from "@suaveplan/storage/factory";
|
||||
|
||||
// Bootstrap once for the process.
|
||||
registerHttpProvider();
|
||||
|
||||
function makeStorage(zone: string) {
|
||||
return createStorage({
|
||||
type: "http",
|
||||
config: {
|
||||
baseUrl: `https://${zone}.b-cdn.net/`,
|
||||
uploader: presignedPutUploader({ signer: presignFor(zone) }),
|
||||
},
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `registerHttpProvider(): void`
|
||||
|
||||
Registers `HttpStorage` against `StorageFactory` under the `"http"` slot if not already registered. Calling it twice is a no-op.
|
||||
|
||||
### Module augmentation
|
||||
|
||||
```ts
|
||||
declare module "@suaveplan/storage/factory" {
|
||||
interface StorageProviderRegistry { http: HttpStorage }
|
||||
interface StorageProviderConfig { http: HttpStorageOptions }
|
||||
}
|
||||
```
|
||||
|
||||
This means:
|
||||
|
||||
- `createStorage<"http">(...)` returns `HttpStorage`.
|
||||
- The `config` parameter is typed as `HttpStorageOptions`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The cast `HttpStorage as unknown as new (config?: unknown) => HttpStorage` is required because the factory's `ProviderCtor` is intentionally loose (`config?: unknown`) — adapter constructors take typed options that don't satisfy the registry's structural shape without the cast.
|
||||
- `hasStorageProvider("http")` is the idempotency gate. After `unregisterStorageProvider("http")` or `clearStorageProviders()` from `@suaveplan/storage`, a subsequent `registerHttpProvider()` call cleanly re-registers.
|
||||
- The augmentation is `interface` merging — pure type-level. There is no runtime cost beyond the one-time `Map.set` in the factory.
|
||||
- This module imports `HttpStorage` directly rather than via the package root barrel to avoid a circular import (the root barrel re-exports `registerHttpProvider`).
|
||||
@@ -0,0 +1,37 @@
|
||||
import {
|
||||
clearStorageProviders,
|
||||
createStorage,
|
||||
hasStorageProvider,
|
||||
} from "@suaveplan/storage/factory";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { HttpStorage } from "../http-storage/http-storage.js";
|
||||
import { registerHttpProvider } from "./registry.js";
|
||||
|
||||
const baseUrl = "https://cdn.example.com/start/here/";
|
||||
|
||||
describe("registerHttpProvider", () => {
|
||||
beforeEach(() => clearStorageProviders());
|
||||
afterEach(() => clearStorageProviders());
|
||||
|
||||
it("createStorage with type http returns HttpStorage", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
registerHttpProvider();
|
||||
const storage = createStorage({ type: "http", config: { baseUrl } });
|
||||
expect(storage).toBeInstanceOf(HttpStorage);
|
||||
});
|
||||
|
||||
it("registerHttpProvider is idempotent", { timeout: 5000 }, () => {
|
||||
registerHttpProvider();
|
||||
registerHttpProvider();
|
||||
expect(hasStorageProvider("http")).toBe(true);
|
||||
const storage = createStorage({ type: "http", config: { baseUrl } });
|
||||
expect(storage).toBeInstanceOf(HttpStorage);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import {
|
||||
hasStorageProvider,
|
||||
registerStorageProvider,
|
||||
} from "@suaveplan/storage/factory";
|
||||
import { HttpStorage } from "../http-storage/http-storage.js";
|
||||
import type { HttpStorageOptions } from "../types/http-storage.types.js";
|
||||
|
||||
declare module "@suaveplan/storage/factory" {
|
||||
interface StorageProviderRegistry {
|
||||
http: HttpStorage;
|
||||
}
|
||||
interface StorageProviderConfig {
|
||||
http: HttpStorageOptions;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register `HttpStorage` against the `@suaveplan/storage` factory under the
|
||||
* canonical type slot `"http"`. Idempotent — repeated calls do nothing once
|
||||
* the slot is occupied.
|
||||
*/
|
||||
export function registerHttpProvider(): void {
|
||||
if (hasStorageProvider("http")) return;
|
||||
registerStorageProvider(
|
||||
"http",
|
||||
HttpStorage as unknown as new (
|
||||
config?: unknown
|
||||
) => HttpStorage
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export type {
|
||||
AttemptOutcome,
|
||||
RetryWithBackoffOptions,
|
||||
} from "./retry-helper.js";
|
||||
export {
|
||||
isTransientError,
|
||||
isTransientStatus,
|
||||
parseRetryAfter,
|
||||
retryWithBackoff,
|
||||
} from "./retry-helper.js";
|
||||
@@ -0,0 +1,79 @@
|
||||
# retry-helper
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Coordinates retry logic for HTTP requests issued by `HttpStorage`. Distinguishes transient outcomes (HTTP `408`/`425`/`429`/`5xx` or `fetch`-level network rejection) from fatal ones, honours `Retry-After` when the server provides it, and falls back to exponential backoff with a configurable cap. AbortSignal is plumbed through every wait so cancellation lands within milliseconds, never blocked on a timer.
|
||||
|
||||
The helper exists because two of the spec's requirements (`Retry-After` honour and the precise classifier) are too provider-specific to delegate to `@suaveplan/retry` directly — the latter is generic across error types, while this module knows about HTTP semantics.
|
||||
|
||||
## Features
|
||||
|
||||
- `isTransientStatus(status)` — `408`, `425`, `429`, or any `5xx`.
|
||||
- `isTransientError(err)` — `StorageHttpError(NETWORK_ERROR)` produced by the `fetch` adapter.
|
||||
- `parseRetryAfter(value, now)` — supports delta-seconds and HTTP-date forms; caps at `RETRY_AFTER_MAX_DELAY_MS` (60s) to defend against malicious origins.
|
||||
- `retryWithBackoff(fn, options)` — runs `fn` up to `maxAttempts`, classifies outcomes, sleeps between attempts, supports cancellation.
|
||||
- Injectable `TimersLike` so tests pin behaviour under a `FakeClock` without real timers.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { retryWithBackoff } from "@suaveplan/storage-http";
|
||||
|
||||
const response = await retryWithBackoff(() => fetch("https://cdn.example.com/file.bin"), {
|
||||
retry: { maxAttempts: 5, baseDelayMs: 200, maxDelayMs: 5_000 },
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { retryWithBackoff, isTransientStatus, parseRetryAfter } from "@suaveplan/storage-http";
|
||||
import { fakeClock } from "@suaveplan/testing/runner";
|
||||
|
||||
const clock = fakeClock();
|
||||
const response = await retryWithBackoff(
|
||||
async () => fetch("https://cdn.example.com/file.bin", { signal }),
|
||||
{
|
||||
retry: { maxAttempts: 3, honourRetryAfter: true },
|
||||
timers: clock.timers,
|
||||
signal,
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `retryWithBackoff(fn, options?): Promise<Response>`
|
||||
|
||||
- `fn: () => Promise<Response>` — attempts the request.
|
||||
- `options.retry?` — `maxAttempts`, `baseDelayMs`, `maxDelayMs`, `honourRetryAfter` (default `true`).
|
||||
- `options.timers?` — defaults to `globalThis.setTimeout`/`clearTimeout`.
|
||||
- `options.signal?` — cancels the wait between attempts; pre-aborted signal short-circuits with `STORAGE_HTTP.ABORTED`.
|
||||
|
||||
Returns the last response after exhaustion (caller decides how to surface a transient final status) or throws the captured error.
|
||||
|
||||
### `isTransientStatus(status: number): boolean`
|
||||
|
||||
True for `408`, `425`, `429`, or any value in `[500, 599]`.
|
||||
|
||||
### `isTransientError(err: unknown): boolean`
|
||||
|
||||
True for `StorageHttpError` with code `STORAGE_HTTP.NETWORK_ERROR`.
|
||||
|
||||
### `parseRetryAfter(value: string | null, now: number): number | undefined`
|
||||
|
||||
- Numeric value → delta-seconds × 1000 (capped at `RETRY_AFTER_MAX_DELAY_MS`).
|
||||
- HTTP-date → `parseDate − now` (clamped to 0 if in the past, capped on the high side).
|
||||
- Missing / empty / unparseable → `undefined`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `5xx` is intentionally handled as a range check, not as set membership, so the constants module doesn't need 100 entries.
|
||||
- The `Retry-After` cap is **distinct from** `maxDelayMs`; the latter governs exponential backoff, the former specifically defends against the "1 hour" abuse pattern.
|
||||
- `sleep` listens to the abort event with `{ once: true }` and tears down both the abort listener and the underlying timer on either resolution path — no listener leaks.
|
||||
- The helper is `fetch`-shaped (returns a `Response`) rather than generic; this is deliberate. HTTP semantics drive every retry decision, and a generic helper would either obscure the classifier or duplicate it.
|
||||
- Exhaustion returns the last response (when one was received) rather than throwing — `HttpStorage`'s wrapper is the right place to map a final `5xx` to `STORAGE_HTTP.UPSTREAM_ERROR` because it has the URL and key context.
|
||||
@@ -0,0 +1,356 @@
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import {
|
||||
isTransientError,
|
||||
isTransientStatus,
|
||||
parseRetryAfter,
|
||||
retryWithBackoff,
|
||||
} from "./retry-helper.js";
|
||||
|
||||
function controlledTimers(): {
|
||||
timers: TimersLike;
|
||||
pending: { fn: () => void; ms: number; id: number }[];
|
||||
scheduled: { ms: number }[];
|
||||
flushNext(): void;
|
||||
} {
|
||||
const pending: { fn: () => void; ms: number; id: number }[] = [];
|
||||
const scheduled: { ms: number }[] = [];
|
||||
let nextId = 1;
|
||||
const timers: TimersLike = {
|
||||
setTimer: (handler, ms) => {
|
||||
const id = nextId++;
|
||||
pending.push({ fn: handler, ms, id });
|
||||
scheduled.push({ ms });
|
||||
return id;
|
||||
},
|
||||
clearTimer: (id) => {
|
||||
const idx = pending.findIndex((p) => p.id === id);
|
||||
if (idx >= 0) pending.splice(idx, 1);
|
||||
},
|
||||
};
|
||||
function flushNext(): void {
|
||||
const next = pending.shift();
|
||||
if (next !== undefined) next.fn();
|
||||
}
|
||||
return { timers, pending, scheduled, flushNext };
|
||||
}
|
||||
|
||||
describe("isTransientStatus", () => {
|
||||
it("returns true for 408, 425, 429", { timeout: 5000 }, () => {
|
||||
expect(isTransientStatus(408)).toBe(true);
|
||||
expect(isTransientStatus(425)).toBe(true);
|
||||
expect(isTransientStatus(429)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for any 5xx", { timeout: 5000 }, () => {
|
||||
expect(isTransientStatus(500)).toBe(true);
|
||||
expect(isTransientStatus(503)).toBe(true);
|
||||
expect(isTransientStatus(599)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for 200, 401, 404, 600", { timeout: 5000 }, () => {
|
||||
expect(isTransientStatus(200)).toBe(false);
|
||||
expect(isTransientStatus(401)).toBe(false);
|
||||
expect(isTransientStatus(404)).toBe(false);
|
||||
expect(isTransientStatus(600)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isTransientError", () => {
|
||||
it("returns true for StorageHttpError(NETWORK_ERROR)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
isTransientError(
|
||||
new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR,
|
||||
"fetch failed"
|
||||
)
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for other errors", { timeout: 5000 }, () => {
|
||||
expect(isTransientError(new Error("boom"))).toBe(false);
|
||||
expect(
|
||||
isTransientError(
|
||||
new StorageHttpError(STORAGE_HTTP_CODES.UNAUTHORIZED, "no")
|
||||
)
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseRetryAfter", () => {
|
||||
it("returns undefined for null / empty / unparseable", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseRetryAfter(null, 0)).toBeUndefined();
|
||||
expect(parseRetryAfter("", 0)).toBeUndefined();
|
||||
expect(parseRetryAfter("garbage", 0)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("parses delta-seconds form into ms", { timeout: 5000 }, () => {
|
||||
expect(parseRetryAfter("2", 0)).toBe(2000);
|
||||
});
|
||||
|
||||
it("caps delta-seconds at the documented ceiling", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(parseRetryAfter("3600", 0)).toBe(60_000);
|
||||
});
|
||||
|
||||
it("parses an HTTP date", { timeout: 5000 }, () => {
|
||||
const now = Date.UTC(2026, 0, 1, 0, 0, 0);
|
||||
const future = new Date(now + 5000).toUTCString();
|
||||
expect(parseRetryAfter(future, now)).toBeLessThanOrEqual(5000);
|
||||
expect(parseRetryAfter(future, now)).toBeGreaterThan(3000);
|
||||
});
|
||||
|
||||
it("clamps a past HTTP date to 0", { timeout: 5000 }, () => {
|
||||
const now = Date.UTC(2026, 0, 1, 0, 0, 0);
|
||||
const past = new Date(now - 5000).toUTCString();
|
||||
expect(parseRetryAfter(past, now)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retryWithBackoff", () => {
|
||||
it("retries 503 honouring Retry-After", { timeout: 5000 }, async () => {
|
||||
const { timers, scheduled, flushNext } = controlledTimers();
|
||||
const fetchFn = vi.fn(async () => {
|
||||
const calls = fetchFn.mock.calls.length;
|
||||
if (calls === 1)
|
||||
return new Response(null, {
|
||||
status: 503,
|
||||
headers: { "retry-after": "2" },
|
||||
});
|
||||
return new Response(new Uint8Array([1]), { status: 200 });
|
||||
});
|
||||
const promise = retryWithBackoff(() => fetchFn(), {
|
||||
retry: { maxAttempts: 3, baseDelayMs: 10, maxDelayMs: 100 },
|
||||
timers,
|
||||
});
|
||||
await Promise.resolve();
|
||||
flushNext();
|
||||
const response = await promise;
|
||||
expect(response.status).toBe(200);
|
||||
expect(fetchFn).toHaveBeenCalledTimes(2);
|
||||
expect(scheduled[0]?.ms).toBeGreaterThanOrEqual(2000);
|
||||
});
|
||||
|
||||
it("does not retry 401 Unauthorized", { timeout: 5000 }, async () => {
|
||||
const fetchFn = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 401 }));
|
||||
const response = await retryWithBackoff(() => fetchFn(), {
|
||||
retry: { maxAttempts: 5 },
|
||||
});
|
||||
expect(response.status).toBe(401);
|
||||
expect(fetchFn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("retries on NETWORK_ERROR", { timeout: 5000 }, async () => {
|
||||
const { timers, flushNext } = controlledTimers();
|
||||
const fetchFn = vi.fn(async () => {
|
||||
const calls = fetchFn.mock.calls.length;
|
||||
if (calls === 1) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR,
|
||||
"connection reset"
|
||||
);
|
||||
}
|
||||
return new Response(null, { status: 200 });
|
||||
});
|
||||
const promise = retryWithBackoff(() => fetchFn(), {
|
||||
retry: { maxAttempts: 3, baseDelayMs: 5, maxDelayMs: 10 },
|
||||
timers,
|
||||
});
|
||||
await Promise.resolve();
|
||||
flushNext();
|
||||
const response = await promise;
|
||||
expect(response.status).toBe(200);
|
||||
expect(fetchFn).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("re-raises AbortError unchanged", { timeout: 5000 }, async () => {
|
||||
const abort = Object.assign(new Error("aborted"), {
|
||||
name: "AbortError",
|
||||
});
|
||||
const fetchFn = vi.fn(async () => {
|
||||
throw abort;
|
||||
});
|
||||
try {
|
||||
await retryWithBackoff(() => fetchFn());
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBe(abort);
|
||||
}
|
||||
});
|
||||
|
||||
it("re-throws non-transient errors immediately", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi.fn(async () => {
|
||||
throw new Error("non-transient");
|
||||
});
|
||||
try {
|
||||
await retryWithBackoff(() => fetchFn(), {});
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error).message).toBe("non-transient");
|
||||
expect(fetchFn).toHaveBeenCalledTimes(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("exhausts attempts and returns the last transient response", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi.fn(async () => new Response(null, { status: 503 }));
|
||||
const response = await retryWithBackoff(() => fetchFn(), {
|
||||
retry: {
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 1,
|
||||
maxDelayMs: 2,
|
||||
honourRetryAfter: false,
|
||||
},
|
||||
});
|
||||
expect(response.status).toBe(503);
|
||||
expect(fetchFn).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("throws ABORTED when a pre-aborted signal is supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const fetchFn = vi.fn();
|
||||
try {
|
||||
await retryWithBackoff(() => fetchFn(), {
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.ABORTED
|
||||
);
|
||||
expect(fetchFn).not.toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
it("throws ABORTED when the signal aborts mid-wait", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { timers } = controlledTimers();
|
||||
const controller = new AbortController();
|
||||
const fetchFn = vi.fn(async () => new Response(null, { status: 503 }));
|
||||
const promise = retryWithBackoff(() => fetchFn(), {
|
||||
retry: {
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 100,
|
||||
honourRetryAfter: false,
|
||||
},
|
||||
timers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
await Promise.resolve();
|
||||
controller.abort();
|
||||
try {
|
||||
await promise;
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses defaultTimers when no timers are supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi.fn(async () => {
|
||||
const calls = fetchFn.mock.calls.length;
|
||||
if (calls === 1) return new Response(null, { status: 503 });
|
||||
return new Response(null, { status: 200 });
|
||||
});
|
||||
const response = await retryWithBackoff(() => fetchFn(), {
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 1 },
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
it("synthesises NETWORK_ERROR when maxAttempts is 0 (no response, no error)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await retryWithBackoff(
|
||||
async () => new Response(null, { status: 503 }),
|
||||
{
|
||||
retry: { maxAttempts: 0 },
|
||||
}
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("hits the default-timer clearTimer path when aborted mid-sleep", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
const fetchFn = vi.fn(async () => {
|
||||
if (fetchFn.mock.calls.length === 1) {
|
||||
globalThis.setTimeout(() => controller.abort(), 5);
|
||||
return new Response(null, { status: 503 });
|
||||
}
|
||||
return new Response(null, { status: 200 });
|
||||
});
|
||||
try {
|
||||
await retryWithBackoff(() => fetchFn(), {
|
||||
retry: {
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
honourRetryAfter: false,
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("hits the synchronous pre-aborted branch inside sleep", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { timers } = controlledTimers();
|
||||
const controller = new AbortController();
|
||||
const fetchFn = vi.fn(async () => {
|
||||
controller.abort();
|
||||
return new Response(null, { status: 503 });
|
||||
});
|
||||
try {
|
||||
await retryWithBackoff(() => fetchFn(), {
|
||||
retry: {
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 100,
|
||||
honourRetryAfter: false,
|
||||
},
|
||||
timers,
|
||||
signal: controller.signal,
|
||||
});
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import {
|
||||
DEFAULT_RETRY_BASE_DELAY_MS,
|
||||
DEFAULT_RETRY_MAX_ATTEMPTS,
|
||||
DEFAULT_RETRY_MAX_DELAY_MS,
|
||||
RETRY_AFTER_MAX_DELAY_MS,
|
||||
TRANSIENT_HTTP_STATUSES,
|
||||
} from "../constants/index.js";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { isAbortError } from "../fetch-adapter/fetch-adapter.js";
|
||||
import type { HttpRetryOptions } from "../types/retry.types.js";
|
||||
|
||||
/**
|
||||
* Returns true when `status` is in the documented transient set —
|
||||
* `408`, `425`, `429`, or any `5xx`.
|
||||
*/
|
||||
export function isTransientStatus(status: number): boolean {
|
||||
if (TRANSIENT_HTTP_STATUSES.has(status)) return true;
|
||||
return status >= 500 && status <= 599;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when `err` is a `StorageHttpError(NETWORK_ERROR)` — i.e.
|
||||
* `fetch` rejected with something other than a signal-driven abort.
|
||||
*/
|
||||
export function isTransientError(err: unknown): boolean {
|
||||
return (
|
||||
err instanceof StorageHttpError &&
|
||||
err.code === STORAGE_HTTP_CODES.NETWORK_ERROR
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `Retry-After` header value into a delay in ms. Supports the
|
||||
* delta-seconds form (e.g. `"2"`) and the HTTP-date form (e.g.
|
||||
* `"Wed, 21 Oct 2015 07:28:00 GMT"`). Capped at `RETRY_AFTER_MAX_DELAY_MS`
|
||||
* to defend against malicious origins that try to stall callers.
|
||||
* Returns `undefined` when the value is missing or unparseable.
|
||||
*/
|
||||
export function parseRetryAfter(
|
||||
value: string | null,
|
||||
now: number
|
||||
): number | undefined {
|
||||
if (value === null) return undefined;
|
||||
const trimmed = value.trim();
|
||||
if (trimmed.length === 0) return undefined;
|
||||
if (/^\d+$/.test(trimmed)) {
|
||||
const seconds = Number.parseInt(trimmed, 10);
|
||||
return Math.min(seconds * 1000, RETRY_AFTER_MAX_DELAY_MS);
|
||||
}
|
||||
const parsed = Date.parse(trimmed);
|
||||
if (Number.isNaN(parsed)) return undefined;
|
||||
const diff = parsed - now;
|
||||
if (diff <= 0) return 0;
|
||||
return Math.min(diff, RETRY_AFTER_MAX_DELAY_MS);
|
||||
}
|
||||
|
||||
export interface AttemptOutcome {
|
||||
/** Truthy when the attempt succeeded — value carried to the caller. */
|
||||
response?: Response;
|
||||
/** When the attempt failed and was retryable, the caller-provided wait. */
|
||||
retryAfterMs?: number;
|
||||
}
|
||||
|
||||
export interface RetryWithBackoffOptions {
|
||||
retry?: HttpRetryOptions;
|
||||
timers?: TimersLike;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt `fn` up to `maxAttempts` times. The fn returns either a `Response`
|
||||
* (caller decides whether it counts as success or transient) or rejects. The
|
||||
* caller signals "this was transient, sleep and retry" by re-running this
|
||||
* function — that's not how this helper works. Instead, the helper itself
|
||||
* inspects each `Response` via `classify` and each thrown error via
|
||||
* `isTransientError`, deciding when to retry.
|
||||
*/
|
||||
export async function retryWithBackoff(
|
||||
fn: () => Promise<Response>,
|
||||
options: RetryWithBackoffOptions = {}
|
||||
): Promise<Response> {
|
||||
const maxAttempts =
|
||||
options.retry?.maxAttempts ?? DEFAULT_RETRY_MAX_ATTEMPTS;
|
||||
const baseDelayMs =
|
||||
options.retry?.baseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS;
|
||||
const maxDelayMs = options.retry?.maxDelayMs ?? DEFAULT_RETRY_MAX_DELAY_MS;
|
||||
const honourRetryAfter = options.retry?.honourRetryAfter ?? true;
|
||||
const timers = options.timers ?? defaultTimers();
|
||||
|
||||
let lastResponse: Response | undefined;
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
throwIfAborted(options.signal);
|
||||
try {
|
||||
const response = await fn();
|
||||
if (response.ok || !isTransientStatus(response.status)) {
|
||||
return response;
|
||||
}
|
||||
lastResponse = response;
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
if (!isTransientError(err)) throw err;
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt === maxAttempts) break;
|
||||
const waitMs = computeDelay(
|
||||
attempt,
|
||||
baseDelayMs,
|
||||
maxDelayMs,
|
||||
honourRetryAfter ? lastResponse : undefined
|
||||
);
|
||||
await sleep(waitMs, timers, options.signal);
|
||||
}
|
||||
|
||||
if (lastResponse !== undefined) return lastResponse;
|
||||
throw (
|
||||
lastError ??
|
||||
new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.NETWORK_ERROR,
|
||||
"retry exhausted with no response or error captured"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function defaultTimers(): TimersLike {
|
||||
return {
|
||||
setTimer: (handler, ms) => Number(globalThis.setTimeout(handler, ms)),
|
||||
clearTimer: (id) => globalThis.clearTimeout(id as number),
|
||||
};
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted === true) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.ABORTED,
|
||||
"Operation aborted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function computeDelay(
|
||||
attempt: number,
|
||||
baseDelayMs: number,
|
||||
maxDelayMs: number,
|
||||
lastResponse: Response | undefined
|
||||
): number {
|
||||
if (lastResponse !== undefined) {
|
||||
const retryAfter = parseRetryAfter(
|
||||
lastResponse.headers.get("retry-after"),
|
||||
Date.now()
|
||||
);
|
||||
if (retryAfter !== undefined) return retryAfter;
|
||||
}
|
||||
const exponential = baseDelayMs * 2 ** (attempt - 1);
|
||||
return Math.min(exponential, maxDelayMs);
|
||||
}
|
||||
|
||||
function sleep(
|
||||
ms: number,
|
||||
timers: TimersLike,
|
||||
signal: AbortSignal | undefined
|
||||
): Promise<void> {
|
||||
if (ms <= 0) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
let cleanup: (() => void) | undefined;
|
||||
const id = timers.setTimer(() => {
|
||||
cleanup?.();
|
||||
resolve();
|
||||
}, ms);
|
||||
if (signal !== undefined) {
|
||||
const onAbort = (): void => {
|
||||
timers.clearTimer(id);
|
||||
cleanup?.();
|
||||
reject(
|
||||
new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.ABORTED,
|
||||
"Operation aborted"
|
||||
)
|
||||
);
|
||||
};
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
cleanup = () => signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
# etag-cache.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the configuration and entry types used by the in-memory `EtagCache`
|
||||
that backs `HttpStorage`'s conditional-GET path. `EtagCacheOptions` is the
|
||||
public configuration the consumer passes via `HttpStorageOptions.etagCache`;
|
||||
`EtagCacheEntry` is the internal record the cache stores per key.
|
||||
|
||||
## Features
|
||||
|
||||
- Single configuration knob (`capacity`) — keeps the cache contract tiny.
|
||||
- Optional fields throughout — every option has a sensible default.
|
||||
- `EtagCacheEntry` carries `lastModified` alongside `etag` and `body` so
|
||||
`getMetadata` can serve from cache without an extra HEAD.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type { EtagCacheOptions } from "@suaveplan/storage-http";
|
||||
|
||||
const options: EtagCacheOptions = { capacity: 256 };
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type {
|
||||
EtagCacheEntry,
|
||||
EtagCacheOptions,
|
||||
} from "@suaveplan/storage-http";
|
||||
import { EtagCache } from "@suaveplan/storage-http";
|
||||
|
||||
const cache = new EtagCache({ capacity: 1024 } satisfies EtagCacheOptions);
|
||||
|
||||
const entry: EtagCacheEntry = {
|
||||
etag: '"v1"',
|
||||
body: new Uint8Array([1, 2, 3]),
|
||||
lastModified: new Date().toUTCString(),
|
||||
};
|
||||
cache.set("a/b.bin", entry);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `interface EtagCacheOptions`
|
||||
|
||||
- `capacity?: number` — maximum entries kept simultaneously. Defaults to
|
||||
`DEFAULT_ETAG_CACHE_CAPACITY` (256). Values < 1 are clamped to 1.
|
||||
|
||||
### `interface EtagCacheEntry`
|
||||
|
||||
- `etag: string` — verbatim from the `ETag` response header.
|
||||
- `body: Uint8Array` — cached body.
|
||||
- `lastModified: string | undefined` — verbatim from `Last-Modified`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Body storage uses the `Uint8Array` reference verbatim; callers that
|
||||
need defensive copies must slice before inserting. The cache treats
|
||||
bodies as immutable from its own perspective, but JavaScript cannot
|
||||
enforce that — production code typically uploads the same buffer it
|
||||
stored, so the lack of copy is fine in practice.
|
||||
- `lastModified` is stored as the raw header string (not a parsed `Date`)
|
||||
so the cache stays trivially serialisable — useful when a future
|
||||
consumer adds a cross-process cache layer that persists entries
|
||||
between processes via JSON or msgpack.
|
||||
- The cache is keyed by sanitised storage key (`HttpStorage` calls
|
||||
`sanitizeKey` before any cache access), so two callers requesting the
|
||||
same logical key always share the same cache slot regardless of
|
||||
whether they passed Windows-style or POSIX-style separators.
|
||||
- Capacity is a hard upper bound, not an aspirational target — the LRU
|
||||
evicts the oldest entry the moment a `set` would otherwise breach it.
|
||||
Callers tuning for high cache-hit ratios should size generously.
|
||||
@@ -0,0 +1,12 @@
|
||||
/** Configuration for the in-memory ETag/body cache. */
|
||||
export interface EtagCacheOptions {
|
||||
/** Maximum entries kept simultaneously. Defaults to `DEFAULT_ETAG_CACHE_CAPACITY`. */
|
||||
capacity?: number;
|
||||
}
|
||||
|
||||
/** One cached read; the LRU keys these by sanitised storage key. */
|
||||
export interface EtagCacheEntry {
|
||||
etag: string;
|
||||
body: Uint8Array;
|
||||
lastModified: string | undefined;
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
# fetch.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares `FetchLike` — the structural shape `HttpStorage` and every helper
|
||||
in this package programs against. By depending on `FetchLike` rather than
|
||||
`globalThis.fetch` directly, the adapter stays trivially testable: tests
|
||||
inject a controlled `FetchLike` instance and never have to spy on the
|
||||
global.
|
||||
|
||||
## Features
|
||||
|
||||
- Single-type module — `FetchLike` only.
|
||||
- Matches `globalThis.fetch`'s public surface (no provider-specific extras).
|
||||
- Accepts `string`, `URL`, or `Request` inputs — all three valid `fetch`
|
||||
argument shapes.
|
||||
- Promise<Response> return matches the WHATWG fetch contract exactly.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type { FetchLike } from "@suaveplan/storage-http";
|
||||
|
||||
const fetchImpl: FetchLike = async (input, init) => {
|
||||
return new Response(new Uint8Array([1, 2, 3]));
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type { FetchLike } from "@suaveplan/storage-http";
|
||||
|
||||
function withLogging(inner: FetchLike): FetchLike {
|
||||
return async (input, init) => {
|
||||
const url = typeof input === "string" ? input : input.toString();
|
||||
console.log("fetch", url);
|
||||
const response = await inner(input, init);
|
||||
console.log("status", response.status);
|
||||
return response;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `type FetchLike`
|
||||
|
||||
```ts
|
||||
(input: string | URL | Request, init?: RequestInit) => Promise<Response>
|
||||
```
|
||||
|
||||
Any function matching that shape is a valid `FetchLike`. The default impl
|
||||
returned by `defaultFetch()` wraps `globalThis.fetch` and adds error
|
||||
mapping.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The type intentionally accepts all three documented fetch input shapes
|
||||
so consumers can pass `URL` or `Request` objects rather than always
|
||||
stringifying first. Tests typically pass strings for clarity; production
|
||||
code may pass `URL` instances to avoid re-parsing on each call.
|
||||
- `init` is `RequestInit` — the same options bag as native `fetch`. Tests
|
||||
inspect `init.signal` to verify AbortSignal threading from each public
|
||||
method on `HttpStorage` all the way down to the underlying request.
|
||||
- No DOM-only types leak into the public API — `RequestInit` is provided
|
||||
by `@types/node` as part of Node 18+'s built-in fetch typings, so the
|
||||
adapter typechecks identically under Node, Bun, Deno, browsers, and
|
||||
Cloudflare Workers without lib augmentation.
|
||||
- Returning a custom `FetchLike` is the canonical way to add cross-cutting
|
||||
concerns (logging, retry-classification overrides, response caching,
|
||||
client-side throttling) without monkey-patching the global `fetch`.
|
||||
Compose multiple `FetchLike` wrappers freely — they are plain async
|
||||
functions and stack cleanly.
|
||||
@@ -0,0 +1,5 @@
|
||||
// `FetchLike` is now canonical in `@suaveplan/types` (added by the
|
||||
// `add-types-fetch-like` OpenSpec change). This file re-exports the alias
|
||||
// so existing consumers that import `FetchLike` from
|
||||
// `@suaveplan/storage-http` continue to work without code changes.
|
||||
export type { FetchLike } from "@suaveplan/types";
|
||||
@@ -0,0 +1,47 @@
|
||||
import z from "zod";
|
||||
|
||||
/** Retry sub-schema; validates only the numeric/boolean knobs. */
|
||||
export const HttpRetryOptionsSchema = z.object({
|
||||
maxAttempts: z.number().int().positive().optional(),
|
||||
baseDelayMs: z.number().int().nonnegative().optional(),
|
||||
maxDelayMs: z.number().int().positive().optional(),
|
||||
honourRetryAfter: z.boolean().optional(),
|
||||
});
|
||||
|
||||
/** ETag-cache sub-schema. */
|
||||
export const EtagCacheOptionsSchema = z.object({
|
||||
capacity: z.number().int().positive().optional(),
|
||||
});
|
||||
|
||||
// `headers`, `fetch`, `uploader`, `keyEncoder`, `timers` carry function or
|
||||
// adapter shapes Zod cannot meaningfully describe; we accept them as
|
||||
// `unknown` here and let TypeScript enforce the structural contract at the
|
||||
// call site. The static type stays precise via the `HttpStorageOptions`
|
||||
// interface.
|
||||
const looseAdapter = z.unknown().optional();
|
||||
|
||||
/**
|
||||
* Constructor-boundary schema for `HttpStorageOptions`. `baseUrl` is enforced
|
||||
* to be an absolute http(s) URL with a trailing slash — without the slash,
|
||||
* `new URL(key, baseUrl)` silently truncates the last path segment.
|
||||
*/
|
||||
export const HttpStorageOptionsSchema = z.object({
|
||||
baseUrl: z
|
||||
.string()
|
||||
.url()
|
||||
.refine(
|
||||
(value) =>
|
||||
value.startsWith("http://") || value.startsWith("https://"),
|
||||
{ message: "baseUrl must use http:// or https://" }
|
||||
)
|
||||
.refine((value) => value.endsWith("/"), {
|
||||
message: "baseUrl must end with '/'",
|
||||
}),
|
||||
headers: looseAdapter,
|
||||
fetch: looseAdapter,
|
||||
uploader: looseAdapter,
|
||||
keyEncoder: looseAdapter,
|
||||
retry: HttpRetryOptionsSchema.optional(),
|
||||
timers: looseAdapter,
|
||||
etagCache: EtagCacheOptionsSchema.optional(),
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
# http-storage.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the public option types accepted by `HttpStorage` and the supporting
|
||||
helper types they reference. These declarations are the contract every
|
||||
consumer programs against: the constructor parses them through
|
||||
`HttpStorageOptionsSchema` at the boundary, and the running adapter trusts
|
||||
the shape thereafter.
|
||||
|
||||
## Features
|
||||
|
||||
- `HttpStorageOptions` — full option bag covering `baseUrl`, `headers`,
|
||||
`fetch`, `uploader`, `keyEncoder`, `retry`, `timers`, `etagCache`.
|
||||
- `HttpStorageOp` — discriminator the `headers` callback receives.
|
||||
- `HttpHeadersResolver` — static record or async per-op function.
|
||||
- `KeyEncoder` — pluggable per-segment encoder for CDNs whose signed-URL
|
||||
signatures need alternate encoding.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type { HttpStorageOptions } from "@suaveplan/storage-http";
|
||||
|
||||
const options: HttpStorageOptions = {
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type {
|
||||
HttpHeadersResolver,
|
||||
HttpStorageOp,
|
||||
HttpStorageOptions,
|
||||
KeyEncoder,
|
||||
} from "@suaveplan/storage-http";
|
||||
|
||||
const headers: HttpHeadersResolver = async (key, op: HttpStorageOp) => ({
|
||||
Authorization: `Bearer ${await fetchToken(op)}`,
|
||||
"x-tenant": tenantFor(key),
|
||||
});
|
||||
|
||||
const encoder: KeyEncoder = (key) =>
|
||||
key.split("/").map(encodeURIComponent).join("/");
|
||||
|
||||
const options: HttpStorageOptions = {
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
headers,
|
||||
keyEncoder: encoder,
|
||||
retry: { maxAttempts: 5, baseDelayMs: 200, honourRetryAfter: true },
|
||||
etagCache: { capacity: 1024 },
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `interface HttpStorageOptions`
|
||||
|
||||
- `baseUrl: string` — required absolute http(s) URL ending with `/`.
|
||||
- `headers?: HttpHeadersResolver` — static record or per-op function.
|
||||
- `fetch?: FetchLike` — injectable fetch; defaults to `defaultFetch()`.
|
||||
- `uploader?: HttpUploader` — pluggable write strategy.
|
||||
- `keyEncoder?: KeyEncoder` — defaults to per-segment `encodeURIComponent`.
|
||||
- `retry?: HttpRetryOptions` — retry tuning.
|
||||
- `timers?: TimersLike` — injectable timers for tests.
|
||||
- `etagCache?: EtagCacheOptions` — opt-in conditional-GET cache.
|
||||
|
||||
### `type HttpStorageOp = "get" | "head" | "getPartial" | "set" | "delete"`
|
||||
|
||||
Discriminator for the `headers` callback so it can return op-specific values
|
||||
(e.g. an upload `Content-Type` distinct from a read `If-None-Match`).
|
||||
|
||||
### `type HttpHeadersResolver`
|
||||
|
||||
Either `Record<string, string>` or
|
||||
`(key: string, op: HttpStorageOp) => Record<string, string> | Promise<…>`.
|
||||
|
||||
### `type KeyEncoder = (key: string) => string`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- All option fields except `baseUrl` are optional; constructing
|
||||
`HttpStorage` with just `{ baseUrl }` produces a fully-functional
|
||||
read-only adapter against a public origin.
|
||||
- `HttpHeadersResolver` may return a Promise — useful when the caller's
|
||||
token store is async (OAuth refresh, secrets manager fetch).
|
||||
- `HttpStorageOp` deliberately omits opaque values like `clear` or `list`
|
||||
because the adapter never invokes the `headers` callback for those —
|
||||
they short-circuit to `READ_ONLY` before any fetch is composed.
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import type { EtagCacheOptions } from "./etag-cache.types.js";
|
||||
import type { FetchLike } from "./fetch.types.js";
|
||||
import type { HttpRetryOptions } from "./retry.types.js";
|
||||
import type { HttpUploader } from "./uploader.types.js";
|
||||
|
||||
/**
|
||||
* Operations the `headers` callback may be invoked for. Lets the caller
|
||||
* inject per-op headers (e.g. an upload `Content-Type` separate from a
|
||||
* `If-None-Match` on reads).
|
||||
*/
|
||||
export type HttpStorageOp = "get" | "head" | "getPartial" | "set" | "delete";
|
||||
|
||||
/**
|
||||
* Per-request header resolver. Either a static record or a function
|
||||
* receiving the sanitised key and the operation. Promise return supported
|
||||
* for async credential lookup.
|
||||
*/
|
||||
export type HttpHeadersResolver =
|
||||
| Record<string, string>
|
||||
| ((
|
||||
key: string,
|
||||
op: HttpStorageOp
|
||||
) => Record<string, string> | Promise<Record<string, string>>);
|
||||
|
||||
/**
|
||||
* Caller-supplied key encoder. The default applies
|
||||
* `encodeURIComponent` per `/`-separated segment, preserving `/`. Override
|
||||
* for CDNs whose signed-URL signatures require alternate encoding.
|
||||
*/
|
||||
export type KeyEncoder = (key: string) => string;
|
||||
|
||||
/**
|
||||
* Public options accepted by `HttpStorage`. Validated at the constructor
|
||||
* boundary via `HttpStorageOptionsSchema`.
|
||||
*/
|
||||
export interface HttpStorageOptions {
|
||||
/**
|
||||
* Absolute http(s) base URL. MUST end with `/` so `new URL(key, baseUrl)`
|
||||
* resolves correctly. Validated by Zod.
|
||||
*/
|
||||
baseUrl: string;
|
||||
/** Static headers or per-request header resolver. */
|
||||
headers?: HttpHeadersResolver;
|
||||
/** Injectable `fetch` for tests; defaults to `defaultFetch()`. */
|
||||
fetch?: FetchLike;
|
||||
/** Optional pluggable write strategy. Without it, writes throw `READ_ONLY`. */
|
||||
uploader?: HttpUploader;
|
||||
/** Custom key encoder. Defaults to per-segment `encodeURIComponent`. */
|
||||
keyEncoder?: KeyEncoder;
|
||||
/** Retry tuning. */
|
||||
retry?: HttpRetryOptions;
|
||||
/** Optional injectable timers (for test-time fake clocks). */
|
||||
timers?: TimersLike;
|
||||
/** When provided, enables the conditional-GET ETag cache. */
|
||||
etagCache?: EtagCacheOptions;
|
||||
}
|
||||
19
packages/persistence/storage/storage-http/src/types/index.ts
Normal file
19
packages/persistence/storage/storage-http/src/types/index.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
export type { EtagCacheEntry, EtagCacheOptions } from "./etag-cache.types.js";
|
||||
export type { FetchLike } from "./fetch.types.js";
|
||||
export {
|
||||
EtagCacheOptionsSchema,
|
||||
HttpRetryOptionsSchema,
|
||||
HttpStorageOptionsSchema,
|
||||
} from "./http-storage.schemas.js";
|
||||
export type {
|
||||
HttpHeadersResolver,
|
||||
HttpStorageOp,
|
||||
HttpStorageOptions,
|
||||
KeyEncoder,
|
||||
} from "./http-storage.types.js";
|
||||
export type { HttpRetryOptions, TimersLike } from "./retry.types.js";
|
||||
export type {
|
||||
HttpUploader,
|
||||
PresignedUrlInfo,
|
||||
PresignedUrlSigner,
|
||||
} from "./uploader.types.js";
|
||||
@@ -0,0 +1,72 @@
|
||||
# retry.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the retry-tuning option shape consumed by both `retryWithBackoff`
|
||||
and `HttpStorage`, plus a re-export of the cross-package `TimersLike`
|
||||
contract from `@suaveplan/types`. Centralising these here keeps the
|
||||
helper module and the public class consistent — both see the same numeric
|
||||
knobs and the same injectable-clock surface.
|
||||
|
||||
## Features
|
||||
|
||||
- Single shape for retry tuning across the package.
|
||||
- `honourRetryAfter` flag lets callers opt out of server-driven waits when
|
||||
they don't trust upstream `Retry-After` headers.
|
||||
- Re-exported `TimersLike` so consumers can import retry types and the
|
||||
clock contract from the same module.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type { HttpRetryOptions } from "@suaveplan/storage-http";
|
||||
|
||||
const retry: HttpRetryOptions = { maxAttempts: 5 };
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type { HttpRetryOptions, TimersLike } from "@suaveplan/storage-http";
|
||||
import { FakeClock } from "@suaveplan/testing";
|
||||
|
||||
const clock = new FakeClock();
|
||||
const timers: TimersLike = {
|
||||
setTimer: (handler, ms) => clock.setTimeout(handler, ms),
|
||||
clearTimer: (id) => clock.clearTimeout(id as number),
|
||||
};
|
||||
const retry: HttpRetryOptions = {
|
||||
maxAttempts: 4,
|
||||
baseDelayMs: 100,
|
||||
maxDelayMs: 5_000,
|
||||
honourRetryAfter: true,
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `interface HttpRetryOptions`
|
||||
|
||||
- `maxAttempts?: number` — inclusive cap. Defaults to
|
||||
`DEFAULT_RETRY_MAX_ATTEMPTS` (3).
|
||||
- `baseDelayMs?: number` — initial exponential-backoff delay. Defaults to
|
||||
`DEFAULT_RETRY_BASE_DELAY_MS` (100).
|
||||
- `maxDelayMs?: number` — backoff cap. Defaults to
|
||||
`DEFAULT_RETRY_MAX_DELAY_MS` (30000).
|
||||
- `honourRetryAfter?: boolean` — defaults to `true`.
|
||||
|
||||
### Re-exported `TimersLike`
|
||||
|
||||
From `@suaveplan/types` — `{ setTimer, clearTimer }`. See that package's
|
||||
docs for the canonical contract.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `honourRetryAfter = true` is the safe default — most well-behaved
|
||||
origins use the header correctly. Set to `false` for origins known to
|
||||
return abusive values that exceed reasonable wait windows.
|
||||
- `RETRY_AFTER_MAX_DELAY_MS` (60s) caps the value the helper honours even
|
||||
when `honourRetryAfter = true` — a single defensive ceiling so
|
||||
malicious or misconfigured origins can't stall callers for hours.
|
||||
- `TimersLike` is re-exported rather than re-declared so there is exactly
|
||||
one canonical contract in the workspace.
|
||||
@@ -0,0 +1,15 @@
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
|
||||
/** Retry tuning for transient HTTP/network failures. */
|
||||
export interface HttpRetryOptions {
|
||||
/** Inclusive cap on attempts. Defaults to `DEFAULT_RETRY_MAX_ATTEMPTS`. */
|
||||
maxAttempts?: number;
|
||||
/** Initial backoff delay in ms. Defaults to `DEFAULT_RETRY_BASE_DELAY_MS`. */
|
||||
baseDelayMs?: number;
|
||||
/** Backoff cap in ms. Defaults to `DEFAULT_RETRY_MAX_DELAY_MS`. */
|
||||
maxDelayMs?: number;
|
||||
/** Whether to honour `Retry-After`. Defaults to `true`. */
|
||||
honourRetryAfter?: boolean;
|
||||
}
|
||||
|
||||
export type { TimersLike };
|
||||
@@ -0,0 +1,82 @@
|
||||
# uploader.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the pluggable write strategy interfaces consumed by `HttpStorage`.
|
||||
`HttpUploader` is the seam that lets `HttpStorage` stay read-only by
|
||||
default while still supporting writes against any HTTP origin a provider
|
||||
package decides to support. `PresignedUrlSigner` is the callback shape
|
||||
`presignedPutUploader` consumes — provider-agnostic glue between an
|
||||
arbitrary URL-signing function and a working uploader.
|
||||
|
||||
## Features
|
||||
|
||||
- `HttpUploader.put` — required write operation.
|
||||
- `HttpUploader.delete?` — optional; absence makes the adapter throw
|
||||
`STORAGE_HTTP.READ_ONLY` on `delete`.
|
||||
- `PresignedUrlSigner` — async function returning a signed URL plus
|
||||
optional headers.
|
||||
- `PresignedUrlInfo` — output shape of `PresignedUrlSigner`.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import type { HttpUploader } from "@suaveplan/storage-http";
|
||||
|
||||
const uploader: HttpUploader = {
|
||||
put: async (key, body) => {
|
||||
await fetch(`https://upload.example.com/${key}`, {
|
||||
method: "PUT",
|
||||
body: body as Uint8Array,
|
||||
});
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import type {
|
||||
HttpUploader,
|
||||
PresignedUrlInfo,
|
||||
PresignedUrlSigner,
|
||||
} from "@suaveplan/storage-http";
|
||||
|
||||
const signer: PresignedUrlSigner = async (key) => {
|
||||
const { url, expires } = await signMyProvider("PUT", key, 60);
|
||||
return {
|
||||
url,
|
||||
headers: { "x-expires": String(expires) },
|
||||
} satisfies PresignedUrlInfo;
|
||||
};
|
||||
|
||||
const uploader: HttpUploader = presignedPutUploader({ signer });
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `interface HttpUploader`
|
||||
|
||||
- `put(key, body, headers?, signal?): Promise<void>` — required.
|
||||
- `delete?(key, signal?): Promise<void>` — optional.
|
||||
|
||||
`body` accepts `Uint8Array` or `ReadableStream<Uint8Array>`.
|
||||
|
||||
### `type PresignedUrlSigner = (key: string) => Promise<PresignedUrlInfo>`
|
||||
|
||||
Async by design — many real signers require an upstream call (STS,
|
||||
signed-cookie API).
|
||||
|
||||
### `interface PresignedUrlInfo`
|
||||
|
||||
`{ url: string; headers?: Record<string, string> }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `delete` is intentionally optional because most signed-URL schemes are
|
||||
write-only; provider packages with native delete support populate it.
|
||||
- `put` accepts `ReadableStream<Uint8Array>` for forward compatibility,
|
||||
but the default `presignedPutUploader` currently treats body as bytes
|
||||
— provider packages with native multipart streaming override this.
|
||||
- Headers from the signer are merged with the adapter's per-request
|
||||
headers (caller wins on conflict) — see `uploader.md`.
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* Pluggable write strategy for `HttpStorage`. Provider-specific packages
|
||||
* implement this for S3 presigned URLs, R2 signed URLs, Rackspace Cloud
|
||||
* Files, BunnyCDN Storage, etc.; the core adapter ships no provider
|
||||
* coupling.
|
||||
*/
|
||||
export interface HttpUploader {
|
||||
/**
|
||||
* Upload bytes (or a stream of bytes) for the given key. The adapter
|
||||
* supplies a sanitised key, the per-request headers from the caller's
|
||||
* `headers` callback, and an `AbortSignal` plumbed from the public API.
|
||||
*/
|
||||
put(
|
||||
key: string,
|
||||
body: Uint8Array | ReadableStream<Uint8Array>,
|
||||
headers?: Record<string, string>,
|
||||
signal?: AbortSignal
|
||||
): Promise<void>;
|
||||
/**
|
||||
* Optional delete. Adapters that lack delete capability omit this; the
|
||||
* `HttpStorage.delete` method then throws `STORAGE_HTTP.READ_ONLY`.
|
||||
*/
|
||||
delete?(key: string, signal?: AbortSignal): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Signing-callback shape for `presignedPutUploader`. Implementations return a
|
||||
* URL (and optional headers) the adapter should `PUT` against. Async to
|
||||
* accommodate signers that require an upstream call (STS, signed-cookie API).
|
||||
*/
|
||||
export type PresignedUrlSigner = (key: string) => Promise<PresignedUrlInfo>;
|
||||
|
||||
/** Result of a `PresignedUrlSigner` call. */
|
||||
export interface PresignedUrlInfo {
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export type { PresignedPutUploaderOptions } from "./uploader.js";
|
||||
export { presignedPutUploader } from "./uploader.js";
|
||||
@@ -0,0 +1,71 @@
|
||||
# uploader
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the pluggable write strategy for `HttpStorage` and ships the only generic uploader implementation: `presignedPutUploader`. `HttpStorage` calls the injected `HttpUploader` on `set`/`delete`; without an uploader, the core is read-only and writes throw `STORAGE_HTTP.READ_ONLY`. Provider-specific packages (S3, R2, GCS, Azure Blob, Rackspace, BunnyCDN) supply their own signing logic via a `PresignedUrlSigner` callback, so this module stays zero-coupling.
|
||||
|
||||
## Features
|
||||
|
||||
- `HttpUploader` interface — `put(key, body, headers?, signal?): Promise<void>` plus optional `delete(key, signal?)`.
|
||||
- `presignedPutUploader({ signer, fetch? })` — composes a signer into a working uploader without baking in any provider's signing scheme.
|
||||
- Per-call signing: every `put` invocation resolves a fresh `(url, headers)` pair so short-lived signed URLs work naturally.
|
||||
- Headers from the signer and the per-request headers from `HttpStorage` are merged (caller's `headers` win on conflict — they are the more specific override).
|
||||
- Non-`ok` responses surface as `StorageHttpError(STORAGE_HTTP.UPSTREAM_ERROR)` with the URL and HTTP status carried for diagnostics.
|
||||
|
||||
## Usage: Basic
|
||||
|
||||
```ts
|
||||
import { presignedPutUploader } from "@suaveplan/storage-http";
|
||||
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async (key) => ({
|
||||
url: await signS3Url(key, { method: "PUT", ttlSeconds: 60 }),
|
||||
}),
|
||||
});
|
||||
await uploader.put("a/b.bin", bytes);
|
||||
```
|
||||
|
||||
## Usage: Advanced
|
||||
|
||||
```ts
|
||||
import { HttpStorage, presignedPutUploader } from "@suaveplan/storage-http";
|
||||
|
||||
const storage = new HttpStorage({
|
||||
baseUrl: "https://cdn.example.com/start/here/",
|
||||
uploader: presignedPutUploader({
|
||||
signer: async (key) => {
|
||||
const { url, headers } = await myPresigner.presign("PUT", key, 60);
|
||||
return { url, headers: { ...headers, "x-amz-meta-uploader": "suaveplan" } };
|
||||
},
|
||||
}),
|
||||
});
|
||||
await storage.set("a/b.bin", bytes);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `interface HttpUploader`
|
||||
|
||||
- `put(key, body, headers?, signal?): Promise<void>` — required.
|
||||
- `delete?(key, signal?): Promise<void>` — optional; when omitted, `HttpStorage.delete` throws `READ_ONLY`.
|
||||
|
||||
### `type PresignedUrlSigner = (key: string) => Promise<PresignedUrlInfo>`
|
||||
|
||||
`PresignedUrlInfo` = `{ url: string; headers?: Record<string, string> }`.
|
||||
|
||||
### `presignedPutUploader(options): HttpUploader`
|
||||
|
||||
- `options.signer: PresignedUrlSigner` — required.
|
||||
- `options.fetch?: FetchLike` — defaults to `globalThis.fetch`.
|
||||
- Returns an `HttpUploader` whose `put` calls `signer(key)`, merges headers, and issues a `PUT`. Errors mapped to `STORAGE_HTTP.UPSTREAM_ERROR` with `httpStatus` populated.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Header merging is `{ ...fromSigner, ...fromCaller }` — caller headers win because they typically encode per-request `Content-Type` or `Content-Disposition` that the signer can't know in advance.
|
||||
- `presignedPutUploader` does **not** sign the body — it expects the signer's URL and headers to already encompass whatever signing the destination requires (S3 SigV4 in the URL, Azure SAS in query, etc.).
|
||||
- A `delete` capability is **not** included by default. Provider-specific packages implement it; `presignedPutUploader` does not because most signed-URL schemes are write-only.
|
||||
- Request body type is `BodyInit` (the standard `fetch` body shape) — `Uint8Array` and `ReadableStream<Uint8Array>` both work; everything else is a programmer error caught at the type level.
|
||||
@@ -0,0 +1,127 @@
|
||||
import { describe, expect, it, vi } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
type StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import { presignedPutUploader } from "./uploader.js";
|
||||
|
||||
describe("presignedPutUploader", () => {
|
||||
it("resolves the URL once per put and issues a PUT to it", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 201 }));
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async (key) => ({
|
||||
url: `https://example.com/upload/${key}`,
|
||||
headers: { "x-signer": "test" },
|
||||
}),
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await uploader.put("a/b.bin", new Uint8Array([1, 2, 3]));
|
||||
expect(fetchFn).toHaveBeenCalledTimes(1);
|
||||
const [url, init] = fetchFn.mock.calls[0] as [string, RequestInit];
|
||||
expect(url).toBe("https://example.com/upload/a/b.bin");
|
||||
expect(init.method).toBe("PUT");
|
||||
expect((init.headers as Record<string, string>)["x-signer"]).toBe(
|
||||
"test"
|
||||
);
|
||||
});
|
||||
|
||||
it("merges per-request headers over signer headers (caller wins)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 200 }));
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async () => ({
|
||||
url: "https://example.com/x",
|
||||
headers: { "x-shared": "from-signer" },
|
||||
}),
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await uploader.put("k", new Uint8Array([1]), {
|
||||
"x-shared": "from-caller",
|
||||
"x-extra": "yes",
|
||||
});
|
||||
const [, init] = fetchFn.mock.calls[0] as [string, RequestInit];
|
||||
const headers = init.headers as Record<string, string>;
|
||||
expect(headers["x-shared"]).toBe("from-caller");
|
||||
expect(headers["x-extra"]).toBe("yes");
|
||||
});
|
||||
|
||||
it("threads the AbortSignal into the fetch init", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 200 }));
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async () => ({ url: "https://example.com/x" }),
|
||||
fetch: fetchFn,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
await uploader.put(
|
||||
"k",
|
||||
new Uint8Array([1]),
|
||||
undefined,
|
||||
controller.signal
|
||||
);
|
||||
const [, init] = fetchFn.mock.calls[0] as [string, RequestInit];
|
||||
expect(init.signal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it("surfaces non-2xx responses as STORAGE_HTTP.UPSTREAM_ERROR", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fetchFn = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(null, { status: 500, statusText: "boom" })
|
||||
);
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async () => ({ url: "https://example.com/x" }),
|
||||
fetch: fetchFn,
|
||||
});
|
||||
try {
|
||||
await uploader.put("k", new Uint8Array([1]));
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageHttpError).code).toBe(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR
|
||||
);
|
||||
expect((err as StorageHttpError).httpStatus).toBe(500);
|
||||
}
|
||||
});
|
||||
|
||||
it("works when the signer omits headers", { timeout: 5000 }, async () => {
|
||||
const fetchFn = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(new Response(null, { status: 200 }));
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async () => ({ url: "https://example.com/x" }),
|
||||
fetch: fetchFn,
|
||||
});
|
||||
await uploader.put("k", new Uint8Array([1]));
|
||||
expect(fetchFn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("falls back to globalThis.fetch when no fetch is supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const spy = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockResolvedValueOnce(new Response(null, { status: 200 }));
|
||||
try {
|
||||
const uploader = presignedPutUploader({
|
||||
signer: async () => ({ url: "https://example.com/x" }),
|
||||
});
|
||||
await uploader.put("k", new Uint8Array([1]));
|
||||
expect(spy).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import {
|
||||
STORAGE_HTTP_CODES,
|
||||
StorageHttpError,
|
||||
} from "../error-codes/error-codes.js";
|
||||
import type { FetchLike } from "../types/fetch.types.js";
|
||||
import type {
|
||||
HttpUploader,
|
||||
PresignedUrlSigner,
|
||||
} from "../types/uploader.types.js";
|
||||
|
||||
export interface PresignedPutUploaderOptions {
|
||||
signer: PresignedUrlSigner;
|
||||
/** Injectable fetch; defaults to `globalThis.fetch`. */
|
||||
fetch?: FetchLike;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct an `HttpUploader` that resolves the destination URL per call via
|
||||
* the caller-supplied `signer`, then issues a `PUT` against it. Provider
|
||||
* packages (S3, R2, GCS, Azure Blob, Rackspace TempURL, BunnyCDN) plug their
|
||||
* signing logic in here; this helper has zero provider knowledge.
|
||||
*/
|
||||
export function presignedPutUploader(
|
||||
options: PresignedPutUploaderOptions
|
||||
): HttpUploader {
|
||||
const fetchImpl: FetchLike =
|
||||
options.fetch ??
|
||||
((input, init) =>
|
||||
globalThis.fetch(
|
||||
input as Parameters<typeof globalThis.fetch>[0],
|
||||
init
|
||||
));
|
||||
return {
|
||||
async put(key, body, headers, signal) {
|
||||
const signed = await options.signer(key);
|
||||
const mergedHeaders: Record<string, string> = {
|
||||
...(signed.headers ?? {}),
|
||||
...(headers ?? {}),
|
||||
};
|
||||
const init: RequestInit = {
|
||||
method: "PUT",
|
||||
body: body as unknown as NonNullable<RequestInit["body"]>,
|
||||
headers: mergedHeaders,
|
||||
};
|
||||
if (signal !== undefined) init.signal = signal;
|
||||
const response = await fetchImpl(signed.url, init);
|
||||
if (!response.ok) {
|
||||
throw new StorageHttpError(
|
||||
STORAGE_HTTP_CODES.UPSTREAM_ERROR,
|
||||
`PUT to presigned URL failed: ${response.status} ${response.statusText}`,
|
||||
{ key, url: signed.url, httpStatus: response.status }
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
// Polyfills for test environment — none required; Node 18+ provides
|
||||
// `fetch`, `Response`, `Request`, `Headers`, `URL`, `AbortController`,
|
||||
// and `ReadableStream` natively. This file exists to mirror the sibling
|
||||
// `storage-file` package shape and to give vitest a stable setup file.
|
||||
export {};
|
||||
12
packages/persistence/storage/storage-http/tsconfig.json
Normal file
12
packages/persistence/storage/storage-http/tsconfig.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": ".",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts", "tests/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
12
packages/persistence/storage/storage-http/tsconfig.lib.json
Normal file
12
packages/persistence/storage/storage-http/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
38
packages/persistence/storage/storage-http/vite.config.ts
Normal file
38
packages/persistence/storage/storage-http/vite.config.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/storage-http",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@opentelemetry/api",
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/retry",
|
||||
/^@suaveplan\/storage/,
|
||||
"@suaveplan/types",
|
||||
"tslib",
|
||||
"zod",
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
28
packages/persistence/storage/storage-http/vitest.config.ts
Normal file
28
packages/persistence/storage/storage-http/vitest.config.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.spec-conformance.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user