Initial Commit
This commit is contained in:
88
packages/persistence/storage/storage-file/README.md
Normal file
88
packages/persistence/storage/storage-file/README.md
Normal file
@@ -0,0 +1,88 @@
|
||||
# @suaveplan/storage-file - Node.js filesystem AbstractStorage adapter for @suaveplan/storage: atomic writes via temp-rename, AbortSignal everywhere, strict key sanitisation, native Web Streams, partial reads, transient-error retry via @suaveplan/retry, full OTel surface.
|
||||
|
||||
## Summary
|
||||
|
||||
Node.js filesystem `AbstractStorage` adapter for `@suaveplan/storage`. Implements atomic writes via temp-file-then-rename, AbortSignal threaded through every async method, strict key sanitisation, transient-error retry via `@suaveplan/retry`, native Web Streams via `node:fs.createReadStream`/`createWriteStream`, and positioned partial reads. Full OpenTelemetry surface: per-op spans, latency histograms, byte-flow histograms, kind-split counters.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [bun-fs](./src/bun-fs/bun-fs.md) — `bun-fs` is a sibling of `node-fs` that satisfies the same structural `NodeFsLike` contract consumed by `NodeFsStorage`…
|
||||
- [bun-fs](./src/bun-fs/bun-fs.md) — `bun-fs` is a sibling of `node-fs` that satisfies the same structural `NodeFsLike` contract consumed by `NodeFsStorage`…
|
||||
- [key-sanitizer](./src/key-sanitizer/key-sanitizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [key-sanitizer](./src/key-sanitizer/key-sanitizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [node-fs](./src/node-fs/node-fs.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [node-fs](./src/node-fs/node-fs.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [node-fs-storage](./src/node-fs-storage/node-fs-storage.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - node-fs-storage.ops — Houses the file-system operat…
|
||||
- [node-fs-storage.helpers](./src/node-fs-storage/node-fs-storage.helpers.md) — Houses the pure helpers consumed by `NodeFsStorage` so the orchestrator class stays under the file-size budget and the…
|
||||
- [node-fs-storage](./src/node-fs-storage/node-fs-storage.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs --> - node-fs-storage.ops — Houses the file-system operat…
|
||||
- [node-fs-storage.ops](./src/node-fs-storage/node-fs-storage.ops.md) — Houses the file-system operation helpers consumed by `NodeFsStorage`. Each function here wraps a single boundary call w…
|
||||
- [otel](./src/otel/otel.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [otel](./src/otel/otel.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [registry](./src/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [registry](./src/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [retry-helper](./src/retry-helper/retry-helper.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [retry-helper](./src/retry-helper/retry-helper.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [streams](./src/streams/streams.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [streams](./src/streams/streams.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [error-codes](./src/error-codes.md) — Defines the frozen `STORAGE_FILE_CODES` registry and the `StorageFileError` class used by every public path in `@suavep…
|
||||
<!-- end:auto-docs -->
|
||||
- [key-sanitizer](./src/key-sanitizer/key-sanitizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [node-fs](./src/node-fs/node-fs.md) — <!-- begin:auto-docs --> _(no submodules)_ - [node-fs](./src/node-fs/node-fs.md) — <!-- begin:auto-docs --> _(no submodules)_- [node-fs-storage](./src/node-fs-storage/node-fs-storage.md) — <!-- begin:auto-docs --> _(no submodules)_ - [node-fs-storage.helpers](./src/node-fs-storage/node-fs-storage.helpers.md) — Houses the pure helpers consumed by `NodeFsStorage` so the orchestrator class stays under the file-size budget and the…
|
||||
- [node-fs-storage](./src/node-fs-storage/node-fs-storage.md) — <!-- begin:auto-docs --> _(no submodules)_ - [node-fs-storage.ops](./src/node-fs-storage/node-fs-storage.ops.md) — Houses the file-system operation helpers consumed by `NodeFsStorage`. Each function here wraps a single boundary call w…
|
||||
- [otel](./src/otel/otel.md) — <!-- begin:auto-docs --> _(no submodules)_ - [otel](./src/otel/otel.md) — <!-- begin:auto-docs --> _(no submodules)_- [registry](./src/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_ - [registry](./src/registry/registry.md) — <!-- begin:auto-docs --> _(no submodules)_- [retry-helper](./src/retry-helper/retry-helper.md) — <!-- begin:auto-docs --> _(no submodules)_ - [retry-helper](./src/retry-helper/retry-helper.md) — <!-- begin:auto-docs --> _(no submodules)_- [streams](./src/streams/streams.md) — <!-- begin:auto-docs --> _(no submodules)_ - [streams](./src/streams/streams.md) — <!-- begin:auto-docs --> _(no submodules)_- [error-codes](./src/error-codes.md) — Defines the frozen `STORAGE_FILE_CODES` registry and the `StorageFileError` class used by every public path in `@suavep…
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/storage-file
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```ts
|
||||
import { NodeFsStorage } from "@suaveplan/storage-file";
|
||||
|
||||
const storage = new NodeFsStorage({ rootDir: "/var/lib/app/storage" });
|
||||
|
||||
await storage.set("orders/42.json", new TextEncoder().encode("{\"id\":42}"));
|
||||
const bytes = await storage.get("orders/42.json");
|
||||
console.log(new TextDecoder().decode(bytes!));
|
||||
|
||||
// Streams + partial reads
|
||||
const writer = storage.createWriteStream("large.bin").getWriter();
|
||||
await writer.write(new Uint8Array(1024 * 1024));
|
||||
await writer.close();
|
||||
const slice = await storage.getPartial("large.bin", 0, 16);
|
||||
|
||||
// Factory registration (optional)
|
||||
import { registerNodeFsProvider } from "@suaveplan/storage-file";
|
||||
import { createStorage } from "@suaveplan/storage/factory";
|
||||
registerNodeFsProvider();
|
||||
const viaFactory = createStorage({ type: "file", config: { rootDir: "/tmp/store" } });
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
- [`src/error-codes`](src/error-codes.md) — `STORAGE_FILE_CODES`, `StorageFileError`, `StorageFileErrorCode`
|
||||
- [`src/key-sanitizer/key-sanitizer`](src/key-sanitizer/key-sanitizer.md) — `sanitizeKey` strict boundary
|
||||
- [`src/node-fs/node-fs`](src/node-fs/node-fs.md) — `defaultNodeFs` adapter wrapping `node:fs/promises`
|
||||
- [`src/bun-fs/bun-fs`](src/bun-fs/bun-fs.md) — `defaultBunFs` / `pickDefaultFs` — Bun-native `Bun.file` / `Bun.write` fast path
|
||||
- [`src/retry-helper/retry-helper`](src/retry-helper/retry-helper.md) — `withFsRetry`, transient classifier
|
||||
- [`src/otel/otel`](src/otel/otel.md) — OTel instruments and `recordOp`
|
||||
- [`src/streams/streams`](src/streams/streams.md) — Web Streams bridge with atomic write
|
||||
- [`src/node-fs-storage/node-fs-storage`](src/node-fs-storage/node-fs-storage.md) — main `NodeFsStorage` class
|
||||
- [`src/node-fs-storage/node-fs-storage.helpers`](src/node-fs-storage/node-fs-storage.helpers.md) — pure helpers
|
||||
- [`src/registry/registry`](src/registry/registry.md) — factory registration hook
|
||||
- [`src/types/fs-adapter.types`](src/types/fs-adapter.types.md) — `NodeFsLike` interface
|
||||
- [`src/types/node-fs-storage.types`](src/types/node-fs-storage.types.md) — options + Zod schema
|
||||
|
||||
## Architecture
|
||||
|
||||
- **Tier**: server (consumes `node:fs/promises` and `node:fs` stream constructors). Browser consumers should pick `@suaveplan/storage/memory`, an HTTP/IndexedDB adapter, or `@suaveplan/storage-s3`.
|
||||
- **Base class**: extends `AbstractStorage` from `@suaveplan/storage/abstract`. Every consumer that talks to `AbstractStorage` (or the `StorageLike` shim) works against this adapter unchanged.
|
||||
- **Atomicity**: every public write (`set`, metadata sidecar, `createWriteStream.close`) writes to a temp file in the destination directory and `fs.rename`s onto the destination. POSIX rename is atomic within a filesystem.
|
||||
- **Cancellation**: every public async method accepts an optional `AbortSignal`. Pre-aborted signals throw `STORAGE_FILE.ABORTED` before any IO call.
|
||||
- **Resilience**: transient errors (`EBUSY`, `EMFILE`, `ENFILE`, `EAGAIN`) are retried via `@suaveplan/retry` with exponential backoff and injectable `TimersLike`. Non-transient errors surface as `STORAGE_FILE.IO_FAILED` or `STORAGE_FILE.PERMISSION_DENIED`.
|
||||
- **Security**: every public method passes its key through `sanitizeKey`. Path-traversal, absolute paths, drive-letter prefixes, null bytes, control chars, and over-length keys all surface as `STORAGE_FILE.INVALID_KEY`/`KEY_TOO_LONG` before any filesystem call.
|
||||
- **Observability**: OTel counters, histograms, and a tracer emit from production paths. Tests assert against a real in-memory MeterProvider — no `vi.mock`.
|
||||
|
||||
156
packages/persistence/storage/storage-file/package.json
Normal file
156
packages/persistence/storage/storage-file/package.json
Normal file
@@ -0,0 +1,156 @@
|
||||
{
|
||||
"name": "@suaveplan/storage-file",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Node.js filesystem AbstractStorage adapter for @suaveplan/storage: atomic writes via temp-rename, AbortSignal everywhere, strict key sanitisation, native Web Streams, partial reads, transient-error retry via @suaveplan/retry, full OTel surface.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./node-fs-storage": {
|
||||
"import": {
|
||||
"types": "./dist/node-fs-storage/index.d.ts",
|
||||
"default": "./dist/node-fs-storage/node-fs-storage.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/node-fs-storage/index.d.ts",
|
||||
"default": "./dist/node-fs-storage/node-fs-storage.js"
|
||||
}
|
||||
},
|
||||
"./key-sanitizer": {
|
||||
"import": {
|
||||
"types": "./dist/key-sanitizer/index.d.ts",
|
||||
"default": "./dist/key-sanitizer/key-sanitizer.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/key-sanitizer/index.d.ts",
|
||||
"default": "./dist/key-sanitizer/key-sanitizer.js"
|
||||
}
|
||||
},
|
||||
"./node-fs": {
|
||||
"import": {
|
||||
"types": "./dist/node-fs/index.d.ts",
|
||||
"default": "./dist/node-fs/node-fs.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/node-fs/index.d.ts",
|
||||
"default": "./dist/node-fs/node-fs.js"
|
||||
}
|
||||
},
|
||||
"./bun-fs": {
|
||||
"import": {
|
||||
"types": "./dist/bun-fs/index.d.ts",
|
||||
"default": "./dist/bun-fs/bun-fs.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/bun-fs/index.d.ts",
|
||||
"default": "./dist/bun-fs/bun-fs.js"
|
||||
}
|
||||
},
|
||||
"./retry-helper": {
|
||||
"import": {
|
||||
"types": "./dist/retry-helper/index.d.ts",
|
||||
"default": "./dist/retry-helper/retry-helper.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/retry-helper/index.d.ts",
|
||||
"default": "./dist/retry-helper/retry-helper.js"
|
||||
}
|
||||
},
|
||||
"./otel": {
|
||||
"import": {
|
||||
"types": "./dist/otel/index.d.ts",
|
||||
"default": "./dist/otel/otel.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/otel/index.d.ts",
|
||||
"default": "./dist/otel/otel.js"
|
||||
}
|
||||
},
|
||||
"./streams": {
|
||||
"import": {
|
||||
"types": "./dist/streams/index.d.ts",
|
||||
"default": "./dist/streams/streams.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/streams/index.d.ts",
|
||||
"default": "./dist/streams/streams.js"
|
||||
}
|
||||
},
|
||||
"./registry": {
|
||||
"import": {
|
||||
"types": "./dist/registry/index.d.ts",
|
||||
"default": "./dist/registry/registry.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/registry/index.d.ts",
|
||||
"default": "./dist/registry/registry.js"
|
||||
}
|
||||
},
|
||||
"./types": {
|
||||
"import": {
|
||||
"types": "./dist/types/index.d.ts",
|
||||
"default": "./dist/types/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/types/index.d.ts",
|
||||
"default": "./dist/types/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/retry": "workspace:^",
|
||||
"@suaveplan/storage": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"fast-check": "catalog:testing",
|
||||
"mitata": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
126
packages/persistence/storage/storage-file/src/bun-fs/bun-fs.md
Normal file
126
packages/persistence/storage/storage-file/src/bun-fs/bun-fs.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# `bun-fs` — Bun-native `NodeFsLike` adapter
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
`bun-fs` is a sibling of [`node-fs`](../node-fs/node-fs.md) that satisfies the
|
||||
same structural `NodeFsLike` contract consumed by `NodeFsStorage`. Where
|
||||
`defaultNodeFs()` wires every method to `node:fs/promises` and the two
|
||||
`node:fs` stream constructors, `defaultBunFs()` overrides only the two
|
||||
methods that have a measurably faster Bun-native path — `readFile` (routed
|
||||
through `Bun.file(path).bytes()`) and `writeFile` (routed through
|
||||
`Bun.write(path, data)`). Every other method delegates to the Node base
|
||||
because under Bun those calls already dispatch to Bun's native (Zig)
|
||||
implementation transparently via the Node-compat layer, leaving no perf on
|
||||
the table.
|
||||
|
||||
This module exists so Suaveplan — which targets Bun as its first
|
||||
runtime — does not pay the Node-compat reshaping cost on the hot read/write
|
||||
loop. It is additive: no behaviour changes for callers that inject a custom
|
||||
`fs`, and the Node adapter remains the explicit choice when running outside
|
||||
Bun or when parity testing under Bun.
|
||||
|
||||
## Features
|
||||
|
||||
- `defaultBunFs(): NodeFsLike` — Bun-native adapter. Throws a plain `Error`
|
||||
if called outside a Bun runtime.
|
||||
- `pickDefaultFs(): NodeFsLike` — runtime-probed selector. Returns
|
||||
`defaultBunFs()` when `globalThis.Bun` is defined, otherwise
|
||||
`defaultNodeFs()`. The probe runs on every call (never cached) so tests
|
||||
can switch adapters deterministically.
|
||||
- Returns a fresh object per call so callers can selectively override any
|
||||
single method without mutating shared state.
|
||||
- Zero runtime branching inside individual methods — composition over
|
||||
conditionals.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { NodeFsStorage, defaultBunFs } from "@suaveplan/storage-file";
|
||||
|
||||
const storage = new NodeFsStorage({
|
||||
rootDir: "/var/data",
|
||||
fs: defaultBunFs(),
|
||||
});
|
||||
|
||||
await storage.set("hello", new TextEncoder().encode("world"));
|
||||
const bytes = await storage.get("hello");
|
||||
console.log(new TextDecoder().decode(bytes!)); // "world"
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
Combine runtime detection with a custom override — useful when you want
|
||||
the Bun fast path everywhere except a tracing wrapper around `writeFile`:
|
||||
|
||||
```ts
|
||||
import {
|
||||
NodeFsStorage,
|
||||
pickDefaultFs,
|
||||
type NodeFsLike,
|
||||
} from "@suaveplan/storage-file";
|
||||
|
||||
function tracedFs(): NodeFsLike {
|
||||
const base = pickDefaultFs();
|
||||
return {
|
||||
...base,
|
||||
async writeFile(path, data) {
|
||||
const start = performance.now();
|
||||
try {
|
||||
await base.writeFile(path, data);
|
||||
} finally {
|
||||
console.info("writeFile", path, performance.now() - start);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const storage = new NodeFsStorage({
|
||||
rootDir: "/var/data",
|
||||
fs: tracedFs(),
|
||||
});
|
||||
```
|
||||
|
||||
When you construct `NodeFsStorage` without passing `fs`, `resolveConfig`
|
||||
calls `pickDefaultFs()` for you — Bun-first, with Node fallback.
|
||||
|
||||
## API Reference
|
||||
|
||||
### `defaultBunFs(): NodeFsLike`
|
||||
|
||||
- **Returns** a fresh `NodeFsLike` whose `readFile` is implemented as
|
||||
`Bun.file(path).bytes()` and whose `writeFile` is implemented as
|
||||
`Bun.write(path, data)`. Every other method (`mkdir`, `readdir`,
|
||||
`rename`, `unlink`, `stat`, `access`, `open`, `createReadStream`,
|
||||
`createWriteStream`) delegates to the `defaultNodeFs()` base.
|
||||
- **Throws** a plain `Error` with the message
|
||||
`defaultBunFs() requires a Bun runtime (globalThis.Bun is undefined)` if
|
||||
the probe finds no Bun global. The intended entry point for adapter
|
||||
selection is `pickDefaultFs`, which performs the probe first.
|
||||
|
||||
### `pickDefaultFs(): NodeFsLike`
|
||||
|
||||
- **Returns** `defaultBunFs()` when `globalThis.Bun` is an object, otherwise
|
||||
`defaultNodeFs()`. Consumed by `resolveConfig` in
|
||||
`node-fs-storage.helpers.ts` to wire the default `fs` for `NodeFsStorage`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Streams are NOT routed through `Bun.file(path).stream()`. `BunFile.stream()`
|
||||
returns a Web `ReadableStream`, but `NodeFsLike.createReadStream` is typed
|
||||
as a Node `Readable`. Translating between the two introduces a copy that
|
||||
erases the speedup, so the Node `node:fs` constructor is used unchanged —
|
||||
it already dispatches to Bun's native impl under Bun.
|
||||
- The detection probe uses `typeof globalThis.Bun === "object"` rather than
|
||||
`"undefined" !== typeof Bun`. The former works without TypeScript ambient
|
||||
globals and is robust to module-scope mocking via
|
||||
`(globalThis as { Bun?: unknown }).Bun = undefined`.
|
||||
- `Bun.write` accepts both `Uint8Array` and `string`, matching
|
||||
`NodeFsLike.writeFile`'s union type. No buffer copying is performed by
|
||||
the adapter.
|
||||
- The adapter has zero `@suaveplan/*` runtime dependencies. It is internally
|
||||
consumed by `node-fs-storage.helpers.resolveConfig`, which is the only
|
||||
module that should call `pickDefaultFs` directly.
|
||||
@@ -0,0 +1,319 @@
|
||||
import {
|
||||
mkdtemp,
|
||||
readFile as nodeReadFile,
|
||||
writeFile as nodeWriteFile,
|
||||
rm,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { NodeFsStorage } from "../node-fs-storage/node-fs-storage.js";
|
||||
import {
|
||||
__setBunProbeForTest,
|
||||
__setBunRuntimeForTest,
|
||||
defaultBunFs,
|
||||
pickDefaultFs,
|
||||
} from "./bun-fs.js";
|
||||
|
||||
interface FakeBunRuntime {
|
||||
file(path: string): { bytes(): Promise<Uint8Array> };
|
||||
write(path: string, data: Uint8Array | string): Promise<number>;
|
||||
}
|
||||
|
||||
// `globalThis.Bun` is published as `configurable: false, writable: false`
|
||||
// on real Bun runtimes — direct assignment throws in strict mode. The
|
||||
// production code exposes `__setBunRuntimeForTest` precisely so tests can
|
||||
// substitute a fake without touching the readonly global.
|
||||
function installFakeBun(): {
|
||||
restore: () => void;
|
||||
calls: { file: number; write: number };
|
||||
} {
|
||||
const calls = { file: 0, write: 0 };
|
||||
const fake: FakeBunRuntime = {
|
||||
file(path) {
|
||||
return {
|
||||
async bytes() {
|
||||
calls.file++;
|
||||
const buf = await nodeReadFile(path);
|
||||
return new Uint8Array(
|
||||
buf.buffer,
|
||||
buf.byteOffset,
|
||||
buf.byteLength
|
||||
);
|
||||
},
|
||||
};
|
||||
},
|
||||
async write(path, data) {
|
||||
calls.write++;
|
||||
const payload =
|
||||
typeof data === "string"
|
||||
? new TextEncoder().encode(data)
|
||||
: data;
|
||||
await nodeWriteFile(path, payload);
|
||||
return payload.byteLength;
|
||||
},
|
||||
};
|
||||
__setBunRuntimeForTest(fake);
|
||||
return {
|
||||
restore: () => {
|
||||
__setBunRuntimeForTest(undefined);
|
||||
},
|
||||
calls,
|
||||
};
|
||||
}
|
||||
|
||||
describe("defaultBunFs", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "storage-file-bun-fs-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("round-trips bytes through Bun.write and Bun.file", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const guard = installFakeBun();
|
||||
try {
|
||||
const fs = defaultBunFs();
|
||||
const path = join(root, "round.bin");
|
||||
const payload = new Uint8Array([0xde, 0xad, 0xbe, 0xef]);
|
||||
await fs.writeFile(path, payload);
|
||||
const read = await fs.readFile(path);
|
||||
expect(Array.from(read)).toEqual(Array.from(payload));
|
||||
expect(guard.calls.write).toBe(1);
|
||||
expect(guard.calls.file).toBe(1);
|
||||
} finally {
|
||||
guard.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards mkdir, readdir, stat, access, rename, unlink, open, and the stream constructors", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const guard = installFakeBun();
|
||||
try {
|
||||
const fs = defaultBunFs();
|
||||
const dir = join(root, "nested");
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const path = join(dir, "k.data");
|
||||
await fs.writeFile(path, new TextEncoder().encode("hi"));
|
||||
await fs.access(path);
|
||||
|
||||
const stat = await fs.stat(path);
|
||||
expect(stat.size).toBe(2);
|
||||
expect(stat.isFile()).toBe(true);
|
||||
expect(stat.isDirectory()).toBe(false);
|
||||
|
||||
const renamed = join(dir, "moved.data");
|
||||
await fs.rename(path, renamed);
|
||||
|
||||
const entries = await fs.readdir(root, {
|
||||
recursive: true,
|
||||
withFileTypes: true,
|
||||
});
|
||||
const dirEntry = entries.find((e) => e.name === "nested");
|
||||
const fileEntry = entries.find((e) => e.name === "moved.data");
|
||||
expect(dirEntry?.isDirectory()).toBe(true);
|
||||
expect(dirEntry?.isFile()).toBe(false);
|
||||
expect(fileEntry?.isFile()).toBe(true);
|
||||
expect(fileEntry?.isDirectory()).toBe(false);
|
||||
expect(typeof fileEntry?.parentPath).toBe("string");
|
||||
|
||||
const handle = await fs.open(renamed, "r");
|
||||
try {
|
||||
const buf = new Uint8Array(2);
|
||||
const result = await handle.read(buf, 0, 2, 0);
|
||||
expect(result.bytesRead).toBe(2);
|
||||
expect(new TextDecoder().decode(result.buffer)).toBe("hi");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
|
||||
const streamPath = join(dir, "s.data");
|
||||
await fs.writeFile(streamPath, new TextEncoder().encode("abc"));
|
||||
const reader = fs.createReadStream(streamPath);
|
||||
const chunks: Buffer[] = [];
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
reader.on("data", (c) => chunks.push(c as Buffer));
|
||||
reader.on("end", () => resolve());
|
||||
reader.on("error", reject);
|
||||
});
|
||||
expect(
|
||||
new TextDecoder().decode(new Uint8Array(Buffer.concat(chunks)))
|
||||
).toBe("abc");
|
||||
|
||||
const writePath = join(dir, "out.data");
|
||||
const writer = fs.createWriteStream(writePath);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
writer.write(Buffer.from("xyz"));
|
||||
writer.end((err: Error | null | undefined) =>
|
||||
err ? reject(err) : resolve()
|
||||
);
|
||||
});
|
||||
expect(new TextDecoder().decode(await fs.readFile(writePath))).toBe(
|
||||
"xyz"
|
||||
);
|
||||
|
||||
await fs.unlink(renamed);
|
||||
} finally {
|
||||
guard.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("Bun.write accepts string data without re-encoding", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const guard = installFakeBun();
|
||||
try {
|
||||
const fs = defaultBunFs();
|
||||
const path = join(root, "string.txt");
|
||||
await fs.writeFile(path, "hello");
|
||||
const read = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(read)).toBe("hello");
|
||||
} finally {
|
||||
guard.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("pickDefaultFs returns a working NodeFsLike under Bun", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const guard = installFakeBun();
|
||||
try {
|
||||
const fs = pickDefaultFs();
|
||||
const path = join(root, "pick.bin");
|
||||
const payload = new TextEncoder().encode("picked");
|
||||
await fs.writeFile(path, payload);
|
||||
const read = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(read)).toBe("picked");
|
||||
expect(guard.calls.write).toBe(1);
|
||||
expect(guard.calls.file).toBe(1);
|
||||
} finally {
|
||||
guard.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("NodeFsStorage round-trips through the auto-picked Bun fs", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const guard = installFakeBun();
|
||||
try {
|
||||
const storage = new NodeFsStorage({ rootDir: root });
|
||||
const payload = new Uint8Array([1, 2, 3, 4, 5]);
|
||||
await storage.set("auto/key", payload);
|
||||
const got = await storage.get("auto/key");
|
||||
expect(got).toBeDefined();
|
||||
expect(got && Array.from(got)).toEqual(Array.from(payload));
|
||||
expect(guard.calls.write).toBeGreaterThan(0);
|
||||
expect(guard.calls.file).toBeGreaterThan(0);
|
||||
} finally {
|
||||
guard.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("throws when invoked outside a Bun runtime", { timeout: 5000 }, () => {
|
||||
try {
|
||||
__setBunRuntimeForTest(null);
|
||||
expect(() => defaultBunFs()).toThrow(/requires a Bun runtime/i);
|
||||
} finally {
|
||||
__setBunRuntimeForTest(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("pickDefaultFs falls back to defaultNodeFs when Bun is absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
__setBunRuntimeForTest(null);
|
||||
const fs = pickDefaultFs();
|
||||
const path = join(root, "fallback.bin");
|
||||
await fs.writeFile(path, new TextEncoder().encode("ok"));
|
||||
const read = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(read)).toBe("ok");
|
||||
} finally {
|
||||
__setBunRuntimeForTest(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("pickDefaultFs falls back when Bun probe returns a non-object", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// Pins the `typeof candidate === "object"` false-branch in
|
||||
// getBunRuntime. Bun publishes globalThis.Bun as
|
||||
// `configurable: false` — `Object.defineProperty` throws under a
|
||||
// real Bun runtime — so tests must inject through the dedicated
|
||||
// probe hook rather than mutating the global.
|
||||
try {
|
||||
__setBunRuntimeForTest(undefined);
|
||||
__setBunProbeForTest(() => "not-an-object");
|
||||
const fs = pickDefaultFs();
|
||||
const path = join(root, "non-object-bun.bin");
|
||||
await fs.writeFile(path, new TextEncoder().encode("ok"));
|
||||
const read = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(read)).toBe("ok");
|
||||
} finally {
|
||||
__setBunProbeForTest(undefined);
|
||||
__setBunRuntimeForTest(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("getBunRuntime returns the Bun probe value when it is an object", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// Pins the truthy-ternary branch (typeof === "object" &&
|
||||
// candidate !== null). Inject the Bun runtime through the probe
|
||||
// hook so the production code path is the same as a real Bun
|
||||
// process.
|
||||
const fakeBun = {
|
||||
file: (p: string) => ({
|
||||
bytes: async () => nodeReadFile(p),
|
||||
}),
|
||||
write: async (p: string, d: Uint8Array | string) => {
|
||||
const payload =
|
||||
typeof d === "string" ? new TextEncoder().encode(d) : d;
|
||||
await nodeWriteFile(p, payload);
|
||||
return payload.byteLength;
|
||||
},
|
||||
};
|
||||
try {
|
||||
__setBunRuntimeForTest(undefined);
|
||||
__setBunProbeForTest(() => fakeBun);
|
||||
const fs = pickDefaultFs();
|
||||
const path = join(root, "globalthis-bun.bin");
|
||||
await fs.writeFile(path, new TextEncoder().encode("ok"));
|
||||
const read = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(read)).toBe("ok");
|
||||
} finally {
|
||||
__setBunProbeForTest(undefined);
|
||||
__setBunRuntimeForTest(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("pickDefaultFs falls back when Bun probe returns null", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// Pins the `candidate !== null` false-branch — typeof null is
|
||||
// "object" in JS, so the first conjunct passes; the null check
|
||||
// forces undefined.
|
||||
try {
|
||||
__setBunRuntimeForTest(undefined);
|
||||
__setBunProbeForTest(() => null);
|
||||
const fs = pickDefaultFs();
|
||||
const path = join(root, "null-bun.bin");
|
||||
await fs.writeFile(path, new TextEncoder().encode("ok"));
|
||||
const read = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(read)).toBe("ok");
|
||||
} finally {
|
||||
__setBunProbeForTest(undefined);
|
||||
__setBunRuntimeForTest(undefined);
|
||||
}
|
||||
});
|
||||
});
|
||||
112
packages/persistence/storage/storage-file/src/bun-fs/bun-fs.ts
Normal file
112
packages/persistence/storage/storage-file/src/bun-fs/bun-fs.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
import { defaultNodeFs } from "../node-fs/node-fs.js";
|
||||
import type { NodeFsLike } from "../types/fs-adapter.types.js";
|
||||
|
||||
/**
|
||||
* Reference to Bun's runtime global, narrowed to the two APIs the adapter
|
||||
* actually consumes. Declared via `globalThis` so the module type-checks
|
||||
* cleanly under both Bun (where `Bun` is ambient) and Node (where it is not).
|
||||
*/
|
||||
interface BunRuntime {
|
||||
file(path: string): { bytes(): Promise<Uint8Array> };
|
||||
write(
|
||||
path: string,
|
||||
data: Uint8Array | string,
|
||||
options?: { createPath?: boolean }
|
||||
): Promise<number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Module-scope test override slot. Tests can't mutate `globalThis.Bun`
|
||||
* directly on real Bun runtimes — Bun publishes it as
|
||||
* `configurable: false, writable: false`, so any `globalThis.Bun = …`
|
||||
* assignment throws `TypeError: Attempted to assign to readonly property`
|
||||
* in strict mode. Tests inject through this slot via
|
||||
* {@link __setBunRuntimeForTest} instead.
|
||||
*/
|
||||
let bunRuntimeOverride: BunRuntime | null | undefined;
|
||||
|
||||
const defaultBunProbe = (): unknown => (globalThis as { Bun?: BunRuntime }).Bun;
|
||||
let bunProbe: () => unknown = defaultBunProbe;
|
||||
|
||||
/**
|
||||
* Test-only hook: substitute (or clear) the Bun runtime probed by
|
||||
* {@link getBunRuntime}. Pass a fake runtime to inject; pass `null` to
|
||||
* force the "no Bun present" branch; pass `undefined` to restore the
|
||||
* real `globalThis.Bun` probe. Not part of the public surface.
|
||||
*/
|
||||
export function __setBunRuntimeForTest(
|
||||
runtime: BunRuntime | null | undefined
|
||||
): void {
|
||||
bunRuntimeOverride = runtime;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only hook: replace the `globalThis.Bun` probe with a custom
|
||||
* factory. Lets tests exercise the typeof/null branches in
|
||||
* {@link getBunRuntime} without mutating `globalThis.Bun` — Bun
|
||||
* publishes that global as `configurable: false`, so `defineProperty`
|
||||
* throws under a real Bun runtime. Pass `undefined` to restore the
|
||||
* default `globalThis.Bun` probe.
|
||||
*/
|
||||
export function __setBunProbeForTest(probe: (() => unknown) | undefined): void {
|
||||
bunProbe = probe ?? defaultBunProbe;
|
||||
}
|
||||
|
||||
function getBunRuntime(): BunRuntime | undefined {
|
||||
if (bunRuntimeOverride !== undefined) {
|
||||
return bunRuntimeOverride ?? undefined;
|
||||
}
|
||||
const candidate = bunProbe();
|
||||
return typeof candidate === "object" && candidate !== null
|
||||
? (candidate as BunRuntime)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bun-native {@link NodeFsLike} adapter. `readFile` and `writeFile` route
|
||||
* through `Bun.file(...).bytes()` and `Bun.write(...)` — the only filesystem
|
||||
* methods on the contract that have a measurably faster Bun-native path than
|
||||
* the `node:fs/promises` Node-compat shim. Every other method delegates to
|
||||
* the {@link defaultNodeFs} base because under Bun those calls already
|
||||
* dispatch to Bun's native Zig implementation transparently.
|
||||
*
|
||||
* Returned as a fresh object per call so callers can selectively override
|
||||
* individual methods without mutating shared state — same convention used by
|
||||
* {@link defaultNodeFs}.
|
||||
*
|
||||
* Throws a plain `Error` at call time if invoked outside a Bun runtime. The
|
||||
* intended entry point for adapter selection is {@link pickDefaultFs}, which
|
||||
* probes `globalThis.Bun` first.
|
||||
*/
|
||||
export function defaultBunFs(): NodeFsLike {
|
||||
const bun = getBunRuntime();
|
||||
if (bun === undefined) {
|
||||
throw new Error(
|
||||
"defaultBunFs() requires a Bun runtime (globalThis.Bun is undefined)"
|
||||
);
|
||||
}
|
||||
const base = defaultNodeFs();
|
||||
return {
|
||||
...base,
|
||||
async readFile(path) {
|
||||
return bun.file(path).bytes();
|
||||
},
|
||||
async writeFile(path, data) {
|
||||
// `Bun.write` auto-creates missing parent directories by default,
|
||||
// which would silently bypass the storage layer's `mkdirp: false`
|
||||
// contract. Disable that to match `node:fs/promises.writeFile`.
|
||||
await bun.write(path, data, { createPath: false });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Select the best-fit default {@link NodeFsLike} for the current runtime.
|
||||
* Returns {@link defaultBunFs} when `globalThis.Bun` is present, otherwise
|
||||
* {@link defaultNodeFs}. The probe runs on every call so tests can swap
|
||||
* adapters by injecting an explicit `fs` option rather than relying on a
|
||||
* cached module-scope choice.
|
||||
*/
|
||||
export function pickDefaultFs(): NodeFsLike {
|
||||
return getBunRuntime() !== undefined ? defaultBunFs() : defaultNodeFs();
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
__setBunProbeForTest,
|
||||
__setBunRuntimeForTest,
|
||||
defaultBunFs,
|
||||
pickDefaultFs,
|
||||
} from "./bun-fs.js";
|
||||
@@ -0,0 +1,28 @@
|
||||
/** Maximum permitted key length in characters (bytes after UTF-8 encoding may be larger). */
|
||||
export const MAX_KEY_LENGTH = 1024;
|
||||
|
||||
/** Default data-file extension appended to every key. */
|
||||
export const DEFAULT_FILE_EXTENSION = ".data";
|
||||
|
||||
/** Default metadata-file extension appended to every key. */
|
||||
export const DEFAULT_METADATA_EXTENSION = ".meta";
|
||||
|
||||
/** Default retry maximum attempts on transient filesystem errors. */
|
||||
export const DEFAULT_RETRY_MAX_ATTEMPTS = 3;
|
||||
|
||||
/** Default retry base delay in milliseconds. */
|
||||
export const DEFAULT_RETRY_BASE_DELAY_MS = 50;
|
||||
|
||||
/** Default retry maximum delay in milliseconds. */
|
||||
export const DEFAULT_RETRY_MAX_DELAY_MS = 5_000;
|
||||
|
||||
/** Prefix used for atomic-write temp files. */
|
||||
export const TEMP_FILE_PREFIX = ".tmp-";
|
||||
|
||||
/** Transient errno values that are eligible for retry. */
|
||||
export const TRANSIENT_ERRNO_CODES = new Set([
|
||||
"EBUSY",
|
||||
"EMFILE",
|
||||
"ENFILE",
|
||||
"EAGAIN",
|
||||
]);
|
||||
65
packages/persistence/storage/storage-file/src/error-codes.md
Normal file
65
packages/persistence/storage/storage-file/src/error-codes.md
Normal file
@@ -0,0 +1,65 @@
|
||||
# `error-codes`
|
||||
|
||||
## Purpose
|
||||
|
||||
Defines the frozen `STORAGE_FILE_CODES` registry and the `StorageFileError` class used by every public path in `@suaveplan/storage-file`. The registry is the contract: every code is intentionally thrown from at least one production path, and downstream consumers assert on `error.code` rather than parsing message text. The class extends `@suaveplan/error.SuaveplanError` so it integrates with the OTel span-event adapter and the error-envelope wire format.
|
||||
|
||||
## Features
|
||||
|
||||
- 8 frozen codes covering missing keys, permissions, IO failures, invalid input, atomic-write rollback, option validation, abort, and key length overrun.
|
||||
- `StorageFileErrorCode` union type derived from the registry — exhaustive `switch` statements get compile-time enforcement.
|
||||
- `StorageFileError` carries an optional `key` discriminant so callers can correlate a failure with the offending key without parsing strings.
|
||||
- All codes are namespaced `STORAGE_FILE.*` for unambiguous registry filtering in centralised error reporters.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { StorageFileError, STORAGE_FILE_CODES } from "@suaveplan/storage-file";
|
||||
|
||||
try {
|
||||
await storage.get("../../etc/passwd");
|
||||
} catch (err) {
|
||||
if (err instanceof StorageFileError && err.code === STORAGE_FILE_CODES.INVALID_KEY) {
|
||||
// caller-friendly message
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { StorageFileError, STORAGE_FILE_CODES, type StorageFileErrorCode } from "@suaveplan/storage-file";
|
||||
|
||||
function classify(code: StorageFileErrorCode): "client" | "server" {
|
||||
switch (code) {
|
||||
case STORAGE_FILE_CODES.INVALID_KEY:
|
||||
case STORAGE_FILE_CODES.KEY_TOO_LONG:
|
||||
case STORAGE_FILE_CODES.INVALID_OPTIONS:
|
||||
return "client";
|
||||
case STORAGE_FILE_CODES.NOT_FOUND:
|
||||
case STORAGE_FILE_CODES.PERMISSION_DENIED:
|
||||
case STORAGE_FILE_CODES.IO_FAILED:
|
||||
case STORAGE_FILE_CODES.ATOMIC_WRITE_FAILED:
|
||||
case STORAGE_FILE_CODES.ABORTED:
|
||||
return "server";
|
||||
}
|
||||
}
|
||||
|
||||
throw new StorageFileError(STORAGE_FILE_CODES.NOT_FOUND, "no such key", {
|
||||
key: "/orders/42.json",
|
||||
cause: undefined,
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `STORAGE_FILE_CODES` — frozen object map of code names to fully-qualified strings (`STORAGE_FILE.NOT_FOUND`, etc.). Exhaustive: 8 codes.
|
||||
- `StorageFileErrorCode` — union derived from `(typeof STORAGE_FILE_CODES)[keyof typeof STORAGE_FILE_CODES]`.
|
||||
- `StorageFileError` — class extending `SuaveplanError<StorageFileErrorCode>`. Constructor: `(code, message, options?)`. Properties: inherited `code`, inherited `cause`, additional `key?: string`.
|
||||
- `StorageFileErrorOptions` — extends `SuaveplanErrorOptions` with optional `key`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Codes are registered by `defineCodes("STORAGE_FILE", [...])`. The registration is idempotent: importing this module multiple times does not re-register or throw.
|
||||
- The list is closed by design — adding a new failure mode requires an spec change so consumers can update their handlers in lockstep.
|
||||
- `name` is set to `"StorageFileError"` so `instanceof` and `error.name` checks both work.
|
||||
43
packages/persistence/storage/storage-file/src/error-codes.ts
Normal file
43
packages/persistence/storage/storage-file/src/error-codes.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import {
|
||||
defineCodes,
|
||||
SuaveplanError,
|
||||
type SuaveplanErrorOptions,
|
||||
} from "@suaveplan/error";
|
||||
|
||||
export const STORAGE_FILE_CODES = defineCodes("STORAGE_FILE", [
|
||||
"NOT_FOUND",
|
||||
"PERMISSION_DENIED",
|
||||
"IO_FAILED",
|
||||
"INVALID_KEY",
|
||||
"ATOMIC_WRITE_FAILED",
|
||||
"INVALID_OPTIONS",
|
||||
"ABORTED",
|
||||
"KEY_TOO_LONG",
|
||||
] as const);
|
||||
|
||||
export type StorageFileErrorCode =
|
||||
(typeof STORAGE_FILE_CODES)[keyof typeof STORAGE_FILE_CODES];
|
||||
|
||||
export interface StorageFileErrorOptions extends SuaveplanErrorOptions {
|
||||
key?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structured error for filesystem storage operations.
|
||||
*
|
||||
* Carries an optional `key` discriminant so callers can correlate failures
|
||||
* with the offending key without parsing message text.
|
||||
*/
|
||||
export class StorageFileError extends SuaveplanError<StorageFileErrorCode> {
|
||||
public readonly key: string | undefined;
|
||||
|
||||
public constructor(
|
||||
code: StorageFileErrorCode,
|
||||
message: string,
|
||||
options?: StorageFileErrorOptions
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.key = options?.key;
|
||||
this.name = "StorageFileError";
|
||||
}
|
||||
}
|
||||
66
packages/persistence/storage/storage-file/src/index.ts
Normal file
66
packages/persistence/storage/storage-file/src/index.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
// Side-effect import: ensures `defineCodes` registers the STORAGE_FILE.*
|
||||
// registry even when subpath consumers never reach this file.
|
||||
import "./error-codes.js";
|
||||
|
||||
export { defaultBunFs, pickDefaultFs } from "./bun-fs/bun-fs.js";
|
||||
export {
|
||||
STORAGE_FILE_CODES,
|
||||
StorageFileError,
|
||||
type StorageFileErrorCode,
|
||||
type StorageFileErrorOptions,
|
||||
} from "./error-codes.js";
|
||||
export { sanitizeKey } from "./key-sanitizer/key-sanitizer.js";
|
||||
export { defaultNodeFs } from "./node-fs/node-fs.js";
|
||||
export {
|
||||
buildListPage,
|
||||
buildTempPath,
|
||||
createNodeFsStorage,
|
||||
entryRelativePath,
|
||||
isEnoent,
|
||||
type ListPageArgs,
|
||||
NodeFsStorage,
|
||||
type ResolvedConfig,
|
||||
type ResolvedListOptions,
|
||||
readMetadataWithStatFallback,
|
||||
resolveConfig,
|
||||
resolveListOptions,
|
||||
syntheticMetadata,
|
||||
} from "./node-fs-storage/index.js";
|
||||
export {
|
||||
getInstruments,
|
||||
type RecordOpInput,
|
||||
recordOp,
|
||||
resetInstruments,
|
||||
type StorageFileInstruments,
|
||||
type StorageFileOp,
|
||||
type StorageFileOpOutcome,
|
||||
setStorageFileMeterProvider,
|
||||
} from "./otel/otel.js";
|
||||
export { registerNodeFsProvider } from "./registry/registry.js";
|
||||
export {
|
||||
isPermissionFsError,
|
||||
isTransientFsError,
|
||||
type WithFsRetryOptions,
|
||||
withFsRetry,
|
||||
} from "./retry-helper/retry-helper.js";
|
||||
export {
|
||||
type CreateReadStreamArgs,
|
||||
type CreateWriteStreamArgs,
|
||||
createNodeReadStream,
|
||||
createNodeWriteStream,
|
||||
} from "./streams/streams.js";
|
||||
export type {
|
||||
NodeFsCreateReadStreamOptions,
|
||||
NodeFsCreateWriteStreamOptions,
|
||||
NodeFsFileHandleLike,
|
||||
NodeFsLike,
|
||||
NodeFsReaddirEntry,
|
||||
NodeFsStatLike,
|
||||
NodeFsStorageOptions,
|
||||
NodeFsStorageRetryOptions,
|
||||
TimersLike,
|
||||
} from "./types/index.js";
|
||||
export {
|
||||
NodeFsStorageOptionsSchema,
|
||||
NodeFsStorageRetryOptionsSchema,
|
||||
} from "./types/index.js";
|
||||
@@ -0,0 +1 @@
|
||||
export { sanitizeKey } from "./key-sanitizer.js";
|
||||
@@ -0,0 +1,59 @@
|
||||
# `key-sanitizer`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Validates and normalises every caller-supplied storage key before any filesystem call. The sanitiser is the security boundary that prevents a caller-controlled key (`../../etc/passwd`) from escaping the configured `rootDir`. Every public method on `NodeFsStorage` passes its key through `sanitizeKey` before resolving any path or issuing any `node:fs` call. Failing the sanitiser is unrecoverable from the caller's perspective — it always reflects a programming error or a malicious input.
|
||||
|
||||
## Features
|
||||
|
||||
- Rejects empty strings, `..` traversal segments, POSIX absolute paths (`/foo`), Windows drive-letter prefixed paths (`C:\foo`, `c:/foo`).
|
||||
- Rejects null bytes (`\x00`) and any control char in the C0 range (`0x00`–`0x1F`) plus DEL (`0x7F`).
|
||||
- Enforces a maximum length of `MAX_KEY_LENGTH` (1024) characters; over-length keys surface as `STORAGE_FILE.KEY_TOO_LONG` (a distinct code from `INVALID_KEY` so callers can hint at the truncation).
|
||||
- Normalises Windows `\` separators to POSIX `/` so the same key resolves identically on every platform.
|
||||
- Pure function — no allocation beyond the normalised string return; safe to call on every operation.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { sanitizeKey } from "@suaveplan/storage-file";
|
||||
|
||||
const safe = sanitizeKey("orders/2026/05/12.json");
|
||||
// → "orders/2026/05/12.json"
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { sanitizeKey, StorageFileError, STORAGE_FILE_CODES } from "@suaveplan/storage-file";
|
||||
|
||||
function safeKey(input: string): string | null {
|
||||
try {
|
||||
return sanitizeKey(input);
|
||||
} catch (err) {
|
||||
if (err instanceof StorageFileError && (err.code === STORAGE_FILE_CODES.INVALID_KEY || err.code === STORAGE_FILE_CODES.KEY_TOO_LONG)) {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const normalised = sanitizeKey("orders\\2026\\05\\12.json");
|
||||
// → "orders/2026/05/12.json"
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `sanitizeKey(key: string): string` — returns the normalised POSIX-separator form on success.
|
||||
- Throws `StorageFileError` with code `STORAGE_FILE.INVALID_KEY` on empty input, control chars, absolute paths, drive-letter prefix, or `..` traversal.
|
||||
- Throws `StorageFileError` with code `STORAGE_FILE.KEY_TOO_LONG` when input length exceeds `MAX_KEY_LENGTH` (1024).
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The control-char check iterates the string code unit by code unit rather than using a regex with a control-char range. This keeps biome's `noControlCharactersInRegex` lint happy without a suppression comment and is also marginally faster on short keys.
|
||||
- `..` is rejected segment-by-segment after `\\` normalisation, so both `foo\\..\\bar` and `foo/../bar` are caught.
|
||||
- Empty segments produced by consecutive separators (`foo//bar`) are NOT rejected — they are allowed for caller convenience and resolve to the same path. The host filesystem normalises them internally.
|
||||
- The key argument is not constrained beyond the rules above; arbitrary Unicode (including emoji and CJK) is permitted because `node:fs` accepts them and most modern filesystems do too.
|
||||
Binary file not shown.
@@ -0,0 +1,84 @@
|
||||
import { MAX_KEY_LENGTH } from "../constants/index.js";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
|
||||
// Matches a leading drive-letter prefix on Windows (`C:\`, `D:/`).
|
||||
const WINDOWS_DRIVE_PREFIX = /^[A-Za-z]:[\\/]/;
|
||||
|
||||
// Returns true when any code unit is a C0 control char (0x00-0x1F) or DEL (0x7F).
|
||||
function containsControlChar(value: string): boolean {
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
const code = value.charCodeAt(i);
|
||||
if (code <= 0x1f || code === 0x7f) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and normalise a caller-supplied storage key. Returns the
|
||||
* normalised POSIX-separator form on success; throws
|
||||
* `StorageFileError(STORAGE_FILE.INVALID_KEY)` or
|
||||
* `StorageFileError(STORAGE_FILE.KEY_TOO_LONG)` on rejection.
|
||||
*
|
||||
* Rejection rules:
|
||||
* - Empty string.
|
||||
* - Length over `MAX_KEY_LENGTH` (1024) chars.
|
||||
* - Contains a `..` path segment (parent-directory traversal).
|
||||
* - Starts with `/` (POSIX absolute) or `<letter>:[\\/]` (Windows drive).
|
||||
* - Contains a null byte (`\x00`) or other control char.
|
||||
*
|
||||
* Normalisation: Windows `\` separators are replaced with POSIX `/`.
|
||||
*/
|
||||
export function sanitizeKey(key: string): string {
|
||||
if (typeof key !== "string") {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"key must be a string"
|
||||
);
|
||||
}
|
||||
if (key.length === 0) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"key must not be empty"
|
||||
);
|
||||
}
|
||||
if (key.length > MAX_KEY_LENGTH) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.KEY_TOO_LONG,
|
||||
`key length ${key.length} exceeds maximum ${MAX_KEY_LENGTH}`,
|
||||
{ key: `${key.slice(0, 64)}…` }
|
||||
);
|
||||
}
|
||||
if (containsControlChar(key)) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"key contains a control character",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
if (key.startsWith("/")) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"absolute paths are not allowed",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
if (WINDOWS_DRIVE_PREFIX.test(key)) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"drive-letter prefixed paths are not allowed",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
const normalised = key.replace(/\\/g, "/");
|
||||
const segments = normalised.split("/");
|
||||
for (const segment of segments) {
|
||||
if (segment === "..") {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"path traversal segment (..) is not allowed",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
}
|
||||
return normalised;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
export {
|
||||
buildListPage,
|
||||
buildTempPath,
|
||||
entryRelativePath,
|
||||
isEnoent,
|
||||
type ListPageArgs,
|
||||
type ResolvedConfig,
|
||||
type ResolvedListOptions,
|
||||
readMetadataWithStatFallback,
|
||||
resolveConfig,
|
||||
resolveListOptions,
|
||||
syntheticMetadata,
|
||||
} from "./node-fs-storage.helpers.js";
|
||||
export { createNodeFsStorage, NodeFsStorage } from "./node-fs-storage.js";
|
||||
@@ -0,0 +1,52 @@
|
||||
# `node-fs-storage.helpers`
|
||||
|
||||
## Purpose
|
||||
|
||||
Houses the pure helpers consumed by `NodeFsStorage` so the orchestrator class stays under the file-size budget and the helpers can be unit-tested without instantiating the full adapter. Every function here is either IO-free or wraps exactly one `NodeFsLike` call with documented error mapping.
|
||||
|
||||
## Features
|
||||
|
||||
- `buildTempPath(forPath)` — same-directory temp filename with the canonical `.tmp-` prefix, a base-36 timestamp, and 8 chars of entropy. Co-located placement is required for POSIX `rename` atomicity.
|
||||
- `buildListPage(args)` — pure key-extraction + pagination over a `NodeFsLike.readdir` result. Sorts keys lexicographically before paginating.
|
||||
- `resolveListOptions(options, total)` — normalises `StorageListOptions` undefined-aware so callers don't repeat `?? 0` / `?? prefix`.
|
||||
- `readMetadataWithStatFallback(fs, metaPath, dataPath)` — reads the sidecar metadata file when present; falls back to synthesising `{ createdAt, modifiedAt, size }` from `fs.stat(dataPath)` when the sidecar is missing; returns `null` only when the data file is also missing.
|
||||
- `syntheticMetadata(clock, size)` — minimal `{ createdAt, modifiedAt, size }` triple built from the injectable clock.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { buildTempPath, buildListPage } from "@suaveplan/storage-file";
|
||||
|
||||
const temp = buildTempPath("/tmp/store/key.data");
|
||||
const page = buildListPage({ entries, fileExtension: ".data", prefix: "user/", offset: 0, limit: 50, continuationToken: undefined });
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { readMetadataWithStatFallback, defaultNodeFs } from "@suaveplan/storage-file";
|
||||
|
||||
const fs = defaultNodeFs();
|
||||
const meta = await readMetadataWithStatFallback(fs, "/tmp/store/k.meta", "/tmp/store/k.data");
|
||||
if (meta === null) {
|
||||
// nothing exists for this key
|
||||
} else {
|
||||
// either a real sidecar or a synthesised stat record
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `buildTempPath(forPath: string): string` — temp filename in the same directory as `forPath`.
|
||||
- `buildListPage(args: ListPageArgs): StorageListResult` — pure pagination over readdir entries.
|
||||
- `resolveListOptions(options, total): ResolvedListOptions` — defaults `{prefix:"", offset:0, limit:total||1, continuationToken: undefined}`.
|
||||
- `readMetadataWithStatFallback(fs, metaPath, dataPath): Promise<StorageMetadata | null>` — metadata read with stat fallback.
|
||||
- `syntheticMetadata(clock: Clock, size: number): StorageMetadata` — synthetic timestamped record.
|
||||
- `ListPageArgs`, `ResolvedListOptions` — explicit interfaces.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The temp filename embeds the destination base name so debugging an orphaned temp file in the directory is easy: `ls .tmp-*` lists every aborted write with its target.
|
||||
- The `limit` default in `resolveListOptions` is `max(total, 1)` rather than `total` so an empty store still returns a meaningful page (zero keys, no continuation) without divide-by-zero or `slice(0, 0)` ambiguity.
|
||||
- `readMetadataWithStatFallback` swallows ENOENT on both files (`return null`) but rethrows every other errno so callers don't need to repeat the classification.
|
||||
- All helpers are stateless and re-entrant; they can be called concurrently against the same `NodeFsLike`.
|
||||
@@ -0,0 +1,341 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { NodeFsLike } from "../types/fs-adapter.types.js";
|
||||
import {
|
||||
buildListPage,
|
||||
buildTempPath,
|
||||
isEnoent,
|
||||
readMetadataWithStatFallback,
|
||||
resolveConfig,
|
||||
resolveListOptions,
|
||||
syntheticMetadata,
|
||||
} from "./node-fs-storage.helpers.js";
|
||||
|
||||
describe("buildTempPath", () => {
|
||||
it("prefixes the canonical .tmp- marker and includes the destination base name", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const out = buildTempPath("/tmp/store/orders.data");
|
||||
expect(out.includes(".tmp-")).toBe(true);
|
||||
expect(out.includes("orders.data")).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to a 'tmp' suffix when the destination is unusual", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const out = buildTempPath("/");
|
||||
expect(out.includes(".tmp-")).toBe(true);
|
||||
});
|
||||
|
||||
it("produces a different filename on each call", { timeout: 5000 }, () => {
|
||||
const a = buildTempPath("/tmp/x.data");
|
||||
const b = buildTempPath("/tmp/x.data");
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe("entryRelativePath", () => {
|
||||
it("returns the entry name when parentPath equals rootDir", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { entryRelativePath } = await import(
|
||||
"./node-fs-storage.helpers.js"
|
||||
);
|
||||
const entry = {
|
||||
name: "k.data",
|
||||
parentPath: "/root",
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
};
|
||||
expect(entryRelativePath(entry, "/root")).toBe("k.data");
|
||||
});
|
||||
|
||||
it("prefixes the parent segment when nested inside rootDir", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { entryRelativePath } = await import(
|
||||
"./node-fs-storage.helpers.js"
|
||||
);
|
||||
const entry = {
|
||||
name: "k.data",
|
||||
parentPath: "/root/user",
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
};
|
||||
expect(entryRelativePath(entry, "/root")).toBe("user/k.data");
|
||||
});
|
||||
|
||||
it("falls back to the parent path when it does not lie inside rootDir", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const { entryRelativePath } = await import(
|
||||
"./node-fs-storage.helpers.js"
|
||||
);
|
||||
const entry = {
|
||||
name: "k.data",
|
||||
parentPath: "/somewhere/else",
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
};
|
||||
expect(entryRelativePath(entry, "/root")).toBe(
|
||||
"/somewhere/else/k.data"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildListPage", () => {
|
||||
const fakeEntry = (
|
||||
name: string,
|
||||
isFile: boolean,
|
||||
parentPath = "/root"
|
||||
) => ({
|
||||
name,
|
||||
parentPath,
|
||||
isFile: () => isFile,
|
||||
isDirectory: () => !isFile,
|
||||
});
|
||||
|
||||
it("filters by extension, prefix, and paginates", { timeout: 5000 }, () => {
|
||||
const entries = [
|
||||
fakeEntry("a.data", true),
|
||||
fakeEntry("b.data", true),
|
||||
fakeEntry("c.data", true),
|
||||
fakeEntry("d.meta", true),
|
||||
fakeEntry("nested", false),
|
||||
];
|
||||
const page = buildListPage({
|
||||
entries,
|
||||
rootDir: "/root",
|
||||
fileExtension: ".data",
|
||||
prefix: "",
|
||||
offset: 0,
|
||||
limit: 2,
|
||||
continuationToken: undefined,
|
||||
});
|
||||
expect(page.keys).toEqual(["a", "b"]);
|
||||
expect(page.hasMore).toBe(true);
|
||||
expect(page.continuationToken).toBe("2");
|
||||
});
|
||||
|
||||
it("honours an explicit continuationToken over the offset", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const entries = [
|
||||
fakeEntry("a.data", true),
|
||||
fakeEntry("b.data", true),
|
||||
fakeEntry("c.data", true),
|
||||
];
|
||||
const page = buildListPage({
|
||||
entries,
|
||||
rootDir: "/root",
|
||||
fileExtension: ".data",
|
||||
prefix: "",
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
continuationToken: "2",
|
||||
});
|
||||
expect(page.keys).toEqual(["c"]);
|
||||
expect(page.hasMore).toBe(false);
|
||||
});
|
||||
|
||||
it("treats an unparseable continuationToken as offset 0", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const page = buildListPage({
|
||||
entries: [fakeEntry("a.data", true)],
|
||||
rootDir: "/root",
|
||||
fileExtension: ".data",
|
||||
prefix: "",
|
||||
offset: 5,
|
||||
limit: 10,
|
||||
continuationToken: "not-a-number",
|
||||
});
|
||||
expect(page.keys).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("honours a prefix filter", { timeout: 5000 }, () => {
|
||||
const entries = [
|
||||
fakeEntry("1.data", true, "/root/user"),
|
||||
fakeEntry("2.data", true, "/root/user"),
|
||||
fakeEntry("1.data", true, "/root/post"),
|
||||
];
|
||||
const page = buildListPage({
|
||||
entries,
|
||||
rootDir: "/root",
|
||||
fileExtension: ".data",
|
||||
prefix: "user/",
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
continuationToken: undefined,
|
||||
});
|
||||
expect(page.keys).toEqual(["user/1", "user/2"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveListOptions", () => {
|
||||
it("applies documented defaults", { timeout: 5000 }, () => {
|
||||
expect(resolveListOptions(undefined, 0)).toEqual({
|
||||
prefix: "",
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
continuationToken: undefined,
|
||||
});
|
||||
expect(resolveListOptions({ prefix: "x", limit: 5 }, 0)).toEqual({
|
||||
prefix: "x",
|
||||
offset: 0,
|
||||
limit: 5,
|
||||
continuationToken: undefined,
|
||||
});
|
||||
expect(resolveListOptions({ continuationToken: "3" }, 10)).toEqual({
|
||||
prefix: "",
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
continuationToken: "3",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("syntheticMetadata", () => {
|
||||
it("builds a fresh metadata record using the injected clock", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const meta = syntheticMetadata(() => 1_700_000_000_000, 42);
|
||||
expect(meta).toEqual({
|
||||
createdAt: 1_700_000_000_000,
|
||||
modifiedAt: 1_700_000_000_000,
|
||||
size: 42,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("isEnoent", () => {
|
||||
it("recognises ENOENT errno-bearing values and rejects everything else", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
isEnoent(Object.assign(new Error("e"), { code: "ENOENT" }))
|
||||
).toBe(true);
|
||||
expect(
|
||||
isEnoent(Object.assign(new Error("e"), { code: "EACCES" }))
|
||||
).toBe(false);
|
||||
expect(isEnoent(null)).toBe(false);
|
||||
expect(isEnoent("ENOENT")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("readMetadataWithStatFallback", () => {
|
||||
function makeFs(overrides: Partial<NodeFsLike>): NodeFsLike {
|
||||
const base: NodeFsLike = {
|
||||
async mkdir() {
|
||||
return undefined;
|
||||
},
|
||||
async readFile() {
|
||||
throw Object.assign(new Error("noent"), { code: "ENOENT" });
|
||||
},
|
||||
async writeFile() {},
|
||||
async rename() {},
|
||||
async unlink() {},
|
||||
async readdir() {
|
||||
return [];
|
||||
},
|
||||
async stat() {
|
||||
throw Object.assign(new Error("noent"), { code: "ENOENT" });
|
||||
},
|
||||
async access() {},
|
||||
async open() {
|
||||
throw new Error("unused");
|
||||
},
|
||||
createReadStream() {
|
||||
throw new Error("unused");
|
||||
},
|
||||
createWriteStream() {
|
||||
throw new Error("unused");
|
||||
},
|
||||
};
|
||||
return { ...base, ...overrides };
|
||||
}
|
||||
|
||||
it("returns the parsed sidecar when present", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fs = makeFs({
|
||||
async readFile() {
|
||||
return new TextEncoder().encode(
|
||||
JSON.stringify({ createdAt: 1, modifiedAt: 2, size: 3 })
|
||||
);
|
||||
},
|
||||
});
|
||||
const result = await readMetadataWithStatFallback(fs, "meta", "data");
|
||||
expect(result).toEqual({ createdAt: 1, modifiedAt: 2, size: 3 });
|
||||
});
|
||||
|
||||
it("falls back to stat when the sidecar is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fs = makeFs({
|
||||
async stat() {
|
||||
return {
|
||||
size: 7,
|
||||
mtimeMs: 222,
|
||||
birthtimeMs: 111,
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
};
|
||||
},
|
||||
});
|
||||
const result = await readMetadataWithStatFallback(fs, "meta", "data");
|
||||
expect(result).toEqual({ createdAt: 111, modifiedAt: 222, size: 7 });
|
||||
});
|
||||
|
||||
it("returns null when both files are missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fs = makeFs({});
|
||||
expect(
|
||||
await readMetadataWithStatFallback(fs, "meta", "data")
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("propagates non-ENOENT errors from readFile", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fs = makeFs({
|
||||
async readFile() {
|
||||
throw Object.assign(new Error("io"), { code: "EIO" });
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
readMetadataWithStatFallback(fs, "meta", "data")
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("propagates non-ENOENT errors from stat", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const fs = makeFs({
|
||||
async stat() {
|
||||
throw Object.assign(new Error("io"), { code: "EIO" });
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
readMetadataWithStatFallback(fs, "meta", "data")
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveConfig", () => {
|
||||
it("applies documented defaults", { timeout: 5000 }, () => {
|
||||
const cfg = resolveConfig({ rootDir: "/tmp" });
|
||||
expect(cfg.rootDir).toBe("/tmp");
|
||||
expect(cfg.mkdirp).toBe(true);
|
||||
expect(cfg.fileExtension).toBe(".data");
|
||||
expect(cfg.metadataExtension).toBe(".meta");
|
||||
expect(typeof cfg.clock).toBe("function");
|
||||
expect(typeof cfg.clock()).toBe("number");
|
||||
});
|
||||
|
||||
it("rejects malformed options as INVALID_OPTIONS", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() => resolveConfig({} as never)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,214 @@
|
||||
import { dirname, join, sep } from "node:path";
|
||||
import type {
|
||||
StorageListOptions,
|
||||
StorageListResult,
|
||||
StorageMetadata,
|
||||
} from "@suaveplan/storage/types";
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
import { pickDefaultFs } from "../bun-fs/bun-fs.js";
|
||||
import {
|
||||
DEFAULT_FILE_EXTENSION,
|
||||
DEFAULT_METADATA_EXTENSION,
|
||||
TEMP_FILE_PREFIX,
|
||||
} from "../constants/index.js";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import type {
|
||||
NodeFsLike,
|
||||
NodeFsReaddirEntry,
|
||||
} from "../types/fs-adapter.types.js";
|
||||
import {
|
||||
type NodeFsStorageOptions,
|
||||
NodeFsStorageOptionsSchema,
|
||||
type NodeFsStorageRetryOptions,
|
||||
type TimersLike,
|
||||
} from "../types/node-fs-storage.types.js";
|
||||
|
||||
export interface ResolvedConfig {
|
||||
rootDir: string;
|
||||
mkdirp: boolean;
|
||||
fileExtension: string;
|
||||
metadataExtension: string;
|
||||
retry: NodeFsStorageRetryOptions | undefined;
|
||||
fs: NodeFsLike;
|
||||
timers: TimersLike | undefined;
|
||||
clock: Clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and normalise constructor options. Throws
|
||||
* `STORAGE_FILE.INVALID_OPTIONS` on parse failure. Pure aside from the
|
||||
* one-time `pickDefaultFs()` factory call (Bun-native under Bun, Node
|
||||
* elsewhere).
|
||||
*/
|
||||
export function resolveConfig(options: NodeFsStorageOptions): ResolvedConfig {
|
||||
let parsed: NodeFsStorageOptions;
|
||||
try {
|
||||
parsed = NodeFsStorageOptionsSchema.parse(
|
||||
options
|
||||
) as NodeFsStorageOptions;
|
||||
} catch (err) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_OPTIONS,
|
||||
`Invalid NodeFsStorage options: ${(err as Error).message}`,
|
||||
{ cause: err as Error }
|
||||
);
|
||||
}
|
||||
return {
|
||||
rootDir: parsed.rootDir,
|
||||
mkdirp: parsed.mkdirp ?? true,
|
||||
fileExtension: parsed.fileExtension ?? DEFAULT_FILE_EXTENSION,
|
||||
metadataExtension:
|
||||
parsed.metadataExtension ?? DEFAULT_METADATA_EXTENSION,
|
||||
retry: parsed.retry,
|
||||
fs: parsed.fs ?? pickDefaultFs(),
|
||||
timers: parsed.timers,
|
||||
clock: parsed.clock ?? (() => Date.now()),
|
||||
};
|
||||
}
|
||||
|
||||
export function isEnoent(err: unknown): boolean {
|
||||
return (err as NodeJS.ErrnoException | null)?.code === "ENOENT";
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique temp filename in the same directory as `forPath`.
|
||||
* Same-directory placement guarantees `fs.rename` is atomic (POSIX rename
|
||||
* is only atomic within a filesystem and is fastest within a directory).
|
||||
*/
|
||||
export function buildTempPath(forPath: string): string {
|
||||
const random = Math.random().toString(36).slice(2, 10);
|
||||
const stamp = Date.now().toString(36);
|
||||
const segments = forPath.split(sep).join("/").split("/");
|
||||
const last = segments[segments.length - 1];
|
||||
const base = last !== undefined && last.length > 0 ? last : "tmp";
|
||||
return join(
|
||||
dirname(forPath),
|
||||
`${TEMP_FILE_PREFIX}${stamp}-${random}-${base}`
|
||||
);
|
||||
}
|
||||
|
||||
export interface ListPageArgs {
|
||||
entries: NodeFsReaddirEntry[];
|
||||
rootDir: string;
|
||||
fileExtension: string;
|
||||
prefix: string;
|
||||
offset: number;
|
||||
limit: number;
|
||||
continuationToken: string | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a `NodeFsReaddirEntry`, return its path relative to `rootDir` using
|
||||
* POSIX separators. Returns just the entry name when `parentPath === rootDir`.
|
||||
*/
|
||||
export function entryRelativePath(
|
||||
entry: NodeFsReaddirEntry,
|
||||
rootDir: string
|
||||
): string {
|
||||
const normalisedParent = entry.parentPath.split(sep).join("/");
|
||||
const normalisedRoot = rootDir.split(sep).join("/");
|
||||
if (normalisedParent === normalisedRoot) return entry.name;
|
||||
const rel = normalisedParent.startsWith(`${normalisedRoot}/`)
|
||||
? normalisedParent.slice(normalisedRoot.length + 1)
|
||||
: normalisedParent;
|
||||
return `${rel}/${entry.name}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reduce a flat entries array into the documented `StorageListResult`
|
||||
* pagination shape. Pure — no IO. Centralised so the list and clear paths
|
||||
* both consume the same key-extraction logic.
|
||||
*/
|
||||
export function buildListPage(args: ListPageArgs): StorageListResult {
|
||||
const {
|
||||
entries,
|
||||
rootDir,
|
||||
fileExtension,
|
||||
prefix,
|
||||
offset,
|
||||
limit,
|
||||
continuationToken,
|
||||
} = args;
|
||||
let effectiveOffset = offset;
|
||||
if (continuationToken !== undefined) {
|
||||
const parsed = Number.parseInt(continuationToken, 10);
|
||||
effectiveOffset = Number.isNaN(parsed) ? 0 : parsed;
|
||||
}
|
||||
const keys: string[] = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
const rel = entryRelativePath(entry, rootDir);
|
||||
if (!rel.endsWith(fileExtension)) continue;
|
||||
const key = rel.slice(0, rel.length - fileExtension.length);
|
||||
if (prefix && !key.startsWith(prefix)) continue;
|
||||
keys.push(key);
|
||||
}
|
||||
keys.sort();
|
||||
const total = keys.length;
|
||||
const page = keys.slice(effectiveOffset, effectiveOffset + limit);
|
||||
const hasMore = effectiveOffset + page.length < total;
|
||||
return {
|
||||
keys: page,
|
||||
total,
|
||||
hasMore,
|
||||
...(hasMore
|
||||
? { continuationToken: String(effectiveOffset + page.length) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface ResolvedListOptions {
|
||||
prefix: string;
|
||||
offset: number;
|
||||
limit: number;
|
||||
continuationToken: string | undefined;
|
||||
}
|
||||
|
||||
export function resolveListOptions(
|
||||
options: StorageListOptions | undefined,
|
||||
total: number
|
||||
): ResolvedListOptions {
|
||||
return {
|
||||
prefix: options?.prefix ?? "",
|
||||
offset: options?.offset ?? 0,
|
||||
limit: options?.limit ?? Math.max(total, 1),
|
||||
continuationToken: options?.continuationToken,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the sidecar metadata file. Returns the parsed `StorageMetadata` on
|
||||
* success, falls back to a synthetic record built from the data file's
|
||||
* stat() on ENOENT, and returns null when both the metadata and data
|
||||
* files are missing.
|
||||
*/
|
||||
export async function readMetadataWithStatFallback(
|
||||
fs: NodeFsLike,
|
||||
metaPath: string,
|
||||
dataPath: string
|
||||
): Promise<StorageMetadata | null> {
|
||||
try {
|
||||
const raw = await fs.readFile(metaPath);
|
||||
return JSON.parse(new TextDecoder().decode(raw)) as StorageMetadata;
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code !== "ENOENT") throw err;
|
||||
}
|
||||
try {
|
||||
const stat = await fs.stat(dataPath);
|
||||
return {
|
||||
createdAt: stat.birthtimeMs,
|
||||
modifiedAt: stat.mtimeMs,
|
||||
size: stat.size,
|
||||
};
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "ENOENT") return null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export function syntheticMetadata(clock: Clock, size: number): StorageMetadata {
|
||||
const now = clock();
|
||||
return { createdAt: now, modifiedAt: now, size };
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
# `node-fs-storage`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs --> - node-fs-storage.ops — Houses the file-system operat…
|
||||
- [node-fs-storage.ops](./node-fs-storage.ops.md) — Houses the file-system operation helpers consumed by `NodeFsStorage`. Each function here wraps a single boundary call w…
|
||||
- [node-fs-storage.ops](./node-fs-storage.ops.md) — Houses the file-system operation helpers consumed by `NodeFsStorage`. Each function here wraps a single boundary call w…
|
||||
|
||||
## Purpose
|
||||
|
||||
`NodeFsStorage` is the package's headline export: a full `AbstractStorage` implementation backed by the Node.js filesystem with atomic writes, AbortSignal everywhere, strict key sanitisation, retry on transient errors, native Web Streams, and partial reads. It is the server-tier sibling of `MemoryStorage` from `@suaveplan/storage` and `S3` from `@suaveplan/storage-s3` — every consumer that talks to `AbstractStorage` works with it unchanged.
|
||||
|
||||
## Features
|
||||
|
||||
- Extends `AbstractStorage` and implements every abstract method (`get`, `set`, `delete`, `exists`, `list`, `clear`, `getWithMetadata`, `getMetadata`, `getSize`, `getPartial`, `setPartial`, `getItemSize`, `createReadStream`, `createWriteStream`).
|
||||
- Atomic writes via temp-file-then-`fs.rename`. The temp file lives in the same directory as the destination so the rename is POSIX-atomic; rollback unlinks the temp on any failure.
|
||||
- AbortSignal threaded through every async method; pre-aborted signals surface as `STORAGE_FILE.ABORTED` before any IO call.
|
||||
- Strict key sanitisation: `sanitizeKey` rejects empty keys, `..` traversal, absolute paths, drive-letter prefix, control chars, null bytes, and over-length keys before any path resolution.
|
||||
- `@suaveplan/retry` integration on `EBUSY`/`EMFILE`/`ENFILE`/`EAGAIN` with injectable `TimersLike` for deterministic test timing.
|
||||
- Native Web Streams via `node:fs.createReadStream`/`createWriteStream`. Streamed writes use the same temp-rename atomic policy.
|
||||
- Partial reads via positioned `fs.read` on a `FileHandle` — O(length) memory, never O(file-size).
|
||||
- Side-car metadata file (`.meta`) for `StorageOptions.metadata`/`ttl`. Missing metadata falls back to a synthetic record from `fs.stat`.
|
||||
- Full OTel surface — duration histogram, byte-flow histogram, per-kind counters, per-op tracer.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { NodeFsStorage } from "@suaveplan/storage-file";
|
||||
|
||||
const storage = new NodeFsStorage({ rootDir: "/tmp/store" });
|
||||
await storage.set("hello", new TextEncoder().encode("world"));
|
||||
const bytes = await storage.get("hello");
|
||||
console.log(new TextDecoder().decode(bytes!));
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { createNodeFsStorage, defaultNodeFs } from "@suaveplan/storage-file";
|
||||
import { fakeClock } from "@suaveplan/testing/runner";
|
||||
|
||||
const fc = fakeClock();
|
||||
const storage = createNodeFsStorage({
|
||||
rootDir: "/tmp/store",
|
||||
mkdirp: true,
|
||||
fileExtension: ".bin",
|
||||
metadataExtension: ".manifest",
|
||||
retry: { maxAttempts: 5, baseDelayMs: 25, maxDelayMs: 1_000 },
|
||||
timers: { setTimer: fc.setTimeout, clearTimer: fc.clearTimeout },
|
||||
clock: () => fc.now(),
|
||||
fs: defaultNodeFs(),
|
||||
});
|
||||
|
||||
const controller = new AbortController();
|
||||
const writer = storage.createWriteStream("large", controller.signal).getWriter();
|
||||
await writer.write(new Uint8Array(1024 * 1024));
|
||||
await writer.close();
|
||||
|
||||
const slice = await storage.getPartial("large", 1024, 256, controller.signal);
|
||||
console.log(slice?.byteLength); // → 256
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `class NodeFsStorage extends AbstractStorage` — constructor `(options: NodeFsStorageOptions)`. Throws `STORAGE_FILE.INVALID_OPTIONS` on schema-validation failure.
|
||||
- `createNodeFsStorage(options): NodeFsStorage` — convenience factory.
|
||||
- `capabilities` — `{ partialReads: true, partialWrites: false, nativeStreams: true, atomicWrites: true, contentAddressable: false }`.
|
||||
- All abstract `AbstractStorage` methods with the exact signatures declared on the base class.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Public methods sanitise their key argument before any internal operation. The sanitised form is what reaches `dataPath`/`metaPath`/`atomicWrite` so directory-traversal cannot be re-introduced by a private helper.
|
||||
- `setPartial` delegates to `AbstractStorage.defaultSetPartial`, which performs a read-modify-write. Native random-access write is a follow-up — the reference path is correct but loads the full value into memory.
|
||||
- `getSize` enumerates `list()` keys and reads each `getMetadata` — `O(N)` calls. This matches the upstream contract; future work may add a single-pass `du`-like helper.
|
||||
- The metadata sidecar file is written atomically using the same temp-rename helper as the data file. A failed metadata write does NOT rollback the data write: the data is durable, the metadata is best-effort. Callers requiring the two to commit together can perform the second write themselves under `set(..., { metadata })`.
|
||||
- `createReadStream` and `createWriteStream` return WHATWG Web Streams; consumers that prefer Node's Readable can pipe via `Readable.fromWeb`.
|
||||
- The OTel `op_duration_ms` histogram timing is wall-clock via `Date.now()` — accurate at the millisecond resolution that OTel histograms typically bucket at. For sub-millisecond benchmarks, use the dedicated mitata suite under `src/__benchmarks__/`.
|
||||
@@ -0,0 +1,44 @@
|
||||
# `node-fs-storage.ops`
|
||||
|
||||
## Purpose
|
||||
|
||||
Houses the file-system operation helpers consumed by `NodeFsStorage`. Each function here wraps a single boundary call with documented error mapping; the orchestrator class delegates so the class itself stays under the source-file size budget. Splitting these out also makes the operations independently testable in isolation.
|
||||
|
||||
## Features
|
||||
|
||||
- `readPositioned` — `fs.open` + `handle.read(offset, length)` + `handle.close()` with `null` on ENOENT.
|
||||
- `assertPositiveInt` — argument-shape guard for the `offset`/`length` pair on `getPartial`.
|
||||
- `buildReadStream` / `buildWriteStream` — thin wrappers that thread the optional `signal` into the underlying stream factories.
|
||||
- `assertKeyExists` — pluggable strict variant of `exists` that throws `STORAGE_FILE.NOT_FOUND` when the key is absent.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { readPositioned, defaultNodeFs } from "@suaveplan/storage-file";
|
||||
|
||||
const fs = defaultNodeFs();
|
||||
const slice = await readPositioned({ fs, path: "/tmp/k.data", offset: 0, length: 16, key: "k" });
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { assertKeyExists, NodeFsStorage } from "@suaveplan/storage-file";
|
||||
|
||||
const storage = new NodeFsStorage({ rootDir: "/tmp/store" });
|
||||
await assertKeyExists("config", (k, signal) => storage.exists(k, signal));
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `readPositioned(args: PartialReadArgs): Promise<Uint8Array | null>` — positioned read.
|
||||
- `assertPositiveInt(key, offset, length): void` — argument-shape guard.
|
||||
- `buildReadStream(args: ReadStreamArgs): ReadableStream<Uint8Array>` — read stream.
|
||||
- `buildWriteStream(args: WriteStreamArgs): WritableStream<Uint8Array>` — write stream.
|
||||
- `assertKeyExists(key, exists, signal?): Promise<void>` — strict key check.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `readPositioned` always closes the handle via a `finally` block; the `.catch(() => undefined)` on `close()` swallows close-only failures so the caller never loses the read result.
|
||||
- `buildWriteStream` fires `fs.mkdir` as a fire-and-forget — the stream constructor needs to return synchronously, so the `mkdir` runs in the background and any failure is swallowed (the subsequent stream open will surface the real error).
|
||||
- `assertKeyExists` accepts the `exists` function as a parameter rather than the storage instance so the helper is decoupled from the class hierarchy and can be tested in isolation.
|
||||
@@ -0,0 +1,156 @@
|
||||
import { dirname } from "node:path";
|
||||
import type { StorageMetadata, StorageOptions } from "@suaveplan/storage/types";
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import {
|
||||
createNodeReadStream,
|
||||
createNodeWriteStream,
|
||||
} from "../streams/streams.js";
|
||||
import type {
|
||||
NodeFsFileHandleLike,
|
||||
NodeFsLike,
|
||||
} from "../types/fs-adapter.types.js";
|
||||
import { buildTempPath, isEnoent } from "./node-fs-storage.helpers.js";
|
||||
|
||||
export interface PartialReadArgs {
|
||||
fs: NodeFsLike;
|
||||
path: string;
|
||||
offset: number;
|
||||
length: number;
|
||||
key: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read `length` bytes from a positioned offset on the data file. Returns
|
||||
* `null` when the file does not exist. Pure helper — the calling adapter
|
||||
* is responsible for `#wrap` / `recordOp` / signal handling.
|
||||
*/
|
||||
export async function readPositioned(
|
||||
args: PartialReadArgs
|
||||
): Promise<Uint8Array | null> {
|
||||
const { fs, path, offset, length } = args;
|
||||
let handle: NodeFsFileHandleLike;
|
||||
try {
|
||||
handle = await fs.open(path, "r");
|
||||
} catch (err) {
|
||||
if (isEnoent(err)) return null;
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const buffer = new Uint8Array(length);
|
||||
const { bytesRead } = await handle.read(buffer, 0, length, offset);
|
||||
return bytesRead === length ? buffer : buffer.subarray(0, bytesRead);
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the offset/length argument pair for a positioned read or write.
|
||||
* Throws `STORAGE_FILE.INVALID_KEY` (used here for argument-shape errors)
|
||||
* when either is negative or non-integer.
|
||||
*/
|
||||
export function assertPositiveInt(
|
||||
key: string,
|
||||
offset: number,
|
||||
length: number
|
||||
): void {
|
||||
if (
|
||||
offset < 0 ||
|
||||
!Number.isInteger(offset) ||
|
||||
length < 0 ||
|
||||
!Number.isInteger(length)
|
||||
) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
`getPartial: offset and length must be non-negative integers (got ${offset}/${length})`,
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReadStreamArgs {
|
||||
fs: NodeFsLike;
|
||||
path: string;
|
||||
key: string;
|
||||
signal: AbortSignal | undefined;
|
||||
}
|
||||
|
||||
export function buildReadStream(
|
||||
args: ReadStreamArgs
|
||||
): ReadableStream<Uint8Array> {
|
||||
const { fs, path, key, signal } = args;
|
||||
const base = { fs, path, key };
|
||||
return createNodeReadStream(
|
||||
signal !== undefined ? { ...base, signal } : base
|
||||
);
|
||||
}
|
||||
|
||||
export interface WriteStreamArgs {
|
||||
fs: NodeFsLike;
|
||||
destination: string;
|
||||
mkdirp: boolean;
|
||||
key: string;
|
||||
signal: AbortSignal | undefined;
|
||||
}
|
||||
|
||||
export function buildWriteStream(
|
||||
args: WriteStreamArgs
|
||||
): WritableStream<Uint8Array> {
|
||||
const { fs, destination, mkdirp, key, signal } = args;
|
||||
if (mkdirp) {
|
||||
void fs
|
||||
.mkdir(dirname(destination), { recursive: true })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
const base = {
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath: buildTempPath(destination),
|
||||
key,
|
||||
};
|
||||
return createNodeWriteStream(
|
||||
signal !== undefined ? { ...base, signal } : base
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `StorageMetadata` record + its encoded JSON form for the
|
||||
* metadata sidecar write. Returns `undefined` when neither `metadata`
|
||||
* nor `ttl` was supplied (caller should skip the write entirely).
|
||||
*/
|
||||
export function buildMetadataPayload(
|
||||
clock: Clock,
|
||||
size: number,
|
||||
options: StorageOptions | undefined
|
||||
): { metadata: StorageMetadata; encoded: Uint8Array } | undefined {
|
||||
if (options?.metadata === undefined && options?.ttl === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const now = clock();
|
||||
const metadata: StorageMetadata = { createdAt: now, modifiedAt: now, size };
|
||||
if (options.ttl !== undefined) metadata.expiresAt = now + options.ttl;
|
||||
if (options.metadata !== undefined) metadata.metadata = options.metadata;
|
||||
return {
|
||||
metadata,
|
||||
encoded: new TextEncoder().encode(JSON.stringify(metadata)),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws `STORAGE_FILE.NOT_FOUND` when `exists(safe)` is false. The strict
|
||||
* companion to {@link AbstractStorage.exists}.
|
||||
*/
|
||||
export async function assertKeyExists(
|
||||
safe: string,
|
||||
exists: (key: string, signal?: AbortSignal) => Promise<boolean>,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
if (!(await exists(safe, signal))) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.NOT_FOUND,
|
||||
`Key not found: ${safe}`,
|
||||
{ key: safe }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type {
|
||||
Counter,
|
||||
Histogram,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import { AbstractStorage } from "@suaveplan/storage/abstract";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
expectTypeOf,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import { resetInstruments, setStorageFileMeterProvider } from "../otel/otel.js";
|
||||
import { NodeFsStorage } from "./node-fs-storage.js";
|
||||
|
||||
interface RecordedCounter {
|
||||
name: string;
|
||||
additions: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
interface RecordedHistogram {
|
||||
name: string;
|
||||
samples: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
|
||||
class StubCounter implements Counter {
|
||||
public readonly entry: RecordedCounter;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedCounter[]
|
||||
) {
|
||||
this.entry = { name, additions: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public add(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.additions.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubHistogram implements Histogram {
|
||||
public readonly entry: RecordedHistogram;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedHistogram[]
|
||||
) {
|
||||
this.entry = { name, samples: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public record(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.samples.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeter implements Meter {
|
||||
public counters: RecordedCounter[] = [];
|
||||
public histograms: RecordedHistogram[] = [];
|
||||
public createCounter(name: string): Counter {
|
||||
return new StubCounter(name, this.counters);
|
||||
}
|
||||
public createHistogram(name: string): Histogram {
|
||||
return new StubHistogram(name, this.histograms);
|
||||
}
|
||||
public createObservableGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public addBatchObservableCallback(): void {}
|
||||
public removeBatchObservableCallback(): void {}
|
||||
public createGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeterProvider implements MeterProvider {
|
||||
public meters: StubMeter[] = [];
|
||||
public getMeter(): Meter {
|
||||
const meter = new StubMeter();
|
||||
this.meters.push(meter);
|
||||
return meter;
|
||||
}
|
||||
}
|
||||
|
||||
const immediateTimers: TimersLike = {
|
||||
setTimer(cb) {
|
||||
queueMicrotask(cb);
|
||||
return 0;
|
||||
},
|
||||
clearTimer() {},
|
||||
};
|
||||
|
||||
describe("NodeFsStorage spec conformance", () => {
|
||||
let root: string;
|
||||
let provider: StubMeterProvider;
|
||||
let storage: NodeFsStorage;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "nfs-spec-"));
|
||||
provider = new StubMeterProvider();
|
||||
setStorageFileMeterProvider(provider);
|
||||
storage = new NodeFsStorage({ rootDir: root, timers: immediateTimers });
|
||||
});
|
||||
afterEach(async () => {
|
||||
setStorageFileMeterProvider(undefined);
|
||||
resetInstruments();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("is structurally an AbstractStorage", { timeout: 5000 }, () => {
|
||||
expect(storage).toBeInstanceOf(AbstractStorage);
|
||||
expectTypeOf<NodeFsStorage>().toExtend<AbstractStorage>();
|
||||
});
|
||||
|
||||
it("declares the documented capabilities matrix", { timeout: 5000 }, () => {
|
||||
expect(storage.capabilities).toEqual({
|
||||
partialReads: true,
|
||||
partialWrites: false,
|
||||
nativeStreams: true,
|
||||
atomicWrites: true,
|
||||
contentAddressable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses options via NodeFsStorageOptionsSchema at construction", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() => new NodeFsStorage({ rootDir: "" })).toThrow(
|
||||
StorageFileError
|
||||
);
|
||||
try {
|
||||
new NodeFsStorage({ rootDir: root, fileExtension: "" });
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.INVALID_OPTIONS
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("emits storage_file.writes_total counter on successful set", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3]));
|
||||
const writes = provider.meters[0]?.counters.find(
|
||||
(c) => c.name === "storage_file.writes_total"
|
||||
);
|
||||
expect(writes?.additions.length).toBeGreaterThan(0);
|
||||
expect(writes?.additions[0]?.attrs).toMatchObject({
|
||||
op: "set",
|
||||
outcome: "success",
|
||||
});
|
||||
const duration = provider.meters[0]?.histograms.find(
|
||||
(h) => h.name === "storage_file.op_duration_ms"
|
||||
);
|
||||
expect(duration?.samples.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("emits errors_total counter on a failing read", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.get("missing");
|
||||
// get-of-missing succeeds with undefined return — no error
|
||||
// Force an error by passing a pre-aborted signal
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
try {
|
||||
await storage.get("k", controller.signal);
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
const errs = provider.meters[0]?.counters.find(
|
||||
(c) => c.name === "storage_file.errors_total"
|
||||
);
|
||||
expect(errs?.additions.length).toBeGreaterThan(0);
|
||||
expect(errs?.additions[0]?.attrs).toMatchObject({ outcome: "aborted" });
|
||||
});
|
||||
|
||||
it("option schema accepts the full set of documented fields", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
mkdirp: true,
|
||||
fileExtension: ".bin",
|
||||
metadataExtension: ".manifest",
|
||||
retry: { maxAttempts: 4, baseDelayMs: 10, maxDelayMs: 100 },
|
||||
timers: immediateTimers,
|
||||
clock: () => 1_700_000_000_000,
|
||||
});
|
||||
expect(s).toBeInstanceOf(NodeFsStorage);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,963 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtemp, readdir, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { AbstractStorage } from "@suaveplan/storage/abstract";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import { TEMP_FILE_PREFIX } from "../constants/index.js";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import { defaultNodeFs } from "../node-fs/node-fs.js";
|
||||
import type { NodeFsLike } from "../types/fs-adapter.types.js";
|
||||
import { createNodeFsStorage, NodeFsStorage } from "./node-fs-storage.js";
|
||||
|
||||
const immediateTimers: TimersLike = {
|
||||
setTimer(cb) {
|
||||
queueMicrotask(cb);
|
||||
return 0;
|
||||
},
|
||||
clearTimer() {
|
||||
/* no-op */
|
||||
},
|
||||
};
|
||||
|
||||
function errnoError(code: string): Error & { code: string } {
|
||||
const err = new Error(code) as Error & { code: string };
|
||||
err.code = code;
|
||||
return err;
|
||||
}
|
||||
|
||||
describe("NodeFsStorage", () => {
|
||||
let root: string;
|
||||
let storage: NodeFsStorage;
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "nfs-storage-"));
|
||||
storage = new NodeFsStorage({ rootDir: root, timers: immediateTimers });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("implements every AbstractStorage abstract method", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(storage).toBeInstanceOf(AbstractStorage);
|
||||
expect(typeof storage.get).toBe("function");
|
||||
expect(typeof storage.set).toBe("function");
|
||||
expect(typeof storage.delete).toBe("function");
|
||||
expect(typeof storage.exists).toBe("function");
|
||||
expect(typeof storage.list).toBe("function");
|
||||
expect(typeof storage.clear).toBe("function");
|
||||
expect(typeof storage.getWithMetadata).toBe("function");
|
||||
expect(typeof storage.getMetadata).toBe("function");
|
||||
expect(typeof storage.getSize).toBe("function");
|
||||
expect(typeof storage.getPartial).toBe("function");
|
||||
expect(typeof storage.setPartial).toBe("function");
|
||||
expect(typeof storage.getItemSize).toBe("function");
|
||||
expect(typeof storage.createReadStream).toBe("function");
|
||||
expect(typeof storage.createWriteStream).toBe("function");
|
||||
});
|
||||
|
||||
it("round-trips Uint8Array values", { timeout: 10000 }, async () => {
|
||||
const bytes = new Uint8Array([1, 2, 3, 4, 5]);
|
||||
await storage.set("k", bytes);
|
||||
const back = await storage.get("k");
|
||||
expect(back).toEqual(bytes);
|
||||
expect(await storage.delete("k")).toBe(true);
|
||||
expect(await storage.get("k")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined from get when the key is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
expect(await storage.get("missing")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("exists returns true after set and false after delete", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
expect(await storage.exists("k")).toBe(true);
|
||||
await storage.delete("k");
|
||||
expect(await storage.exists("k")).toBe(false);
|
||||
});
|
||||
|
||||
it("delete returns false for missing keys", { timeout: 5000 }, async () => {
|
||||
expect(await storage.delete("missing")).toBe(false);
|
||||
});
|
||||
|
||||
it("persists nested directory paths via mkdirp", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("a/b/c/key", new Uint8Array([9]));
|
||||
expect(await storage.get("a/b/c/key")).toEqual(new Uint8Array([9]));
|
||||
});
|
||||
|
||||
it("rejects missing rootDir with INVALID_OPTIONS", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
new NodeFsStorage({} as never);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.INVALID_OPTIONS
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects with STORAGE_FILE.ABORTED when signal is pre-aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
for (const op of [
|
||||
() => storage.get("k", controller.signal),
|
||||
() =>
|
||||
storage.set(
|
||||
"k",
|
||||
new Uint8Array([1]),
|
||||
undefined,
|
||||
controller.signal
|
||||
),
|
||||
() => storage.delete("k", controller.signal),
|
||||
() => storage.exists("k", controller.signal),
|
||||
() => storage.list(undefined, controller.signal),
|
||||
() => storage.clear(controller.signal),
|
||||
() => storage.getMetadata("k", controller.signal),
|
||||
() => storage.getPartial("k", 0, 1, controller.signal),
|
||||
]) {
|
||||
try {
|
||||
await op();
|
||||
expect.fail("expected abort");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("writes atomically — no temp files remain after success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3]));
|
||||
const entries = await readdir(root);
|
||||
expect(entries.some((e) => e.startsWith(TEMP_FILE_PREFIX))).toBe(false);
|
||||
});
|
||||
|
||||
it("unlinks the temp file when the underlying writeFile throws", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async writeFile() {
|
||||
throw new Error("simulated failure");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.set("k", new Uint8Array([1]));
|
||||
expect.fail("expected throw");
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
const entries = await readdir(root);
|
||||
expect(entries.some((e) => e.startsWith(TEMP_FILE_PREFIX))).toBe(false);
|
||||
});
|
||||
|
||||
it("retries on EBUSY and resolves on success", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
let attempts = 0;
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async writeFile(path, data) {
|
||||
attempts++;
|
||||
if (attempts === 1) throw errnoError("EBUSY");
|
||||
return base.writeFile(path, data);
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 },
|
||||
});
|
||||
await s.set("k", new Uint8Array([7]));
|
||||
expect(await s.get("k")).toEqual(new Uint8Array([7]));
|
||||
});
|
||||
|
||||
it("maps EACCES to PERMISSION_DENIED on exists()", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async access() {
|
||||
throw errnoError("EACCES");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.exists("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.PERMISSION_DENIED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("maps non-ENOENT non-EACCES failures on exists() to IO_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async access() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.exists("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("writes and reads back metadata sidecar files", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3]), {
|
||||
metadata: { mimeType: "test" },
|
||||
ttl: 60_000,
|
||||
});
|
||||
const meta = await storage.getMetadata("k");
|
||||
expect(meta?.metadata).toEqual({ mimeType: "test" });
|
||||
expect(meta?.size).toBe(3);
|
||||
expect(meta?.expiresAt).toBeGreaterThan(Date.now());
|
||||
const result = await storage.getWithMetadata("k");
|
||||
expect(result?.value).toEqual(new Uint8Array([1, 2, 3]));
|
||||
expect(result?.metadata?.metadata).toEqual({ mimeType: "test" });
|
||||
});
|
||||
|
||||
it("synthesises metadata from stat when sidecar is absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3]));
|
||||
const meta = await storage.getMetadata("k");
|
||||
expect(meta?.size).toBe(3);
|
||||
expect(meta?.metadata).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns null metadata when neither sidecar nor data file exist", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
expect(await storage.getMetadata("missing")).toBeNull();
|
||||
expect(await storage.getWithMetadata("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to synthetic metadata when stat is available but sidecar is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
// Delete metadata file directly via base fs
|
||||
const meta = await storage.getMetadata("k");
|
||||
expect(meta).not.toBeNull();
|
||||
});
|
||||
|
||||
it("lists keys with prefix and pagination", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
for (let i = 0; i < 5; i++)
|
||||
await storage.set(`user/${i}`, new Uint8Array([i]));
|
||||
for (let i = 0; i < 3; i++)
|
||||
await storage.set(`post/${i}`, new Uint8Array([i]));
|
||||
const userList = await storage.list({ prefix: "user/" });
|
||||
expect(userList.keys).toHaveLength(5);
|
||||
expect(userList.keys.every((k) => k.startsWith("user/"))).toBe(true);
|
||||
const page1 = await storage.list({ limit: 3 });
|
||||
expect(page1.keys).toHaveLength(3);
|
||||
expect(page1.hasMore).toBe(true);
|
||||
const page2 = await storage.list({
|
||||
limit: 3,
|
||||
continuationToken: page1.continuationToken,
|
||||
});
|
||||
expect(page2.keys.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("list returns empty for a non-existent rootDir", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: join(root, "nope"),
|
||||
timers: immediateTimers,
|
||||
});
|
||||
const result = await s.list();
|
||||
expect(result).toEqual({ keys: [], total: 0, hasMore: false });
|
||||
});
|
||||
|
||||
it("clear removes all data and metadata files", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
await storage.set("a", new Uint8Array([1]), { metadata: { x: 1 } });
|
||||
await storage.set("b", new Uint8Array([2]));
|
||||
await storage.clear();
|
||||
expect((await storage.list()).keys).toHaveLength(0);
|
||||
const remaining = await readdir(root, {
|
||||
recursive: true,
|
||||
withFileTypes: true,
|
||||
});
|
||||
const files = remaining.filter((e) => e.isFile()).map((e) => e.name);
|
||||
expect(files).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("clear is a no-op when rootDir does not exist", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: join(root, "nope"),
|
||||
timers: immediateTimers,
|
||||
});
|
||||
await s.clear();
|
||||
});
|
||||
|
||||
it("getSize sums sizes across all keys", { timeout: 5000 }, async () => {
|
||||
await storage.set("a", new Uint8Array([1, 2, 3]));
|
||||
await storage.set("b", new Uint8Array([4, 5]));
|
||||
expect(await storage.getSize()).toBe(5);
|
||||
});
|
||||
|
||||
it("reads a positioned slice via getPartial", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new TextEncoder().encode("0123456789"));
|
||||
const slice = await storage.getPartial("k", 3, 4);
|
||||
expect(new TextDecoder().decode(slice ?? new Uint8Array())).toBe(
|
||||
"3456"
|
||||
);
|
||||
});
|
||||
|
||||
it("truncates getPartial at EOF", { timeout: 5000 }, async () => {
|
||||
await storage.set("k", new TextEncoder().encode("0123456789"));
|
||||
const slice = await storage.getPartial("k", 8, 100);
|
||||
expect(slice?.byteLength).toBe(2);
|
||||
});
|
||||
|
||||
it("getPartial returns null when the key is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
expect(await storage.getPartial("missing", 0, 10)).toBeNull();
|
||||
});
|
||||
|
||||
it("getPartial rejects negative or non-integer offset/length", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
for (const args of [
|
||||
[-1, 1],
|
||||
[0, -1],
|
||||
[0.5, 1],
|
||||
[0, 0.5],
|
||||
] as const) {
|
||||
try {
|
||||
await storage.getPartial("k", args[0], args[1]);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.INVALID_KEY
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("getItemSize returns size from metadata", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3]));
|
||||
expect(await storage.getItemSize("k")).toBe(3);
|
||||
expect(await storage.getItemSize("missing")).toBeNull();
|
||||
});
|
||||
|
||||
it("setPartial delegates to defaultSetPartial and writes the result", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3, 4, 5]));
|
||||
await storage.setPartial("k", 2, new Uint8Array([9, 9]));
|
||||
expect(await storage.get("k")).toEqual(new Uint8Array([1, 2, 9, 9, 5]));
|
||||
});
|
||||
|
||||
it("createReadStream + createWriteStream round-trips data", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const writer = storage.createWriteStream("stream").getWriter();
|
||||
await writer.write(new TextEncoder().encode("hello"));
|
||||
await writer.close();
|
||||
const r = storage.createReadStream("stream").getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
for (;;) {
|
||||
const { done, value } = await r.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
}
|
||||
const concat = Buffer.concat(chunks);
|
||||
expect(new TextDecoder().decode(new Uint8Array(concat))).toBe("hello");
|
||||
});
|
||||
|
||||
it("factory createNodeFsStorage returns a NodeFsStorage instance", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const s = createNodeFsStorage({ rootDir: root });
|
||||
expect(s).toBeInstanceOf(NodeFsStorage);
|
||||
});
|
||||
|
||||
it("skips mkdirp when set to false and bubbles ENOENT", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: join(root, "nope"),
|
||||
mkdirp: false,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
// Non-transient errnos pass through unwrapped so callers can map ENOENT.
|
||||
await expect(s.set("a/b/c", new Uint8Array([1]))).rejects.toMatchObject(
|
||||
{ code: "ENOENT" }
|
||||
);
|
||||
});
|
||||
|
||||
it("signal aborts during a write still leave no temp file", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const controller = new AbortController();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async writeFile(path, data) {
|
||||
controller.abort();
|
||||
return base.writeFile(path, data);
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
await s.set("k", new Uint8Array([1])); // first set succeeds
|
||||
try {
|
||||
await s.set(
|
||||
"k2",
|
||||
new Uint8Array([2]),
|
||||
undefined,
|
||||
controller.signal
|
||||
);
|
||||
} catch {
|
||||
// expected — signal already aborted by hook fires after some writes
|
||||
}
|
||||
const orphans = (await readdir(root)).filter((e) =>
|
||||
e.startsWith(TEMP_FILE_PREFIX)
|
||||
);
|
||||
expect(orphans).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("writes work without retry configuration", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const s = new NodeFsStorage({ rootDir: root });
|
||||
await s.set("k", new Uint8Array([1]));
|
||||
expect(await s.get("k")).toEqual(new Uint8Array([1]));
|
||||
});
|
||||
|
||||
it("forwards a non-aborted signal through retry options", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
await storage.set(
|
||||
"k",
|
||||
new Uint8Array([1, 2, 3]),
|
||||
undefined,
|
||||
controller.signal
|
||||
);
|
||||
expect(await storage.get("k", controller.signal)).toEqual(
|
||||
new Uint8Array([1, 2, 3])
|
||||
);
|
||||
});
|
||||
|
||||
it("set preserves bytes across very small and 1 MiB payloads", {
|
||||
timeout: 15000,
|
||||
}, async () => {
|
||||
await storage.set("small", new Uint8Array([0xff]));
|
||||
expect(await storage.get("small")).toEqual(new Uint8Array([0xff]));
|
||||
const big = new Uint8Array(1024 * 1024).fill(0x55);
|
||||
await storage.set("big", big);
|
||||
const back = await storage.get("big");
|
||||
expect(back?.byteLength).toBe(big.byteLength);
|
||||
expect(back?.[1024]).toBe(0x55);
|
||||
});
|
||||
|
||||
it("propagates non-StorageFileError from get when retry classifies it", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readFile() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.get("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error & { code?: string }).code).toBe("EIO");
|
||||
}
|
||||
});
|
||||
|
||||
it("clear skips directory entries when walking recursive readdir output", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("nested/deep/key", new Uint8Array([1]));
|
||||
await storage.clear();
|
||||
expect((await storage.list()).keys).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("clear ignores unrelated files in the rootDir", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
// Drop a foreign file
|
||||
const foreign = join(root, "README.txt");
|
||||
await defaultNodeFs().writeFile(
|
||||
foreign,
|
||||
new TextEncoder().encode("readme")
|
||||
);
|
||||
await storage.clear();
|
||||
expect(existsSync(foreign)).toBe(true);
|
||||
expect(await readFile(foreign, "utf-8")).toBe("readme");
|
||||
});
|
||||
|
||||
it("delete propagates non-ENOENT non-transient errors", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async unlink() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.delete("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error & { code?: string }).code).toBe("EIO");
|
||||
}
|
||||
});
|
||||
|
||||
it("list propagates non-ENOENT readdir errors", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readdir() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.list();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error & { code?: string }).code).toBe("EIO");
|
||||
}
|
||||
});
|
||||
|
||||
it("clear propagates non-ENOENT readdir errors", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readdir() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.clear();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error & { code?: string }).code).toBe("EIO");
|
||||
}
|
||||
});
|
||||
|
||||
it("getPartial propagates non-ENOENT open errors", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async open() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.getPartial("k", 0, 1);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error & { code?: string }).code).toBe("EIO");
|
||||
}
|
||||
});
|
||||
|
||||
it("getMetadata throws on a corrupted sidecar", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]), { metadata: { x: 1 } });
|
||||
const metaPath = join(root, "k.meta");
|
||||
await defaultNodeFs().writeFile(
|
||||
metaPath,
|
||||
new TextEncoder().encode("not-json{")
|
||||
);
|
||||
try {
|
||||
await storage.getMetadata("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
}
|
||||
});
|
||||
|
||||
it("createReadStream and createWriteStream accept an AbortSignal", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
const w = storage.createWriteStream("k", ac.signal).getWriter();
|
||||
await w.write(new TextEncoder().encode("hi"));
|
||||
await w.close();
|
||||
const r = storage.createReadStream("k", ac.signal).getReader();
|
||||
const { value, done } = await r.read();
|
||||
expect(done).toBe(false);
|
||||
expect(new TextDecoder().decode(value)).toBe("hi");
|
||||
await r.cancel();
|
||||
});
|
||||
|
||||
it("createWriteStream skips mkdirp when disabled", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
mkdirp: false,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
const w = s.createWriteStream("inline").getWriter();
|
||||
await w.write(new Uint8Array([1]));
|
||||
await w.close();
|
||||
expect(await s.get("inline")).toEqual(new Uint8Array([1]));
|
||||
});
|
||||
|
||||
it("get rewraps an internal StorageFileError when the retry helper raises one", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const innerErr = new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"inner"
|
||||
);
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readFile() {
|
||||
throw innerErr;
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.get("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBe(innerErr);
|
||||
}
|
||||
});
|
||||
|
||||
it("delete rewraps an internal StorageFileError when the retry helper raises one", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const innerErr = new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"inner"
|
||||
);
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async unlink() {
|
||||
throw innerErr;
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.delete("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBe(innerErr);
|
||||
}
|
||||
});
|
||||
|
||||
it("clear ignores files lacking both data and metadata extensions", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
// directly write a non-extension file inside the rootDir
|
||||
await defaultNodeFs().writeFile(
|
||||
join(root, "loose.txt"),
|
||||
new TextEncoder().encode("x")
|
||||
);
|
||||
await storage.clear();
|
||||
expect((await storage.list()).keys).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("getWithMetadata synthesises metadata when the sidecar is missing", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
// Set without metadata so no sidecar — but `getMetadata` falls back to stat,
|
||||
// which returns a real record. To exercise the syntheticMetadata branch,
|
||||
// simulate a hook where getMetadata returns null.
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readFile(path) {
|
||||
if (path.endsWith(".meta")) {
|
||||
throw Object.assign(new Error("noent"), { code: "ENOENT" });
|
||||
}
|
||||
return base.readFile(path);
|
||||
},
|
||||
async stat(path) {
|
||||
throw Object.assign(new Error("noent"), { code: "ENOENT" });
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
clock: () => 1_700_000_000_000,
|
||||
});
|
||||
await defaultNodeFs().writeFile(
|
||||
join(root, "k.data"),
|
||||
new TextEncoder().encode("hi")
|
||||
);
|
||||
const result = await s.getWithMetadata("k");
|
||||
expect(result?.value).toBeDefined();
|
||||
expect(result?.metadata.size).toBe(2);
|
||||
expect(result?.metadata.createdAt).toBe(1_700_000_000_000);
|
||||
});
|
||||
|
||||
it("getSize skips keys whose metadata getter returns null", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
// Hook so that 'ghost' appears in list() but getMetadata returns null.
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readFile(path) {
|
||||
if (
|
||||
path.endsWith("ghost.meta") ||
|
||||
path.endsWith("ghost.data")
|
||||
) {
|
||||
throw Object.assign(new Error("noent"), { code: "ENOENT" });
|
||||
}
|
||||
return base.readFile(path);
|
||||
},
|
||||
async stat(path) {
|
||||
if (path.endsWith("ghost.data")) {
|
||||
throw Object.assign(new Error("noent"), { code: "ENOENT" });
|
||||
}
|
||||
return base.stat(path);
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
await s.set("a", new Uint8Array([1, 2, 3]));
|
||||
// Create a ghost data file that list() will see but the hook makes getMetadata return null
|
||||
await defaultNodeFs().writeFile(
|
||||
join(root, "ghost.data"),
|
||||
new Uint8Array(99)
|
||||
);
|
||||
const size = await s.getSize();
|
||||
expect(size).toBe(3);
|
||||
});
|
||||
|
||||
it("set writes metadata when ttl is provided without explicit metadata object", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]), { ttl: 60_000 });
|
||||
const meta = await storage.getMetadata("k");
|
||||
expect(meta?.expiresAt).toBeGreaterThan(Date.now());
|
||||
expect(meta?.metadata).toBeUndefined();
|
||||
});
|
||||
|
||||
it("clear swallows unlink failures for individual files", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async unlink() {
|
||||
throw Object.assign(new Error("io"), { code: "EIO" });
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
await s.clear();
|
||||
});
|
||||
|
||||
it("getPartial swallows handle close failures", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1, 2, 3, 4, 5]));
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async open(path, flags) {
|
||||
const handle = await base.open(path, flags);
|
||||
return {
|
||||
...handle,
|
||||
async close() {
|
||||
throw new Error("close failed");
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
const slice = await s.getPartial("k", 1, 2);
|
||||
expect(slice).toEqual(new Uint8Array([2, 3]));
|
||||
});
|
||||
|
||||
it("assertExists resolves when present and throws NOT_FOUND when absent", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
await storage.assertExists("k");
|
||||
try {
|
||||
await storage.assertExists("missing");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.NOT_FOUND
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("createWriteStream swallows mkdir failures", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async mkdir() {
|
||||
throw Object.assign(new Error("io"), { code: "EIO" });
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
// Creating the stream synchronously kicks off the background mkdir;
|
||||
// the .catch swallows the error. We just need to construct the stream
|
||||
// to fire the path — actual write may or may not succeed.
|
||||
const w = s.createWriteStream("k");
|
||||
// Wait a microtask for the void mkdir to settle.
|
||||
await new Promise((r) => setImmediate(r));
|
||||
expect(w).toBeInstanceOf(WritableStream);
|
||||
});
|
||||
|
||||
it("getMetadata propagates a non-ENOENT stat error in the fallback branch", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const base = defaultNodeFs();
|
||||
const hooked: NodeFsLike = {
|
||||
...base,
|
||||
async readFile() {
|
||||
throw errnoError("ENOENT");
|
||||
},
|
||||
async stat() {
|
||||
throw errnoError("EIO");
|
||||
},
|
||||
};
|
||||
const s = new NodeFsStorage({
|
||||
rootDir: root,
|
||||
fs: hooked,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
try {
|
||||
await s.getMetadata("k");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as Error & { code?: string }).code).toBe("EIO");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,421 @@
|
||||
import { dirname, join } from "node:path";
|
||||
import { AbstractStorage } from "@suaveplan/storage/abstract";
|
||||
import type {
|
||||
StorageCapabilities,
|
||||
StorageListOptions,
|
||||
StorageListResult,
|
||||
StorageMetadata,
|
||||
StorageOptions,
|
||||
StorageResult,
|
||||
} from "@suaveplan/storage/types";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import { sanitizeKey } from "../key-sanitizer/key-sanitizer.js";
|
||||
import { recordOp, type StorageFileOp } from "../otel/otel.js";
|
||||
import {
|
||||
isPermissionFsError,
|
||||
type WithFsRetryOptions,
|
||||
withFsRetry,
|
||||
} from "../retry-helper/retry-helper.js";
|
||||
import type { NodeFsReaddirEntry } from "../types/fs-adapter.types.js";
|
||||
import type { NodeFsStorageOptions } from "../types/node-fs-storage.types.js";
|
||||
import {
|
||||
buildListPage,
|
||||
buildTempPath,
|
||||
entryRelativePath,
|
||||
isEnoent,
|
||||
type ResolvedConfig,
|
||||
readMetadataWithStatFallback,
|
||||
resolveConfig,
|
||||
resolveListOptions,
|
||||
syntheticMetadata,
|
||||
} from "./node-fs-storage.helpers.js";
|
||||
import {
|
||||
assertKeyExists,
|
||||
assertPositiveInt,
|
||||
buildMetadataPayload,
|
||||
buildReadStream,
|
||||
buildWriteStream,
|
||||
readPositioned,
|
||||
} from "./node-fs-storage.ops.js";
|
||||
|
||||
export class NodeFsStorage extends AbstractStorage {
|
||||
readonly #config: ResolvedConfig;
|
||||
|
||||
public override readonly capabilities: StorageCapabilities = {
|
||||
partialReads: true,
|
||||
partialWrites: false,
|
||||
nativeStreams: true,
|
||||
atomicWrites: true,
|
||||
contentAddressable: false,
|
||||
};
|
||||
|
||||
public constructor(options: NodeFsStorageOptions) {
|
||||
super();
|
||||
this.#config = resolveConfig(options);
|
||||
}
|
||||
|
||||
#dataPath(key: string): string {
|
||||
return join(
|
||||
this.#config.rootDir,
|
||||
`${key}${this.#config.fileExtension}`
|
||||
);
|
||||
}
|
||||
|
||||
#metaPath(key: string): string {
|
||||
return join(
|
||||
this.#config.rootDir,
|
||||
`${key}${this.#config.metadataExtension}`
|
||||
);
|
||||
}
|
||||
|
||||
#retryOpts(
|
||||
key: string,
|
||||
signal: AbortSignal | undefined
|
||||
): WithFsRetryOptions {
|
||||
const opts: WithFsRetryOptions = { key };
|
||||
if (this.#config.retry !== undefined) opts.retry = this.#config.retry;
|
||||
if (this.#config.timers !== undefined)
|
||||
opts.timers = this.#config.timers;
|
||||
if (signal !== undefined) opts.signal = signal;
|
||||
return opts;
|
||||
}
|
||||
|
||||
#assertSignal(
|
||||
signal: AbortSignal | undefined,
|
||||
key: string | undefined
|
||||
): void {
|
||||
if (signal?.aborted) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.ABORTED,
|
||||
"Operation aborted",
|
||||
key !== undefined ? { key } : {}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async #wrap<T>(
|
||||
op: StorageFileOp,
|
||||
key: string | undefined,
|
||||
signal: AbortSignal | undefined,
|
||||
fn: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const start = Date.now();
|
||||
try {
|
||||
this.#assertSignal(signal, key);
|
||||
const value = await fn();
|
||||
recordOp({
|
||||
op,
|
||||
outcome: "success",
|
||||
durationMs: Date.now() - start,
|
||||
});
|
||||
return value;
|
||||
} catch (err) {
|
||||
const outcome =
|
||||
err instanceof StorageFileError &&
|
||||
err.code === STORAGE_FILE_CODES.ABORTED
|
||||
? "aborted"
|
||||
: "error";
|
||||
recordOp({ op, outcome, durationMs: Date.now() - start });
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async #atomicWrite(
|
||||
destination: string,
|
||||
payload: Uint8Array,
|
||||
key: string,
|
||||
signal: AbortSignal | undefined
|
||||
): Promise<void> {
|
||||
const temp = buildTempPath(destination);
|
||||
try {
|
||||
await withFsRetry(
|
||||
() => this.#config.fs.writeFile(temp, payload),
|
||||
this.#retryOpts(key, signal)
|
||||
);
|
||||
await withFsRetry(
|
||||
() => this.#config.fs.rename(temp, destination),
|
||||
this.#retryOpts(key, signal)
|
||||
);
|
||||
} catch (err) {
|
||||
await this.#config.fs.unlink(temp).catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
override async get(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<Uint8Array | undefined> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("get", safe, signal, async () => {
|
||||
try {
|
||||
const bytes = await withFsRetry(
|
||||
() => this.#config.fs.readFile(this.#dataPath(safe)),
|
||||
this.#retryOpts(safe, signal)
|
||||
);
|
||||
recordOp({
|
||||
op: "get",
|
||||
outcome: "success",
|
||||
durationMs: 0,
|
||||
bytes: bytes.byteLength,
|
||||
direction: "read",
|
||||
});
|
||||
return bytes;
|
||||
} catch (err) {
|
||||
if (err instanceof StorageFileError) throw err;
|
||||
if (isEnoent(err)) return undefined;
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
override async set(
|
||||
key: string,
|
||||
value: Uint8Array,
|
||||
options?: StorageOptions,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const safe = sanitizeKey(key);
|
||||
await this.validateValueSize(safe, value, signal);
|
||||
await this.#wrap("set", safe, signal, async () => {
|
||||
const path = this.#dataPath(safe);
|
||||
if (this.#config.mkdirp)
|
||||
await this.#config.fs.mkdir(dirname(path), { recursive: true });
|
||||
await this.#atomicWrite(path, value, safe, signal);
|
||||
recordOp({
|
||||
op: "set",
|
||||
outcome: "success",
|
||||
durationMs: 0,
|
||||
bytes: value.byteLength,
|
||||
direction: "write",
|
||||
});
|
||||
const payload = buildMetadataPayload(
|
||||
this.#config.clock,
|
||||
value.byteLength,
|
||||
options
|
||||
);
|
||||
if (payload !== undefined) {
|
||||
await this.#atomicWrite(
|
||||
this.#metaPath(safe),
|
||||
payload.encoded,
|
||||
safe,
|
||||
signal
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
override async delete(key: string, signal?: AbortSignal): Promise<boolean> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("delete", safe, signal, async () => {
|
||||
try {
|
||||
await withFsRetry(
|
||||
() => this.#config.fs.unlink(this.#dataPath(safe)),
|
||||
this.#retryOpts(safe, signal)
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof StorageFileError) throw err;
|
||||
if (isEnoent(err)) return false;
|
||||
throw err;
|
||||
}
|
||||
await this.#config.fs
|
||||
.unlink(this.#metaPath(safe))
|
||||
.catch(() => undefined);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
override async exists(key: string, signal?: AbortSignal): Promise<boolean> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("exists", safe, signal, async () => {
|
||||
try {
|
||||
await this.#config.fs.access(this.#dataPath(safe));
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (isEnoent(err)) return false;
|
||||
if (isPermissionFsError(err)) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.PERMISSION_DENIED,
|
||||
`Permission denied: ${(err as Error).message}`,
|
||||
{ key: safe, cause: err as Error }
|
||||
);
|
||||
}
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.IO_FAILED,
|
||||
`exists failed: ${(err as Error).message}`,
|
||||
{ key: safe, cause: err as Error }
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
override async list(
|
||||
options?: StorageListOptions,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageListResult> {
|
||||
return this.#wrap("list", undefined, signal, async () => {
|
||||
try {
|
||||
const entries = await this.#config.fs.readdir(
|
||||
this.#config.rootDir,
|
||||
{ recursive: true, withFileTypes: true }
|
||||
);
|
||||
const resolved = resolveListOptions(options, entries.length);
|
||||
return buildListPage({
|
||||
entries,
|
||||
rootDir: this.#config.rootDir,
|
||||
fileExtension: this.#config.fileExtension,
|
||||
...resolved,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isEnoent(err))
|
||||
return { keys: [], total: 0, hasMore: false };
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
override async clear(signal?: AbortSignal): Promise<void> {
|
||||
await this.#wrap("clear", undefined, signal, async () => {
|
||||
let entries: NodeFsReaddirEntry[];
|
||||
try {
|
||||
entries = await this.#config.fs.readdir(this.#config.rootDir, {
|
||||
recursive: true,
|
||||
withFileTypes: true,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isEnoent(err)) return;
|
||||
throw err;
|
||||
}
|
||||
const ext = this.#config.fileExtension;
|
||||
const metaExt = this.#config.metadataExtension;
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (!entry.name.endsWith(ext) && !entry.name.endsWith(metaExt))
|
||||
continue;
|
||||
const rel = entryRelativePath(entry, this.#config.rootDir);
|
||||
await this.#config.fs
|
||||
.unlink(join(this.#config.rootDir, rel))
|
||||
.catch(() => undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
override async getWithMetadata(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageResult<Uint8Array> | null> {
|
||||
const value = await this.get(key, signal);
|
||||
if (value === undefined) return null;
|
||||
const metadata =
|
||||
(await this.getMetadata(key, signal)) ??
|
||||
syntheticMetadata(this.#config.clock, value.byteLength);
|
||||
return { value, metadata };
|
||||
}
|
||||
|
||||
override async getMetadata(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<StorageMetadata | null> {
|
||||
const safe = sanitizeKey(key);
|
||||
return this.#wrap("getMetadata", safe, signal, () =>
|
||||
readMetadataWithStatFallback(
|
||||
this.#config.fs,
|
||||
this.#metaPath(safe),
|
||||
this.#dataPath(safe)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
override async getSize(signal?: AbortSignal): Promise<number> {
|
||||
const { keys } = await this.list(undefined, signal);
|
||||
let total = 0;
|
||||
for (const key of keys) {
|
||||
const meta = await this.getMetadata(key, signal);
|
||||
if (meta) total += meta.size;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
override async getPartial(
|
||||
key: string,
|
||||
offset: number,
|
||||
length: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<Uint8Array | null> {
|
||||
const safe = sanitizeKey(key);
|
||||
assertPositiveInt(safe, offset, length);
|
||||
return this.#wrap("getPartial", safe, signal, () =>
|
||||
readPositioned({
|
||||
fs: this.#config.fs,
|
||||
path: this.#dataPath(safe),
|
||||
offset,
|
||||
length,
|
||||
key: safe,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
override async setPartial(
|
||||
key: string,
|
||||
offset: number,
|
||||
bytes: Uint8Array,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const safe = sanitizeKey(key);
|
||||
await this.defaultSetPartial(safe, offset, bytes, signal);
|
||||
}
|
||||
|
||||
override async getItemSize(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<number | null> {
|
||||
const meta = await this.getMetadata(key, signal);
|
||||
return meta?.size ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict variant of {@link exists}: throws `STORAGE_FILE.NOT_FOUND` when
|
||||
* the key is absent. Useful for callers that prefer error-throwing over
|
||||
* boolean return values (e.g. assertions during migrations).
|
||||
*/
|
||||
public async assertExists(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
const safe = sanitizeKey(key);
|
||||
await assertKeyExists(safe, (k, s) => this.exists(k, s), signal);
|
||||
}
|
||||
|
||||
override createReadStream(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): ReadableStream<Uint8Array> {
|
||||
const safe = sanitizeKey(key);
|
||||
return buildReadStream({
|
||||
fs: this.#config.fs,
|
||||
path: this.#dataPath(safe),
|
||||
key: safe,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
|
||||
override createWriteStream(
|
||||
key: string,
|
||||
signal?: AbortSignal
|
||||
): WritableStream<Uint8Array> {
|
||||
const safe = sanitizeKey(key);
|
||||
return buildWriteStream({
|
||||
fs: this.#config.fs,
|
||||
destination: this.#dataPath(safe),
|
||||
mkdirp: this.#config.mkdirp,
|
||||
key: safe,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function createNodeFsStorage(
|
||||
options: NodeFsStorageOptions
|
||||
): NodeFsStorage {
|
||||
return new NodeFsStorage(options);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { defaultNodeFs } from "./node-fs.js";
|
||||
@@ -0,0 +1,60 @@
|
||||
# `node-fs`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Provides the default `NodeFsLike` implementation backed by `node:fs/promises` and the two stream constructors from `node:fs`. The factory is the single binding point between this package and Node's filesystem APIs — every other module talks to the structural `NodeFsLike` interface so tests can substitute real in-memory or hooked adapters without `vi.mock`.
|
||||
|
||||
## Features
|
||||
|
||||
- Returns a fresh `NodeFsLike` object per call so callers can spread-and-override individual methods without mutating shared state.
|
||||
- Forwards `readFile` results through `new Uint8Array(buffer, byteOffset, byteLength)` so consumers get a stable `Uint8Array` view regardless of the underlying Buffer subclass.
|
||||
- Wraps `node:fs/promises.open` so the returned handle matches the structural `NodeFsFileHandleLike` interface (only `read` + `close`).
|
||||
- Translates `readdir(path, { recursive: true, withFileTypes: true })` Dirent entries into the lighter `NodeFsReaddirEntry` shape exposed to the rest of the package.
|
||||
- Translates `stat` into the lighter `NodeFsStatLike` shape exposing only the four fields the adapter reads.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { defaultNodeFs } from "@suaveplan/storage-file";
|
||||
|
||||
const fs = defaultNodeFs();
|
||||
await fs.mkdir("/tmp/store", { recursive: true });
|
||||
await fs.writeFile("/tmp/store/hello.data", new TextEncoder().encode("hi"));
|
||||
const data = await fs.readFile("/tmp/store/hello.data");
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { defaultNodeFs, NodeFsStorage } from "@suaveplan/storage-file";
|
||||
import type { NodeFsLike } from "@suaveplan/storage-file/types";
|
||||
|
||||
const base = defaultNodeFs();
|
||||
|
||||
// Wrap stat to ensure mtimes never go backwards under wall-clock skew.
|
||||
const monotonicStat: NodeFsLike = {
|
||||
...base,
|
||||
async stat(path) {
|
||||
const s = await base.stat(path);
|
||||
return { ...s, mtimeMs: Math.max(s.mtimeMs, Date.now()) };
|
||||
},
|
||||
};
|
||||
|
||||
const storage = new NodeFsStorage({ rootDir: "/tmp/store", fs: monotonicStat });
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `defaultNodeFs(): NodeFsLike` — returns a fresh adapter object.
|
||||
|
||||
Every method on the returned object forwards to its `node:fs/promises` counterpart with minimal shape translation. See `fs-adapter.types.md` for the full method list.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The factory pattern (vs a module-level singleton) is deliberate: returning a fresh object lets tests construct independent adapters per `it(...)` block and verify ownership semantics.
|
||||
- The factory is dependency-free at module evaluation time — every Node import is at the top of the file, but `defaultNodeFs()` itself doesn't reach into globals or open any handles. Callers can build the storage adapter at import time without paying any filesystem cost.
|
||||
- The lazy buffer-view translation in `readFile` avoids a copy when callers operate on the result via DataView or pass it back into a hash function.
|
||||
@@ -0,0 +1,113 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { defaultNodeFs } from "./node-fs.js";
|
||||
|
||||
describe("defaultNodeFs", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "storage-file-node-fs-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("forwards every method to node:fs/promises", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const fs = defaultNodeFs();
|
||||
const dir = join(root, "nested");
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const path = join(dir, "k.data");
|
||||
await fs.writeFile(path, new TextEncoder().encode("hi"));
|
||||
await fs.access(path);
|
||||
const stat = await fs.stat(path);
|
||||
expect(stat.size).toBe(2);
|
||||
expect(stat.isFile()).toBe(true);
|
||||
expect(stat.isDirectory()).toBe(false);
|
||||
const bytes = await fs.readFile(path);
|
||||
expect(new TextDecoder().decode(bytes)).toBe("hi");
|
||||
const renamed = join(dir, "moved.data");
|
||||
await fs.rename(path, renamed);
|
||||
const entries = await fs.readdir(root, {
|
||||
recursive: true,
|
||||
withFileTypes: true,
|
||||
});
|
||||
const names = entries.map((e) => e.name).sort();
|
||||
expect(names).toContain("nested");
|
||||
expect(
|
||||
entries.some((e) => e.isFile() && e.name.includes("moved"))
|
||||
).toBe(true);
|
||||
await fs.unlink(renamed);
|
||||
});
|
||||
|
||||
it("readdir entries expose isFile and isDirectory predicates", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const fs = defaultNodeFs();
|
||||
await fs.mkdir(join(root, "child"), { recursive: true });
|
||||
await fs.writeFile(join(root, "file.data"), new Uint8Array([1]));
|
||||
const entries = await fs.readdir(root, {
|
||||
recursive: true,
|
||||
withFileTypes: true,
|
||||
});
|
||||
const file = entries.find((e) => e.name === "file.data");
|
||||
const dir = entries.find((e) => e.name === "child");
|
||||
expect(file?.isFile()).toBe(true);
|
||||
expect(file?.isDirectory()).toBe(false);
|
||||
expect(dir?.isDirectory()).toBe(true);
|
||||
expect(dir?.isFile()).toBe(false);
|
||||
});
|
||||
|
||||
it("opens a file handle that supports positioned read and close", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const fs = defaultNodeFs();
|
||||
const path = join(root, "h.data");
|
||||
await fs.writeFile(path, new TextEncoder().encode("0123456789"));
|
||||
const handle = await fs.open(path, "r");
|
||||
try {
|
||||
const buffer = new Uint8Array(3);
|
||||
const result = await handle.read(buffer, 0, 3, 5);
|
||||
expect(result.bytesRead).toBe(3);
|
||||
expect(new TextDecoder().decode(result.buffer)).toBe("567");
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns Node streams from createReadStream and createWriteStream", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const fs = defaultNodeFs();
|
||||
const path = join(root, "stream.data");
|
||||
await fs.writeFile(path, new TextEncoder().encode("abc"));
|
||||
const r = fs.createReadStream(path);
|
||||
const chunks: Buffer[] = [];
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
r.on("data", (c) => chunks.push(c as Buffer));
|
||||
r.on("end", () => resolve());
|
||||
r.on("error", reject);
|
||||
});
|
||||
const buf = Buffer.concat(chunks);
|
||||
expect(new TextDecoder().decode(new Uint8Array(buf))).toBe("abc");
|
||||
const writePath = join(root, "out.data");
|
||||
const w = fs.createWriteStream(writePath);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
w.write(Buffer.from("xyz"));
|
||||
w.end((err: Error | null | undefined) =>
|
||||
err ? reject(err) : resolve()
|
||||
);
|
||||
});
|
||||
expect(new TextDecoder().decode(await fs.readFile(writePath))).toBe(
|
||||
"xyz"
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import {
|
||||
access,
|
||||
mkdir,
|
||||
open,
|
||||
readdir,
|
||||
readFile,
|
||||
rename,
|
||||
stat,
|
||||
unlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import type {
|
||||
NodeFsCreateReadStreamOptions,
|
||||
NodeFsCreateWriteStreamOptions,
|
||||
NodeFsFileHandleLike,
|
||||
NodeFsLike,
|
||||
NodeFsReaddirEntry,
|
||||
NodeFsStatLike,
|
||||
} from "../types/fs-adapter.types.js";
|
||||
|
||||
/**
|
||||
* Default {@link NodeFsLike} implementation backed by `node:fs/promises`
|
||||
* and the two `node:fs` stream constructors. Returned as a fresh object
|
||||
* per call so a caller can selectively override individual methods
|
||||
* without mutating shared state.
|
||||
*/
|
||||
export function defaultNodeFs(): NodeFsLike {
|
||||
return {
|
||||
async mkdir(path, options) {
|
||||
return mkdir(path, options);
|
||||
},
|
||||
async readFile(path) {
|
||||
const buf = await readFile(path);
|
||||
return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
||||
},
|
||||
async writeFile(path, data) {
|
||||
await writeFile(path, data);
|
||||
},
|
||||
async rename(oldPath, newPath) {
|
||||
await rename(oldPath, newPath);
|
||||
},
|
||||
async unlink(path) {
|
||||
await unlink(path);
|
||||
},
|
||||
async readdir(path, options): Promise<NodeFsReaddirEntry[]> {
|
||||
const entries = await readdir(path, options);
|
||||
return entries.map((entry) => ({
|
||||
name: entry.name,
|
||||
parentPath: entry.parentPath,
|
||||
isFile: () => entry.isFile(),
|
||||
isDirectory: () => entry.isDirectory(),
|
||||
}));
|
||||
},
|
||||
async stat(path): Promise<NodeFsStatLike> {
|
||||
const s = await stat(path);
|
||||
return {
|
||||
size: s.size,
|
||||
mtimeMs: s.mtimeMs,
|
||||
birthtimeMs: s.birthtimeMs,
|
||||
isFile: () => s.isFile(),
|
||||
isDirectory: () => s.isDirectory(),
|
||||
};
|
||||
},
|
||||
async access(path) {
|
||||
await access(path);
|
||||
},
|
||||
async open(path, flags): Promise<NodeFsFileHandleLike> {
|
||||
const handle = await open(path, flags);
|
||||
return {
|
||||
async read(buffer, offset, length, position) {
|
||||
const result = await handle.read(
|
||||
buffer,
|
||||
offset,
|
||||
length,
|
||||
position
|
||||
);
|
||||
return { bytesRead: result.bytesRead, buffer };
|
||||
},
|
||||
async close() {
|
||||
await handle.close();
|
||||
},
|
||||
};
|
||||
},
|
||||
createReadStream(path, options?: NodeFsCreateReadStreamOptions) {
|
||||
return createReadStream(path, options);
|
||||
},
|
||||
createWriteStream(path, options?: NodeFsCreateWriteStreamOptions) {
|
||||
return createWriteStream(path, options);
|
||||
},
|
||||
};
|
||||
}
|
||||
12
packages/persistence/storage/storage-file/src/otel/index.ts
Normal file
12
packages/persistence/storage/storage-file/src/otel/index.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
export type {
|
||||
RecordOpInput,
|
||||
StorageFileInstruments,
|
||||
StorageFileOp,
|
||||
StorageFileOpOutcome,
|
||||
} from "./otel.js";
|
||||
export {
|
||||
getInstruments,
|
||||
recordOp,
|
||||
resetInstruments,
|
||||
setStorageFileMeterProvider,
|
||||
} from "./otel.js";
|
||||
64
packages/persistence/storage/storage-file/src/otel/otel.md
Normal file
64
packages/persistence/storage/storage-file/src/otel/otel.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# `otel`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Centralises OpenTelemetry instrumentation for `@suaveplan/storage-file`. Every public path in `NodeFsStorage` records the operation outcome via `recordOp`, so a deployment that installs an OTel SDK gets per-operation counters, latency histograms, and byte-flow histograms for free — no plumbing in user code. The instruments are bound lazily so consumers can install a custom `MeterProvider`/`TracerProvider` before the first operation runs and have it picked up automatically.
|
||||
|
||||
## Features
|
||||
|
||||
- 4 counters: `storage_file.reads_total`, `storage_file.writes_total`, `storage_file.deletes_total`, `storage_file.errors_total` — all attributed by `op` and `outcome`.
|
||||
- 2 histograms: `storage_file.op_duration_ms{op,outcome}` and `storage_file.io_bytes{op,direction}`.
|
||||
- 1 tracer: `@suaveplan/storage-file` (spans created at the call site, not auto-recorded here, so a caller can decorate them with key/extras).
|
||||
- Lazy instrument cache — `getInstruments()` builds the set on first call and reuses it thereafter. `resetInstruments()` blows the cache so tests can rebuild against a freshly-installed in-memory MeterProvider.
|
||||
- Outcome enum closed to `success`/`error`/`aborted` — bounded cardinality for dashboards.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { recordOp } from "@suaveplan/storage-file";
|
||||
|
||||
const start = performance.now();
|
||||
try {
|
||||
await fs.writeFile(path, data);
|
||||
recordOp({ op: "set", outcome: "success", durationMs: performance.now() - start, bytes: data.byteLength, direction: "write" });
|
||||
} catch (err) {
|
||||
recordOp({ op: "set", outcome: "error", durationMs: performance.now() - start });
|
||||
throw err;
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { resetInstruments, getInstruments } from "@suaveplan/storage-file";
|
||||
import { metrics } from "@opentelemetry/api";
|
||||
|
||||
class CollectingMeter { /* … real in-memory MeterProvider … */ }
|
||||
|
||||
const provider = new CollectingMeter();
|
||||
metrics.setGlobalMeterProvider(provider as never);
|
||||
resetInstruments(); // rebuild against the freshly-installed provider
|
||||
const inst = getInstruments();
|
||||
// emit ops through your call site, then read the recorded values via the provider's collector.
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `getInstruments(force?: boolean): StorageFileInstruments` — returns the lazily-built instrument set. Pass `force: true` to rebuild after installing a new MeterProvider.
|
||||
- `recordOp(input: RecordOpInput): void` — emits the counter increment and histogram observations for one operation.
|
||||
- `resetInstruments(): void` — clears the cache; the next `getInstruments()` rebuilds.
|
||||
- `StorageFileOp` — closed string union enumerating the 14 instrumented operations.
|
||||
- `StorageFileOpOutcome` — `"success" | "error" | "aborted"`.
|
||||
- `RecordOpInput` — `{ op, outcome, durationMs, bytes?, direction? }`.
|
||||
- `StorageFileInstruments` — `{ tracer, meter, readsTotal, writesTotal, deletesTotal, errorsTotal, opDurationMs, ioBytes }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The counters are split by operation kind (reads/writes/deletes) rather than rolled into one total counter so dashboards can compute hit/miss/error rates without joining histograms.
|
||||
- `io_bytes` is recorded under a `direction` attribute (`"read"` or `"write"`) to keep both flows in one histogram while remaining splittable.
|
||||
- `errors_total` counts both `outcome=error` and `outcome=aborted` ops — aborted operations are still failures from the caller's perspective.
|
||||
- `resetInstruments` is exported only because tests need it; production code never calls it.
|
||||
201
packages/persistence/storage/storage-file/src/otel/otel.test.ts
Normal file
201
packages/persistence/storage/storage-file/src/otel/otel.test.ts
Normal file
@@ -0,0 +1,201 @@
|
||||
import type {
|
||||
Counter,
|
||||
Histogram,
|
||||
Meter,
|
||||
MeterProvider,
|
||||
} from "@opentelemetry/api";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import {
|
||||
getInstruments,
|
||||
recordOp,
|
||||
resetInstruments,
|
||||
setStorageFileMeterProvider,
|
||||
} from "./otel.js";
|
||||
|
||||
interface RecordedHistogram {
|
||||
name: string;
|
||||
samples: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
interface RecordedCounter {
|
||||
name: string;
|
||||
additions: { value: number; attrs: Record<string, unknown> | undefined }[];
|
||||
}
|
||||
|
||||
class StubHistogram implements Histogram {
|
||||
public readonly entry: RecordedHistogram;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedHistogram[]
|
||||
) {
|
||||
this.entry = { name, samples: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public record(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.samples.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubCounter implements Counter {
|
||||
public readonly entry: RecordedCounter;
|
||||
public constructor(
|
||||
public readonly name: string,
|
||||
bag: RecordedCounter[]
|
||||
) {
|
||||
this.entry = { name, additions: [] };
|
||||
bag.push(this.entry);
|
||||
}
|
||||
public add(value: number, attrs?: Record<string, unknown>): void {
|
||||
this.entry.additions.push({ value, attrs });
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeter implements Meter {
|
||||
public histograms: RecordedHistogram[] = [];
|
||||
public counters: RecordedCounter[] = [];
|
||||
public createHistogram(name: string): Histogram {
|
||||
return new StubHistogram(name, this.histograms);
|
||||
}
|
||||
public createCounter(name: string): Counter {
|
||||
return new StubCounter(name, this.counters);
|
||||
}
|
||||
public createObservableGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public createObservableUpDownCounter(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
public addBatchObservableCallback(): void {}
|
||||
public removeBatchObservableCallback(): void {}
|
||||
public createGauge(): never {
|
||||
throw new Error("unused");
|
||||
}
|
||||
}
|
||||
|
||||
class StubMeterProvider implements MeterProvider {
|
||||
public meters: StubMeter[] = [];
|
||||
public getMeter(): Meter {
|
||||
const meter = new StubMeter();
|
||||
this.meters.push(meter);
|
||||
return meter;
|
||||
}
|
||||
}
|
||||
|
||||
describe("storage-file otel", () => {
|
||||
let provider: StubMeterProvider;
|
||||
beforeEach(() => {
|
||||
provider = new StubMeterProvider();
|
||||
setStorageFileMeterProvider(provider);
|
||||
});
|
||||
afterEach(() => {
|
||||
setStorageFileMeterProvider(undefined);
|
||||
resetInstruments();
|
||||
});
|
||||
|
||||
function findCounter(name: string): RecordedCounter | undefined {
|
||||
return provider.meters[0]?.counters.find((c) => c.name === name);
|
||||
}
|
||||
function findHistogram(name: string): RecordedHistogram | undefined {
|
||||
return provider.meters[0]?.histograms.find((h) => h.name === name);
|
||||
}
|
||||
|
||||
it("records duration and byte counters on a successful op", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({
|
||||
op: "set",
|
||||
outcome: "success",
|
||||
durationMs: 12,
|
||||
bytes: 100,
|
||||
direction: "write",
|
||||
});
|
||||
const dur = findHistogram("storage_file.op_duration_ms");
|
||||
expect(dur?.samples).toEqual([
|
||||
{ value: 12, attrs: { op: "set", outcome: "success" } },
|
||||
]);
|
||||
const writes = findCounter("storage_file.writes_total");
|
||||
expect(writes?.additions).toEqual([
|
||||
{ value: 1, attrs: { op: "set", outcome: "success" } },
|
||||
]);
|
||||
const io = findHistogram("storage_file.io_bytes");
|
||||
expect(io?.samples).toEqual([
|
||||
{ value: 100, attrs: { op: "set", direction: "write" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("records error counter on a failed op", { timeout: 5000 }, () => {
|
||||
recordOp({ op: "get", outcome: "error", durationMs: 3 });
|
||||
const errs = findCounter("storage_file.errors_total");
|
||||
expect(errs?.additions).toEqual([
|
||||
{ value: 1, attrs: { op: "get", outcome: "error" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("records aborted operations on the errors counter", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({ op: "get", outcome: "aborted", durationMs: 0 });
|
||||
expect(
|
||||
findCounter("storage_file.errors_total")?.additions[0]?.attrs
|
||||
).toEqual({ op: "get", outcome: "aborted" });
|
||||
});
|
||||
|
||||
it("increments reads/writes/deletes counters by op kind", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({ op: "get", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "getPartial", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "getWithMetadata", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "set", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "setPartial", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "delete", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "clear", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "exists", outcome: "success", durationMs: 1 });
|
||||
recordOp({ op: "list", outcome: "success", durationMs: 1 });
|
||||
expect(findCounter("storage_file.reads_total")?.additions).toHaveLength(
|
||||
3
|
||||
);
|
||||
expect(
|
||||
findCounter("storage_file.writes_total")?.additions
|
||||
).toHaveLength(2);
|
||||
expect(
|
||||
findCounter("storage_file.deletes_total")?.additions
|
||||
).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns a cached instrument set on repeat getInstruments calls", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const a = getInstruments();
|
||||
const b = getInstruments();
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("rebuilds the instrument set when getInstruments(true) is called", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const a = getInstruments();
|
||||
const b = getInstruments(true);
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
it("omits the io_bytes record when bytes is undefined", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
recordOp({ op: "list", outcome: "success", durationMs: 5 });
|
||||
expect(
|
||||
findHistogram("storage_file.io_bytes")?.samples ?? []
|
||||
).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
142
packages/persistence/storage/storage-file/src/otel/otel.ts
Normal file
142
packages/persistence/storage/storage-file/src/otel/otel.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
import {
|
||||
type Counter,
|
||||
type Histogram,
|
||||
type Meter,
|
||||
type MeterProvider,
|
||||
metrics,
|
||||
type Tracer,
|
||||
trace,
|
||||
} from "@opentelemetry/api";
|
||||
|
||||
const METER_NAME = "@suaveplan/storage-file";
|
||||
const TRACER_NAME = "@suaveplan/storage-file";
|
||||
|
||||
let overrideMeterProvider: MeterProvider | undefined;
|
||||
|
||||
/**
|
||||
* Override the {@link MeterProvider} used to acquire the storage-file
|
||||
* meter. Pass `undefined` to fall back to the global provider. The
|
||||
* instrument cache is rebuilt automatically on next access.
|
||||
*/
|
||||
export function setStorageFileMeterProvider(
|
||||
provider: MeterProvider | undefined
|
||||
): void {
|
||||
overrideMeterProvider = provider;
|
||||
cachedInstruments = undefined;
|
||||
}
|
||||
|
||||
export type StorageFileOp =
|
||||
| "get"
|
||||
| "set"
|
||||
| "delete"
|
||||
| "exists"
|
||||
| "list"
|
||||
| "clear"
|
||||
| "getMetadata"
|
||||
| "getWithMetadata"
|
||||
| "getSize"
|
||||
| "getPartial"
|
||||
| "setPartial"
|
||||
| "getItemSize"
|
||||
| "createReadStream"
|
||||
| "createWriteStream";
|
||||
|
||||
export type StorageFileOpOutcome = "success" | "error" | "aborted";
|
||||
|
||||
export interface StorageFileInstruments {
|
||||
tracer: Tracer;
|
||||
meter: Meter;
|
||||
readsTotal: Counter;
|
||||
writesTotal: Counter;
|
||||
deletesTotal: Counter;
|
||||
errorsTotal: Counter;
|
||||
opDurationMs: Histogram;
|
||||
ioBytes: Histogram;
|
||||
}
|
||||
|
||||
let cachedInstruments: StorageFileInstruments | undefined;
|
||||
|
||||
/**
|
||||
* Lazily construct the OTel instrument set so consumers can install a
|
||||
* custom global MeterProvider/TracerProvider BEFORE this package's
|
||||
* production code path executes — exactly matches the pattern used by
|
||||
* `@suaveplan/vault` and `@suaveplan/rules`.
|
||||
*
|
||||
* Pass `force: true` to rebuild the instruments against a freshly-set
|
||||
* provider. Tests use this hook after `metrics.setGlobalMeterProvider`.
|
||||
*/
|
||||
export function getInstruments(force = false): StorageFileInstruments {
|
||||
if (cachedInstruments !== undefined && !force) return cachedInstruments;
|
||||
const tracer = trace.getTracer(TRACER_NAME);
|
||||
const meter = (overrideMeterProvider ?? metrics).getMeter(METER_NAME);
|
||||
cachedInstruments = {
|
||||
tracer,
|
||||
meter,
|
||||
readsTotal: meter.createCounter("storage_file.reads_total", {
|
||||
description: "Total successful read operations",
|
||||
}),
|
||||
writesTotal: meter.createCounter("storage_file.writes_total", {
|
||||
description: "Total successful write operations",
|
||||
}),
|
||||
deletesTotal: meter.createCounter("storage_file.deletes_total", {
|
||||
description: "Total successful delete operations",
|
||||
}),
|
||||
errorsTotal: meter.createCounter("storage_file.errors_total", {
|
||||
description: "Total operations that ended in error",
|
||||
}),
|
||||
opDurationMs: meter.createHistogram("storage_file.op_duration_ms", {
|
||||
description: "Per-operation duration in milliseconds",
|
||||
unit: "ms",
|
||||
}),
|
||||
ioBytes: meter.createHistogram("storage_file.io_bytes", {
|
||||
description: "Bytes read or written per operation",
|
||||
unit: "By",
|
||||
}),
|
||||
};
|
||||
return cachedInstruments;
|
||||
}
|
||||
|
||||
export interface RecordOpInput {
|
||||
op: StorageFileOp;
|
||||
outcome: StorageFileOpOutcome;
|
||||
durationMs: number;
|
||||
bytes?: number;
|
||||
direction?: "read" | "write";
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a complete observability record for one operation: increments the
|
||||
* appropriate `{reads,writes,deletes,errors}_total` counter, records a
|
||||
* duration histogram observation, and (when `bytes` is provided) records
|
||||
* an `io_bytes` histogram observation.
|
||||
*/
|
||||
export function recordOp(input: RecordOpInput): void {
|
||||
const { op, outcome, durationMs, bytes, direction } = input;
|
||||
const inst = getInstruments();
|
||||
const attrs = { op, outcome };
|
||||
inst.opDurationMs.record(durationMs, attrs);
|
||||
if (outcome === "error" || outcome === "aborted") {
|
||||
inst.errorsTotal.add(1, { op, outcome });
|
||||
} else if (
|
||||
op === "get" ||
|
||||
op === "getPartial" ||
|
||||
op === "getWithMetadata"
|
||||
) {
|
||||
inst.readsTotal.add(1, { op, outcome });
|
||||
} else if (op === "set" || op === "setPartial") {
|
||||
inst.writesTotal.add(1, { op, outcome });
|
||||
} else if (op === "delete" || op === "clear") {
|
||||
inst.deletesTotal.add(1, { op, outcome });
|
||||
}
|
||||
if (bytes !== undefined && direction !== undefined) {
|
||||
inst.ioBytes.record(bytes, { op, direction });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the cached instrument set. Tests call this between `it(...)`
|
||||
* blocks after installing a fresh in-memory MeterProvider.
|
||||
*/
|
||||
export function resetInstruments(): void {
|
||||
cachedInstruments = undefined;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { registerNodeFsProvider } from "./registry.js";
|
||||
@@ -0,0 +1,49 @@
|
||||
# `registry`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Registers `NodeFsStorage` against the module-augmentation registry maintained by `@suaveplan/storage/factory`. After `registerNodeFsProvider()` runs once, the `createStorage({ type: "file", config: NodeFsStorageOptions })` factory dispatches to `new NodeFsStorage(config)` with full TypeScript narrowing on the `file` slot.
|
||||
|
||||
## Features
|
||||
|
||||
- Registers the `file` provider type with the storage factory.
|
||||
- Idempotent — repeated `registerNodeFsProvider()` calls are no-ops, so the function is safe to call from any consumer's startup path without worrying about double-registration.
|
||||
- Declaration-merges `StorageProviderRegistry` and `StorageProviderConfig` so `createStorage` is fully typed.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { registerNodeFsProvider } from "@suaveplan/storage-file";
|
||||
import { createStorage } from "@suaveplan/storage/factory";
|
||||
|
||||
registerNodeFsProvider();
|
||||
const storage = createStorage({ type: "file", config: { rootDir: "/tmp/store" } });
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { registerNodeFsProvider, NodeFsStorage } from "@suaveplan/storage-file";
|
||||
import { createStorage } from "@suaveplan/storage/factory";
|
||||
|
||||
// Safe in any startup module — idempotent.
|
||||
registerNodeFsProvider();
|
||||
registerNodeFsProvider();
|
||||
|
||||
const storage = createStorage({ type: "file", config: { rootDir: "/var/lib/app/storage", retry: { maxAttempts: 5 } } });
|
||||
console.log(storage instanceof NodeFsStorage); // → true
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `registerNodeFsProvider(): void` — register the `file` provider. Idempotent.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The registration delegates to `registerStorageProvider("file", NodeFsStorage)` from `@suaveplan/storage/factory`. The cast through `unknown` is required because the factory's loose `ProviderCtor` signature accepts `unknown` for config — `NodeFsStorage` accepts a stricter type but is structurally compatible.
|
||||
- Registration is NOT triggered on import — consumers must call `registerNodeFsProvider()` explicitly. This keeps `import "@suaveplan/storage-file"` side-effect-free per `"sideEffects": false`.
|
||||
- The module-augmentation `declare module "@suaveplan/storage/factory"` is type-only — it does not require the consumer to import this file at runtime to benefit from the typing.
|
||||
@@ -0,0 +1,52 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
clearStorageProviders,
|
||||
createStorage,
|
||||
hasStorageProvider,
|
||||
} from "@suaveplan/storage/factory";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { NodeFsStorage } from "../node-fs-storage/node-fs-storage.js";
|
||||
import { registerNodeFsProvider } from "./registry.js";
|
||||
|
||||
describe("registerNodeFsProvider", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "nfs-registry-"));
|
||||
clearStorageProviders();
|
||||
});
|
||||
afterEach(async () => {
|
||||
clearStorageProviders();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("registers the file provider and resolves it via createStorage", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
registerNodeFsProvider();
|
||||
expect(hasStorageProvider("file")).toBe(true);
|
||||
const storage = createStorage({
|
||||
type: "file" as never,
|
||||
config: { rootDir: root } as never,
|
||||
});
|
||||
expect(storage).toBeInstanceOf(NodeFsStorage);
|
||||
await storage.set("k", new Uint8Array([1]));
|
||||
expect(await storage.get("k")).toEqual(new Uint8Array([1]));
|
||||
});
|
||||
|
||||
it("is idempotent across repeated registerNodeFsProvider calls", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
registerNodeFsProvider();
|
||||
registerNodeFsProvider();
|
||||
registerNodeFsProvider();
|
||||
expect(hasStorageProvider("file")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import {
|
||||
hasStorageProvider,
|
||||
registerStorageProvider,
|
||||
} from "@suaveplan/storage/factory";
|
||||
import { NodeFsStorage } from "../node-fs-storage/node-fs-storage.js";
|
||||
import type { NodeFsStorageOptions } from "../types/node-fs-storage.types.js";
|
||||
|
||||
declare module "@suaveplan/storage/factory" {
|
||||
interface StorageProviderRegistry {
|
||||
file: NodeFsStorage;
|
||||
}
|
||||
interface StorageProviderConfig {
|
||||
file: NodeFsStorageOptions;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register `NodeFsStorage` against the `@suaveplan/storage` factory under
|
||||
* the canonical type slot `"file"`. Idempotent — repeated calls do nothing
|
||||
* once the slot is occupied, and re-register cleanly after a downstream
|
||||
* `clearStorageProviders()` call.
|
||||
*/
|
||||
export function registerNodeFsProvider(): void {
|
||||
if (hasStorageProvider("file")) return;
|
||||
registerStorageProvider(
|
||||
"file",
|
||||
NodeFsStorage as unknown as new (
|
||||
config?: unknown
|
||||
) => NodeFsStorage
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export type { WithFsRetryOptions } from "./retry-helper.js";
|
||||
export {
|
||||
isPermissionFsError,
|
||||
isTransientFsError,
|
||||
withFsRetry,
|
||||
} from "./retry-helper.js";
|
||||
@@ -0,0 +1,60 @@
|
||||
# `retry-helper`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Wraps `@suaveplan/retry.retry` with the transient-error classifier specific to filesystem operations and the error-mapping policy specific to this package. Every public `NodeFsStorage` operation that performs IO routes through `withFsRetry` so the retry policy lives in one place. Tests can drive the retry loop deterministically by injecting a `TimersLike` pair (typically the FakeClock-backed pair from `@suaveplan/testing`).
|
||||
|
||||
## Features
|
||||
|
||||
- Classifies `EBUSY`/`EMFILE`/`ENFILE`/`EAGAIN` as transient; every other errno is fatal.
|
||||
- Maps `EACCES`/`EPERM` to `STORAGE_FILE.PERMISSION_DENIED` after exhausting retries.
|
||||
- Maps any other terminal error to `STORAGE_FILE.IO_FAILED` and preserves `cause`.
|
||||
- Lets a previously-thrown `StorageFileError` bubble through unwrapped so error codes don't get clobbered.
|
||||
- Defaults to `globalThis.setTimeout`/`clearTimeout` but accepts an injectable `TimersLike` pair for deterministic timing.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { withFsRetry } from "@suaveplan/storage-file";
|
||||
|
||||
const data = await withFsRetry(() => fs.readFile("/tmp/store/k.data"));
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { withFsRetry, type WithFsRetryOptions } from "@suaveplan/storage-file";
|
||||
import { fakeClock } from "@suaveplan/testing/runner";
|
||||
|
||||
const fc = fakeClock();
|
||||
const options: WithFsRetryOptions = {
|
||||
retry: { maxAttempts: 5, baseDelayMs: 10, maxDelayMs: 200 },
|
||||
timers: { setTimer: fc.setTimeout, clearTimer: fc.clearTimeout },
|
||||
signal: controller.signal,
|
||||
key: "k",
|
||||
};
|
||||
|
||||
await withFsRetry(async () => {
|
||||
throw Object.assign(new Error("EBUSY"), { code: "EBUSY" });
|
||||
}, options).catch((err) => {
|
||||
// → StorageFileError(STORAGE_FILE.IO_FAILED, cause: { code: "EBUSY" })
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `withFsRetry<T>(fn, options?): Promise<T>` — runs `fn` with retry on transient fs errors. Throws `StorageFileError` with `IO_FAILED` or `PERMISSION_DENIED` after exhaustion.
|
||||
- `WithFsRetryOptions` — `{ retry?, timers?, signal?, key? }`. `retry.maxAttempts` default 3, `retry.baseDelayMs` default 50, `retry.maxDelayMs` default 5000.
|
||||
- `isTransientFsError(err): boolean` — true when `err.code` is one of `EBUSY`/`EMFILE`/`ENFILE`/`EAGAIN`.
|
||||
- `isPermissionFsError(err): boolean` — true when `err.code` is `EACCES` or `EPERM`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The exponential backoff is built via `createExponentialBackoff` from `@suaveplan/retry`. The jitter policy is the package default (equal jitter).
|
||||
- `signal` is forwarded to the inner `retry` call so an abort surfaces as a `RetryAbortedError` from the retry package; the outer `catch` rewraps it via the IO_FAILED path.
|
||||
- `key` (when provided) is propagated into the rethrown `StorageFileError` so OTel attribute extraction works downstream without additional context plumbing.
|
||||
- Pre-existing `StorageFileError` instances are NOT rewrapped — this keeps a `STORAGE_FILE.INVALID_KEY` raised inside `fn` visible as `INVALID_KEY` at the call site, not as a meaningless `IO_FAILED` wrapper.
|
||||
@@ -0,0 +1,285 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import {
|
||||
isPermissionFsError,
|
||||
isTransientFsError,
|
||||
withFsRetry,
|
||||
} from "./retry-helper.js";
|
||||
|
||||
function errnoError(code: string): Error & { code: string } {
|
||||
const err = new Error(code) as Error & { code: string };
|
||||
err.code = code;
|
||||
return err;
|
||||
}
|
||||
|
||||
// Trivial in-process timers — execute callbacks immediately so retries don't block.
|
||||
const immediateTimers: TimersLike = {
|
||||
setTimer(cb) {
|
||||
queueMicrotask(cb);
|
||||
return 0;
|
||||
},
|
||||
clearTimer() {
|
||||
/* no-op */
|
||||
},
|
||||
};
|
||||
|
||||
describe("isTransientFsError", () => {
|
||||
it("classifies the documented transient codes", { timeout: 5000 }, () => {
|
||||
for (const code of ["EBUSY", "EMFILE", "ENFILE", "EAGAIN"]) {
|
||||
expect(isTransientFsError(errnoError(code))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-transient errnos and non-error inputs", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(isTransientFsError(errnoError("ENOENT"))).toBe(false);
|
||||
expect(isTransientFsError(errnoError("EACCES"))).toBe(false);
|
||||
expect(isTransientFsError(null)).toBe(false);
|
||||
expect(isTransientFsError(undefined)).toBe(false);
|
||||
expect(isTransientFsError("plain string")).toBe(false);
|
||||
expect(isTransientFsError({})).toBe(false);
|
||||
expect(isTransientFsError({ code: 42 })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isPermissionFsError", () => {
|
||||
it("classifies EACCES and EPERM as permission errors", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(isPermissionFsError(errnoError("EACCES"))).toBe(true);
|
||||
expect(isPermissionFsError(errnoError("EPERM"))).toBe(true);
|
||||
expect(isPermissionFsError(errnoError("ENOENT"))).toBe(false);
|
||||
expect(isPermissionFsError(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("withFsRetry", () => {
|
||||
it("retries on EBUSY and resolves on success", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let attempts = 0;
|
||||
const result = await withFsRetry(
|
||||
async () => {
|
||||
attempts++;
|
||||
if (attempts < 2) throw errnoError("EBUSY");
|
||||
return "ok";
|
||||
},
|
||||
{
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 },
|
||||
timers: immediateTimers,
|
||||
}
|
||||
);
|
||||
expect(result).toBe("ok");
|
||||
expect(attempts).toBe(2);
|
||||
});
|
||||
|
||||
it("does not retry on EACCES", { timeout: 5000 }, async () => {
|
||||
let attempts = 0;
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
attempts++;
|
||||
throw errnoError("EACCES");
|
||||
},
|
||||
{
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 },
|
||||
timers: immediateTimers,
|
||||
key: "x",
|
||||
}
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(StorageFileError);
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.PERMISSION_DENIED
|
||||
);
|
||||
expect((err as StorageFileError).key).toBe("x");
|
||||
}
|
||||
expect(attempts).toBe(1);
|
||||
});
|
||||
|
||||
it("exhausts retries on persistent EBUSY and surfaces IO_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let attempts = 0;
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
attempts++;
|
||||
throw errnoError("EBUSY");
|
||||
},
|
||||
{
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 },
|
||||
timers: immediateTimers,
|
||||
key: "y",
|
||||
}
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
expect((err as StorageFileError).key).toBe("y");
|
||||
}
|
||||
expect(attempts).toBe(3);
|
||||
});
|
||||
|
||||
it("propagates pre-existing StorageFileError unwrapped", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const original = new StorageFileError(
|
||||
STORAGE_FILE_CODES.INVALID_KEY,
|
||||
"bad"
|
||||
);
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
throw original;
|
||||
},
|
||||
{ timers: immediateTimers }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBe(original);
|
||||
}
|
||||
});
|
||||
|
||||
it("uses default settings when no options are supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const value = await withFsRetry(async () => 42);
|
||||
expect(value).toBe(42);
|
||||
});
|
||||
|
||||
it("wraps non-errno exceptions as IO_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
throw new Error("generic");
|
||||
},
|
||||
{ timers: immediateTimers }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards an externally-supplied AbortSignal to the retry loop", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
try {
|
||||
await withFsRetry(async () => "x", {
|
||||
signal: controller.signal,
|
||||
timers: immediateTimers,
|
||||
});
|
||||
expect.fail("expected abort");
|
||||
} catch (err) {
|
||||
// Aborted before fn runs → RetryAbortedError surface; outer catch wraps.
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
}
|
||||
});
|
||||
|
||||
it("maps EACCES to PERMISSION_DENIED without a key when none supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
throw errnoError("EACCES");
|
||||
},
|
||||
{ timers: immediateTimers }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.PERMISSION_DENIED
|
||||
);
|
||||
expect((err as StorageFileError).key).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps exhausted retries without a key when none supplied", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
throw errnoError("EBUSY");
|
||||
},
|
||||
{
|
||||
retry: { maxAttempts: 2, baseDelayMs: 1, maxDelayMs: 2 },
|
||||
timers: immediateTimers,
|
||||
}
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
expect((err as StorageFileError).key).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps non-Error throwables as IO_FAILED", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
try {
|
||||
await withFsRetry(
|
||||
async () => {
|
||||
const literal: unknown = "literal-throwable";
|
||||
throw literal;
|
||||
},
|
||||
{ timers: immediateTimers }
|
||||
);
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves using the default global timers when none are injected", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
let attempts = 0;
|
||||
const result = await withFsRetry(
|
||||
async () => {
|
||||
attempts++;
|
||||
if (attempts === 1) throw errnoError("EBUSY");
|
||||
return "ok";
|
||||
},
|
||||
{ retry: { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 } }
|
||||
);
|
||||
expect(result).toBe("ok");
|
||||
});
|
||||
|
||||
it("calls the default clearTimer when a retry is aborted mid-sleep", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const controller = new AbortController();
|
||||
const pending = withFsRetry(
|
||||
async () => {
|
||||
// Schedule the abort 5ms in the future — long enough that the
|
||||
// retry sleep registers a real default-timers setTimeout, but
|
||||
// short enough to land before the sleep completes (50ms base).
|
||||
setTimeout(() => controller.abort(), 5);
|
||||
throw errnoError("EBUSY");
|
||||
},
|
||||
{
|
||||
signal: controller.signal,
|
||||
retry: { maxAttempts: 5, baseDelayMs: 50, maxDelayMs: 200 },
|
||||
}
|
||||
);
|
||||
await pending.catch(() => undefined);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
import { createExponentialBackoff, retry } from "@suaveplan/retry";
|
||||
import type { TimersLike } from "@suaveplan/types";
|
||||
import {
|
||||
DEFAULT_RETRY_BASE_DELAY_MS,
|
||||
DEFAULT_RETRY_MAX_ATTEMPTS,
|
||||
DEFAULT_RETRY_MAX_DELAY_MS,
|
||||
TRANSIENT_ERRNO_CODES,
|
||||
} from "../constants/index.js";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import type { NodeFsStorageRetryOptions } from "../types/node-fs-storage.types.js";
|
||||
|
||||
interface ErrnoCarrier {
|
||||
code?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when `err` carries an errno string in the documented transient
|
||||
* set (`EBUSY`, `EMFILE`, `ENFILE`, `EAGAIN`). Anything else — including
|
||||
* `ENOENT`, `EACCES`, `EISDIR`, `EXDEV` — is considered fatal.
|
||||
*/
|
||||
export function isTransientFsError(err: unknown): boolean {
|
||||
if (err === null || typeof err !== "object") return false;
|
||||
const code = (err as ErrnoCarrier).code;
|
||||
if (typeof code !== "string") return false;
|
||||
return TRANSIENT_ERRNO_CODES.has(code);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when `err` carries an errno that should surface as a
|
||||
* permission-denied error to the caller.
|
||||
*/
|
||||
export function isPermissionFsError(err: unknown): boolean {
|
||||
if (err === null || typeof err !== "object") return false;
|
||||
const code = (err as ErrnoCarrier).code;
|
||||
return code === "EACCES" || code === "EPERM";
|
||||
}
|
||||
|
||||
export interface WithFsRetryOptions {
|
||||
retry?: NodeFsStorageRetryOptions;
|
||||
timers?: TimersLike;
|
||||
signal?: AbortSignal;
|
||||
key?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `fn` with exponential-backoff retry on transient filesystem errors.
|
||||
*
|
||||
* Transient classification is delegated to {@link isTransientFsError}.
|
||||
* Non-transient errors are re-thrown immediately. After `maxAttempts` the
|
||||
* last error surfaces as a `StorageFileError` with code `IO_FAILED`
|
||||
* (or `PERMISSION_DENIED` when the underlying errno classifies as such).
|
||||
*
|
||||
* Timing is fully injectable via `options.timers` — defaults to
|
||||
* `globalThis.setTimeout`/`clearTimeout` for production use.
|
||||
*/
|
||||
export async function withFsRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: WithFsRetryOptions = {}
|
||||
): Promise<T> {
|
||||
const maxAttempts =
|
||||
options.retry?.maxAttempts ?? DEFAULT_RETRY_MAX_ATTEMPTS;
|
||||
const baseDelayMs =
|
||||
options.retry?.baseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS;
|
||||
const maxDelayMs = options.retry?.maxDelayMs ?? DEFAULT_RETRY_MAX_DELAY_MS;
|
||||
const delay = createExponentialBackoff({
|
||||
initial: baseDelayMs,
|
||||
max: maxDelayMs,
|
||||
});
|
||||
const defaultTimers: TimersLike = {
|
||||
setTimer: (handler, ms) => Number(globalThis.setTimeout(handler, ms)),
|
||||
clearTimer: (id) => globalThis.clearTimeout(id),
|
||||
};
|
||||
const timers = options.timers ?? defaultTimers;
|
||||
let lastError: unknown;
|
||||
try {
|
||||
const result = await retry(
|
||||
async () => {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
{
|
||||
maxAttempts,
|
||||
delay,
|
||||
shouldRetry: (err) => isTransientFsError(err),
|
||||
timers,
|
||||
...(options.signal !== undefined
|
||||
? { signal: options.signal }
|
||||
: {}),
|
||||
}
|
||||
);
|
||||
return result.value;
|
||||
} catch (err) {
|
||||
const cause = (lastError ?? err) as Error;
|
||||
if (cause instanceof StorageFileError) {
|
||||
throw cause;
|
||||
}
|
||||
if (isPermissionFsError(cause)) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.PERMISSION_DENIED,
|
||||
`Permission denied: ${cause.message}`,
|
||||
options.key !== undefined
|
||||
? { key: options.key, cause }
|
||||
: { cause }
|
||||
);
|
||||
}
|
||||
// Pass through expected errnos (ENOENT, EEXIST, EISDIR, EXDEV) so callers
|
||||
// can map them to domain semantics; only wrap when retries were exhausted
|
||||
// against a transient error or when the cause is unrecognised.
|
||||
const code = (cause as NodeJS.ErrnoException).code;
|
||||
if (typeof code === "string" && !TRANSIENT_ERRNO_CODES.has(code)) {
|
||||
throw cause;
|
||||
}
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.IO_FAILED,
|
||||
`Filesystem operation failed: ${cause.message ?? String(cause)}`,
|
||||
options.key !== undefined ? { key: options.key, cause } : { cause }
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export type {
|
||||
CreateReadStreamArgs,
|
||||
CreateWriteStreamArgs,
|
||||
} from "./streams.js";
|
||||
export {
|
||||
createNodeReadStream,
|
||||
createNodeWriteStream,
|
||||
} from "./streams.js";
|
||||
@@ -0,0 +1,63 @@
|
||||
# `streams`
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
Bridges `node:fs` Readable/Writable streams to Web `ReadableStream`/`WritableStream` for use by `NodeFsStorage.createReadStream`/`createWriteStream`. The bridge threads AbortSignal end-to-end and applies the same temp-file-then-rename atomic-write policy as the buffered `set` path, so a large streamed write never leaves a partial file at the destination path.
|
||||
|
||||
## Features
|
||||
|
||||
- `createNodeReadStream` returns a `ReadableStream<Uint8Array>` backed by `fs.createReadStream`. Forwards `data`/`end`/`error` events; surfaces both transport failures and abort signals as `StorageFileError`.
|
||||
- `createNodeWriteStream` returns a `WritableStream<Uint8Array>` backed by `fs.createWriteStream` writing to a caller-supplied `tempPath`. On `close` it `fs.rename`s onto `destinationPath`. On abort or error it unlinks the temp file.
|
||||
- Both bridges destroy the underlying Node stream on cancel/abort so file descriptors don't leak. On the read path, `cancel()` also installs a no-op error listener on the inner Node stream before destroying it — when cancel races an in-flight `open()`, Node may still emit a late `"error"` (typically ENOENT if the file was removed between creation and cancel) after our own error handler has been torn down. Without the absorber that error would surface as an unhandled exception and could crash a vitest worker.
|
||||
- The temp-path validation enforces the canonical `.tmp-` prefix at boundary entry — passing a malformed path surfaces as `STORAGE_FILE.ATOMIC_WRITE_FAILED` before any IO.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { createNodeReadStream, defaultNodeFs } from "@suaveplan/storage-file";
|
||||
|
||||
const fs = defaultNodeFs();
|
||||
const reader = createNodeReadStream({ fs, path: "/tmp/store/large.data", key: "large" });
|
||||
for await (const chunk of reader) {
|
||||
console.log(chunk.byteLength);
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { createNodeWriteStream, defaultNodeFs } from "@suaveplan/storage-file";
|
||||
|
||||
const fs = defaultNodeFs();
|
||||
const controller = new AbortController();
|
||||
const writer = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: "/tmp/store/k.data",
|
||||
tempPath: "/tmp/store/.tmp-k.data",
|
||||
signal: controller.signal,
|
||||
key: "k",
|
||||
});
|
||||
|
||||
const sink = writer.getWriter();
|
||||
await sink.write(new Uint8Array(1024 * 1024));
|
||||
await sink.write(new Uint8Array(1024 * 1024));
|
||||
await sink.close();
|
||||
// → /tmp/store/k.data now contains 2 MiB; no .tmp-k.data remains.
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `createNodeReadStream({ fs, path, signal?, key }): ReadableStream<Uint8Array>` — Web ReadableStream over `node:fs.createReadStream(path)`.
|
||||
- `createNodeWriteStream({ fs, destinationPath, tempPath, signal?, key }): WritableStream<Uint8Array>` — Web WritableStream that writes to `tempPath` and renames onto `destinationPath` on close. Throws `STORAGE_FILE.ATOMIC_WRITE_FAILED` if `tempPath` does not contain the canonical temp prefix.
|
||||
- `CreateReadStreamArgs`, `CreateWriteStreamArgs` — explicit interface bags for the two factories.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The read bridge copies each chunk into a fresh `Uint8Array(buffer, byteOffset, byteLength)` view so consumers don't accidentally retain a reference to Node's Buffer pool.
|
||||
- The write bridge installs the abort listener inside `start` and removes it in `close`/`abort` — both branches are exercised by the test suite.
|
||||
- `abort()` on the WritableStream is best-effort: the temp file unlink is `.catch(() => undefined)` so we never throw an abort error and lose the cleanup signal.
|
||||
- On a successful `close` the temp file is renamed atomically. On a failed rename the temp file is removed and an `ATOMIC_WRITE_FAILED` surfaces — never both an orphaned temp and an incomplete destination.
|
||||
@@ -0,0 +1,532 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
afterEach,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
} from "@suaveplan/testing/runner";
|
||||
import { TEMP_FILE_PREFIX } from "../constants/index.js";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import { defaultNodeFs } from "../node-fs/node-fs.js";
|
||||
import { createNodeReadStream, createNodeWriteStream } from "./streams.js";
|
||||
|
||||
const fs = defaultNodeFs();
|
||||
|
||||
describe("createNodeReadStream", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "storage-file-streams-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns a ReadableStream that yields the file contents", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const path = join(root, "k.data");
|
||||
await writeFile(path, new TextEncoder().encode("hello"));
|
||||
const r = createNodeReadStream({ fs, path, key: "k" });
|
||||
const reader = r.getReader();
|
||||
const out: Uint8Array[] = [];
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
out.push(value);
|
||||
}
|
||||
const combined = Buffer.concat(out);
|
||||
expect(new TextDecoder().decode(new Uint8Array(combined))).toBe(
|
||||
"hello"
|
||||
);
|
||||
});
|
||||
|
||||
it("errors the stream when the file does not exist", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const r = createNodeReadStream({
|
||||
fs,
|
||||
path: join(root, "missing.data"),
|
||||
key: "missing",
|
||||
});
|
||||
const reader = r.getReader();
|
||||
try {
|
||||
await reader.read();
|
||||
expect.fail("expected error");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("errors immediately when the signal is pre-aborted", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const path = join(root, "k.data");
|
||||
await writeFile(path, new TextEncoder().encode("hi"));
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const r = createNodeReadStream({
|
||||
fs,
|
||||
path,
|
||||
key: "k",
|
||||
signal: controller.signal,
|
||||
});
|
||||
try {
|
||||
await r.getReader().read();
|
||||
expect.fail("expected error");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels the underlying node stream when the consumer cancels", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const path = join(root, "big.data");
|
||||
await writeFile(path, new Uint8Array(1024 * 1024));
|
||||
const r = createNodeReadStream({ fs, path, key: "big" });
|
||||
const reader = r.getReader();
|
||||
await reader.cancel();
|
||||
});
|
||||
|
||||
it("swallows late errors from the inner stream after cancel", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const { Readable } = await import("node:stream");
|
||||
const inner = new Readable({ read() {} });
|
||||
const fakeFs = {
|
||||
...fs,
|
||||
createReadStream: () => inner,
|
||||
} as typeof fs;
|
||||
const r = createNodeReadStream({
|
||||
fs: fakeFs,
|
||||
path: "ignored",
|
||||
key: "k",
|
||||
});
|
||||
const reader = r.getReader();
|
||||
await reader.cancel();
|
||||
// Emit an error after cancel + cleanup have run — without the
|
||||
// post-cancel error absorber this would become an unhandled exception.
|
||||
inner.emit("error", new Error("late ENOENT"));
|
||||
// Yield a microtask so any unhandled rejection has time to surface.
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
});
|
||||
});
|
||||
|
||||
describe("createNodeWriteStream", () => {
|
||||
let root: string;
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), "storage-file-streams-w-"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("round-trips a 1 MiB payload", { timeout: 15000 }, async () => {
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}out.data`);
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
const payload = new Uint8Array(1024 * 1024).fill(0x42);
|
||||
await writer.write(payload);
|
||||
await writer.close();
|
||||
const back = await readFile(destination);
|
||||
expect(back.byteLength).toBe(1024 * 1024);
|
||||
expect(back[1023]).toBe(0x42);
|
||||
expect(existsSync(tempPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("throws when tempPath does not contain the canonical prefix", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() =>
|
||||
createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: join(root, "x.data"),
|
||||
tempPath: join(root, "plain"),
|
||||
key: "x",
|
||||
})
|
||||
).toThrow(StorageFileError);
|
||||
});
|
||||
|
||||
it("errors immediately when the signal is pre-aborted", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}out.data`);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
signal: controller.signal,
|
||||
});
|
||||
try {
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
expect.fail("expected error");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("aborts mid-write and removes the temp file", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}abort.data`);
|
||||
const controller = new AbortController();
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
signal: controller.signal,
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array(1024));
|
||||
controller.abort();
|
||||
try {
|
||||
await writer.close();
|
||||
expect.fail("expected close to throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
expect(existsSync(tempPath)).toBe(false);
|
||||
expect(existsSync(destination)).toBe(false);
|
||||
});
|
||||
|
||||
it("swallows unlink failures during abort cleanup", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const failingUnlinkFs = {
|
||||
...fs,
|
||||
async unlink() {
|
||||
throw Object.assign(new Error("io"), { code: "EIO" });
|
||||
},
|
||||
};
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}fail-unlink.data`);
|
||||
const controller = new AbortController();
|
||||
const w = createNodeWriteStream({
|
||||
fs: failingUnlinkFs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
signal: controller.signal,
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
controller.abort();
|
||||
try {
|
||||
await writer.close();
|
||||
} catch {
|
||||
// expected
|
||||
}
|
||||
// Calling .abort() through the writer also exercises the abort cleanup
|
||||
const w2 = createNodeWriteStream({
|
||||
fs: failingUnlinkFs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer2 = w2.getWriter();
|
||||
await writer2.abort();
|
||||
});
|
||||
|
||||
it("swallows unlink failures during atomic rename rollback", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const failingFs = {
|
||||
...fs,
|
||||
async rename() {
|
||||
throw Object.assign(new Error("xdev"), { code: "EXDEV" });
|
||||
},
|
||||
async unlink() {
|
||||
throw Object.assign(new Error("io"), { code: "EIO" });
|
||||
},
|
||||
};
|
||||
const destination = join(root, "child", "out.data"); // parent missing → rename fails
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}rollback.data`);
|
||||
const w = createNodeWriteStream({
|
||||
fs: failingFs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
try {
|
||||
await writer.close();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ATOMIC_WRITE_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("close is a no-op when the stream was never initialised (pre-aborted)", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}preab.data`);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
signal: controller.signal,
|
||||
});
|
||||
// Getting the writer triggers start() which errored — but close should still resolve.
|
||||
try {
|
||||
const writer = w.getWriter();
|
||||
await writer.close();
|
||||
} catch {
|
||||
// Either resolves or rejects with ABORTED — both acceptable; coverage only.
|
||||
}
|
||||
});
|
||||
|
||||
it("throws ABORTED when write is called after the signal aborts", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}late-write.data`);
|
||||
const controller = new AbortController();
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
signal: controller.signal,
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
controller.abort();
|
||||
try {
|
||||
await writer.write(new Uint8Array([2]));
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans up the temp file when the writable is aborted", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}aborted.data`);
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array(64));
|
||||
await writer.abort(new Error("caller abort"));
|
||||
expect(existsSync(tempPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("surfaces an error reported by the underlying stream end callback", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const { Writable } = await import("node:stream");
|
||||
const fakeFs = {
|
||||
...fs,
|
||||
createWriteStream() {
|
||||
return new Writable({
|
||||
write(_chunk, _enc, cb) {
|
||||
cb();
|
||||
},
|
||||
final(cb) {
|
||||
cb(new Error("end-fail"));
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}endfail.data`);
|
||||
const w = createNodeWriteStream({
|
||||
fs: fakeFs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
try {
|
||||
await writer.close();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("errors when write fails after start", { timeout: 10000 }, async () => {
|
||||
// Use a hooked fs where the writable stream emits an error during write.
|
||||
const { Writable } = await import("node:stream");
|
||||
const fakeFs = {
|
||||
...fs,
|
||||
createWriteStream() {
|
||||
return new Writable({
|
||||
write(_chunk, _enc, cb) {
|
||||
cb(new Error("boom"));
|
||||
},
|
||||
});
|
||||
},
|
||||
};
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}fail.data`);
|
||||
const w = createNodeWriteStream({
|
||||
fs: fakeFs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
try {
|
||||
await writer.write(new Uint8Array([1]));
|
||||
expect.fail("expected error");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("propagates abort that fires after end() but before rename", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
// Schedule the abort to fire on the microtask queue between end() and rename().
|
||||
const destination = join(root, "out.data");
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}late.data`);
|
||||
const controller = new AbortController();
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
signal: controller.signal,
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array([1]));
|
||||
// Schedule abort to fire concurrently with close.
|
||||
queueMicrotask(() => controller.abort());
|
||||
try {
|
||||
await writer.close();
|
||||
} catch (err) {
|
||||
// Either ABORTED (caught in close pre-end) or normal close — both acceptable; coverage only.
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces a read stream error from the underlying Node stream", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const { Readable } = await import("node:stream");
|
||||
const fakeFs = {
|
||||
...fs,
|
||||
createReadStream() {
|
||||
const r = new Readable({ read() {} });
|
||||
queueMicrotask(() => r.emit("error", new Error("read-fail")));
|
||||
return r;
|
||||
},
|
||||
};
|
||||
const r = createNodeReadStream({
|
||||
fs: fakeFs,
|
||||
path: join(root, "any.data"),
|
||||
key: "any",
|
||||
});
|
||||
try {
|
||||
const reader = r.getReader();
|
||||
while (true) {
|
||||
const { done } = await reader.read();
|
||||
if (done) break;
|
||||
}
|
||||
expect.fail("expected error");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.IO_FAILED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("aborts the read stream when the signal fires during reading", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const { Readable } = await import("node:stream");
|
||||
const fakeFs = {
|
||||
...fs,
|
||||
createReadStream() {
|
||||
// Slow reader — never emits data; allows abort to land mid-read.
|
||||
return new Readable({ read() {} });
|
||||
},
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const r = createNodeReadStream({
|
||||
fs: fakeFs,
|
||||
path: join(root, "x.data"),
|
||||
key: "x",
|
||||
signal: controller.signal,
|
||||
});
|
||||
const reader = r.getReader();
|
||||
queueMicrotask(() => controller.abort());
|
||||
try {
|
||||
await reader.read();
|
||||
expect.fail("expected abort");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ABORTED
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("surfaces ATOMIC_WRITE_FAILED when rename fails", {
|
||||
timeout: 10000,
|
||||
}, async () => {
|
||||
const destination = join(root, "child", "out.data"); // parent doesn't exist
|
||||
const tempPath = join(root, `${TEMP_FILE_PREFIX}out.data`);
|
||||
const w = createNodeWriteStream({
|
||||
fs,
|
||||
destinationPath: destination,
|
||||
tempPath,
|
||||
key: "out",
|
||||
});
|
||||
const writer = w.getWriter();
|
||||
await writer.write(new Uint8Array(8));
|
||||
try {
|
||||
await writer.close();
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect((err as StorageFileError).code).toBe(
|
||||
STORAGE_FILE_CODES.ATOMIC_WRITE_FAILED
|
||||
);
|
||||
}
|
||||
expect(existsSync(tempPath)).toBe(false);
|
||||
});
|
||||
});
|
||||
183
packages/persistence/storage/storage-file/src/streams/streams.ts
Normal file
183
packages/persistence/storage/storage-file/src/streams/streams.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import type { Readable, Writable } from "node:stream";
|
||||
import { TEMP_FILE_PREFIX } from "../constants/index.js";
|
||||
import { STORAGE_FILE_CODES, StorageFileError } from "../error-codes.js";
|
||||
import type { NodeFsLike } from "../types/fs-adapter.types.js";
|
||||
|
||||
export interface CreateReadStreamArgs {
|
||||
fs: NodeFsLike;
|
||||
path: string;
|
||||
signal?: AbortSignal;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export interface CreateWriteStreamArgs {
|
||||
fs: NodeFsLike;
|
||||
destinationPath: string;
|
||||
tempPath: string;
|
||||
signal?: AbortSignal;
|
||||
key: string;
|
||||
}
|
||||
|
||||
function abortError(key: string): StorageFileError {
|
||||
return new StorageFileError(
|
||||
STORAGE_FILE_CODES.ABORTED,
|
||||
"Operation aborted",
|
||||
{
|
||||
key,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function ioError(key: string, cause: Error): StorageFileError {
|
||||
return new StorageFileError(
|
||||
STORAGE_FILE_CODES.IO_FAILED,
|
||||
`Stream failed: ${cause.message}`,
|
||||
{ key, cause }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a `node:fs` Readable into a Web ReadableStream. Aborts the underlying
|
||||
* Node stream when `signal` fires; surfaces structured `StorageFileError`
|
||||
* for both transport and abort failures.
|
||||
*/
|
||||
export function createNodeReadStream(
|
||||
args: CreateReadStreamArgs
|
||||
): ReadableStream<Uint8Array> {
|
||||
const { fs, path, signal, key } = args;
|
||||
let inner: Readable | undefined;
|
||||
let cleanup: (() => void) | undefined;
|
||||
return new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
if (signal?.aborted) {
|
||||
controller.error(abortError(key));
|
||||
return;
|
||||
}
|
||||
inner = fs.createReadStream(path);
|
||||
const onData = (chunk: unknown): void => {
|
||||
const buf = chunk as Buffer;
|
||||
controller.enqueue(
|
||||
new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
|
||||
);
|
||||
};
|
||||
const onEnd = (): void => {
|
||||
controller.close();
|
||||
cleanup?.();
|
||||
};
|
||||
const onError = (err: Error): void => {
|
||||
controller.error(ioError(key, err));
|
||||
cleanup?.();
|
||||
};
|
||||
const onAbort = (): void => {
|
||||
inner?.destroy();
|
||||
controller.error(abortError(key));
|
||||
cleanup?.();
|
||||
};
|
||||
inner.on("data", onData);
|
||||
inner.on("end", onEnd);
|
||||
inner.on("error", onError);
|
||||
signal?.addEventListener("abort", onAbort);
|
||||
cleanup = () => {
|
||||
inner?.off("data", onData);
|
||||
inner?.off("end", onEnd);
|
||||
inner?.off("error", onError);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
};
|
||||
},
|
||||
cancel() {
|
||||
// Absorb late-arriving errors from the underlying stream: when
|
||||
// cancel runs before the async open() completes, Node may still
|
||||
// emit "error" (e.g. ENOENT if the file was removed between
|
||||
// creation and cancel). Our `onError` listener is torn down by
|
||||
// `cleanup()` below, so without this absorber that error would
|
||||
// surface as an unhandled exception and crash the test worker.
|
||||
inner?.on("error", () => undefined);
|
||||
inner?.destroy();
|
||||
cleanup?.();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap a `node:fs` Writable into a Web WritableStream. Writes to a temp file
|
||||
* (`tempPath`) and renames onto `destinationPath` on close. On abort or
|
||||
* underlying-stream error the temp file is unlinked.
|
||||
*/
|
||||
export function createNodeWriteStream(
|
||||
args: CreateWriteStreamArgs
|
||||
): WritableStream<Uint8Array> {
|
||||
const { fs, destinationPath, tempPath, signal, key } = args;
|
||||
if (!tempPath.includes(TEMP_FILE_PREFIX)) {
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.ATOMIC_WRITE_FAILED,
|
||||
"tempPath must contain the canonical temp prefix",
|
||||
{ key }
|
||||
);
|
||||
}
|
||||
let inner: Writable | undefined;
|
||||
let aborted = false;
|
||||
const abortListener = (): void => {
|
||||
aborted = true;
|
||||
inner?.destroy();
|
||||
};
|
||||
return new WritableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
if (signal?.aborted) {
|
||||
controller.error(abortError(key));
|
||||
return;
|
||||
}
|
||||
inner = fs.createWriteStream(tempPath);
|
||||
inner.on("error", (err) => controller.error(ioError(key, err)));
|
||||
signal?.addEventListener("abort", abortListener);
|
||||
},
|
||||
write(chunk) {
|
||||
if (aborted || signal?.aborted || !inner) {
|
||||
throw abortError(key);
|
||||
}
|
||||
const stream = inner;
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
stream.write(chunk, (err) => {
|
||||
if (err) reject(ioError(key, err));
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
},
|
||||
async close() {
|
||||
signal?.removeEventListener("abort", abortListener);
|
||||
// `inner` is set during start; the only way it can be undefined here
|
||||
// is the pre-aborted-signal path, which already left the stream in an
|
||||
// errored state — the WritableStream contract means close() in that
|
||||
// case never reaches this sink callback. The non-null assertion is
|
||||
// therefore safe and lets us avoid an unreachable defensive branch.
|
||||
const stream = inner as NonNullable<typeof inner>;
|
||||
if (aborted || signal?.aborted) {
|
||||
await fs.unlink(tempPath).catch(() => undefined);
|
||||
throw abortError(key);
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
stream.end((err: Error | null | undefined) => {
|
||||
if (err) {
|
||||
reject(ioError(key, err));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
try {
|
||||
await fs.rename(tempPath, destinationPath);
|
||||
} catch (err) {
|
||||
await fs.unlink(tempPath).catch(() => undefined);
|
||||
throw new StorageFileError(
|
||||
STORAGE_FILE_CODES.ATOMIC_WRITE_FAILED,
|
||||
`Atomic rename failed: ${(err as Error).message}`,
|
||||
{ key, cause: err as Error }
|
||||
);
|
||||
}
|
||||
},
|
||||
async abort() {
|
||||
signal?.removeEventListener("abort", abortListener);
|
||||
inner?.destroy();
|
||||
await fs.unlink(tempPath).catch(() => undefined);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
# `fs-adapter.types`
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the `NodeFsLike` structural interface that `NodeFsStorage` consumes for every filesystem call. The interface is the seam that keeps the adapter injectable in tests without `vi.mock`/`vi.stubGlobal`: production code resolves to a real `node:fs/promises`-backed implementation, tests substitute a real in-memory or hooked implementation that points at a real `fs.mkdtemp` directory or selectively overrides one method to simulate a transient `EBUSY`.
|
||||
|
||||
## Features
|
||||
|
||||
- Mirrors only the subset of `node:fs` methods `NodeFsStorage` actually uses — no leaky abstraction.
|
||||
- Returns Node's `Readable` and `Writable` from the two stream constructors so consumers can convert to/from Web Streams without `as` casts.
|
||||
- `NodeFsStatLike` exposes only `size`, `mtimeMs`, `birthtimeMs`, `isFile()`, `isDirectory()` — the four properties the adapter reads.
|
||||
- `NodeFsFileHandleLike` exposes the positioned `read` + `close` pair needed for `getPartial`.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type { NodeFsLike } from "@suaveplan/storage-file/types";
|
||||
import { defaultNodeFs } from "@suaveplan/storage-file";
|
||||
|
||||
const fs: NodeFsLike = defaultNodeFs();
|
||||
const stat = await fs.stat("/tmp/example.data");
|
||||
console.log(stat.size);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { NodeFsLike } from "@suaveplan/storage-file/types";
|
||||
import { defaultNodeFs, NodeFsStorage } from "@suaveplan/storage-file";
|
||||
|
||||
// A hooked adapter that injects a single transient failure on writeFile.
|
||||
function hookOneEbusy(base: NodeFsLike): NodeFsLike {
|
||||
let fired = false;
|
||||
return {
|
||||
...base,
|
||||
async writeFile(path, data) {
|
||||
if (!fired) {
|
||||
fired = true;
|
||||
const err = Object.assign(new Error("EBUSY"), { code: "EBUSY" });
|
||||
throw err;
|
||||
}
|
||||
return base.writeFile(path, data);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const storage = new NodeFsStorage({ rootDir: "/tmp/store", fs: hookOneEbusy(defaultNodeFs()) });
|
||||
await storage.set("k", new Uint8Array([1, 2, 3])); // retried once, succeeds
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `NodeFsLike` — structural interface with 11 methods (`mkdir`, `readFile`, `writeFile`, `rename`, `unlink`, `readdir`, `stat`, `access`, `open`, `createReadStream`, `createWriteStream`).
|
||||
- `NodeFsStatLike` — `{ size, mtimeMs, birthtimeMs, isFile(), isDirectory() }`.
|
||||
- `NodeFsFileHandleLike` — `{ read(buffer, offset, length, position), close() }`.
|
||||
- `NodeFsReaddirEntry` — `{ name, isFile(), isDirectory() }`.
|
||||
- `NodeFsCreateReadStreamOptions` — `{ start?, end?, highWaterMark? }`.
|
||||
- `NodeFsCreateWriteStreamOptions` — `{ flags?, highWaterMark? }`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Declared as `interface`s (not `type` aliases) so downstream consumers can declaration-merge if they need to extend the surface for a specialised backend (e.g. a fuse mount with a custom flag).
|
||||
- Returns `Promise<string | undefined>` from `mkdir` to match Node's API — `recursive: true` returns the first path created or `undefined` if nothing was created. The adapter never reads the value.
|
||||
- `readdir` is fixed to `withFileTypes: true` for type narrowing; the adapter never uses the legacy string-array shape.
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Structural interface representing the slice of `node:fs/promises` (plus the
|
||||
* two `node:fs` stream constructors and `node:fs/promises.open`) that
|
||||
* `NodeFsStorage` consumes. Declaring this as an interface (rather than
|
||||
* importing the Node types directly) keeps the adapter injectable for tests
|
||||
* without `vi.mock`/`vi.stubGlobal` and lets fixtures replace any subset of
|
||||
* methods deterministically.
|
||||
*
|
||||
* Every method here mirrors the shape of its Node counterpart — but the
|
||||
* adapter never reads from the global `node:fs` module at import time. The
|
||||
* default implementation in `./node-fs/node-fs.ts` does the binding lazily
|
||||
* inside a factory so consumers that bundle for non-Node runtimes can
|
||||
* substitute a compatible shim.
|
||||
*/
|
||||
|
||||
import type { Readable, Writable } from "node:stream";
|
||||
|
||||
export interface NodeFsStatLike {
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
birthtimeMs: number;
|
||||
isFile(): boolean;
|
||||
isDirectory(): boolean;
|
||||
}
|
||||
|
||||
export interface NodeFsFileHandleLike {
|
||||
read(
|
||||
buffer: Uint8Array,
|
||||
offset: number,
|
||||
length: number,
|
||||
position: number
|
||||
): Promise<{ bytesRead: number; buffer: Uint8Array }>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface NodeFsReaddirEntry {
|
||||
name: string;
|
||||
/** Absolute path of the parent directory. Populated by Node's Dirent.parentPath. */
|
||||
parentPath: string;
|
||||
isFile(): boolean;
|
||||
isDirectory(): boolean;
|
||||
}
|
||||
|
||||
export interface NodeFsCreateReadStreamOptions {
|
||||
start?: number;
|
||||
end?: number;
|
||||
highWaterMark?: number;
|
||||
}
|
||||
|
||||
export interface NodeFsCreateWriteStreamOptions {
|
||||
flags?: string;
|
||||
highWaterMark?: number;
|
||||
}
|
||||
|
||||
export interface NodeFsLike {
|
||||
mkdir(
|
||||
path: string,
|
||||
options: { recursive: boolean }
|
||||
): Promise<string | undefined>;
|
||||
readFile(path: string): Promise<Uint8Array>;
|
||||
writeFile(path: string, data: Uint8Array | string): Promise<void>;
|
||||
rename(oldPath: string, newPath: string): Promise<void>;
|
||||
unlink(path: string): Promise<void>;
|
||||
readdir(
|
||||
path: string,
|
||||
options: { recursive: boolean; withFileTypes: true }
|
||||
): Promise<NodeFsReaddirEntry[]>;
|
||||
stat(path: string): Promise<NodeFsStatLike>;
|
||||
access(path: string): Promise<void>;
|
||||
open(path: string, flags: string): Promise<NodeFsFileHandleLike>;
|
||||
createReadStream(
|
||||
path: string,
|
||||
options?: NodeFsCreateReadStreamOptions
|
||||
): Readable;
|
||||
createWriteStream(
|
||||
path: string,
|
||||
options?: NodeFsCreateWriteStreamOptions
|
||||
): Writable;
|
||||
}
|
||||
17
packages/persistence/storage/storage-file/src/types/index.ts
Normal file
17
packages/persistence/storage/storage-file/src/types/index.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
export type {
|
||||
NodeFsCreateReadStreamOptions,
|
||||
NodeFsCreateWriteStreamOptions,
|
||||
NodeFsFileHandleLike,
|
||||
NodeFsLike,
|
||||
NodeFsReaddirEntry,
|
||||
NodeFsStatLike,
|
||||
} from "./fs-adapter.types.js";
|
||||
export type {
|
||||
NodeFsStorageOptions,
|
||||
NodeFsStorageRetryOptions,
|
||||
TimersLike,
|
||||
} from "./node-fs-storage.types.js";
|
||||
export {
|
||||
NodeFsStorageOptionsSchema,
|
||||
NodeFsStorageRetryOptionsSchema,
|
||||
} from "./node-fs-storage.types.js";
|
||||
@@ -0,0 +1,55 @@
|
||||
# `node-fs-storage.types`
|
||||
|
||||
## Purpose
|
||||
|
||||
Declares the option shape and Zod schema consumed by `NodeFsStorage`'s constructor. The schema is the runtime boundary — every option that flows into the adapter is parsed exactly once, at construction time, and a parse failure surfaces as `STORAGE_FILE.INVALID_OPTIONS`. Co-locating the type and the schema in a single file keeps inferred type ⇄ schema in lockstep across edits.
|
||||
|
||||
## Features
|
||||
|
||||
- `NodeFsStorageOptions` interface enumerating every constructor option with inline JSDoc.
|
||||
- `NodeFsStorageOptionsSchema` Zod object schema validating the option bag at the public boundary.
|
||||
- `NodeFsStorageRetryOptions` carrying the retry knobs forwarded to `@suaveplan/retry`.
|
||||
- `TimersLike` interface — the injectable timer pair the retry loop consumes (`setTimer`/`clearTimer`) — declared here once so consumers can ship deterministic FakeClock-backed fakes without importing test-only helpers.
|
||||
- Loose `unknown` shape for the injectable adapters (`fs`, `timers`, `clock`) — Zod cannot fully describe their structural shape without losing the structural contract, so we accept any value and rely on TypeScript at the call site.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { NodeFsStorageOptionsSchema } from "@suaveplan/storage-file";
|
||||
|
||||
const parsed = NodeFsStorageOptionsSchema.parse({ rootDir: "/tmp/store" });
|
||||
console.log(parsed.rootDir);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { NodeFsStorage, type NodeFsStorageOptions } from "@suaveplan/storage-file";
|
||||
import { fakeClock } from "@suaveplan/testing/runner";
|
||||
|
||||
const fc = fakeClock({ now: 1_700_000_000_000 });
|
||||
const options: NodeFsStorageOptions = {
|
||||
rootDir: "/tmp/store",
|
||||
mkdirp: true,
|
||||
fileExtension: ".bin",
|
||||
retry: { maxAttempts: 5, baseDelayMs: 25, maxDelayMs: 1_000 },
|
||||
clock: () => fc.now(),
|
||||
timers: { setTimer: fc.setTimeout, clearTimer: fc.clearTimeout },
|
||||
};
|
||||
const storage = new NodeFsStorage(options);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- `NodeFsStorageOptions` — interface; required `rootDir`, optional `mkdirp`, `fileExtension`, `metadataExtension`, `retry`, `fs`, `timers`, `clock`.
|
||||
- `NodeFsStorageRetryOptions` — interface; optional `maxAttempts`, `baseDelayMs`, `maxDelayMs`.
|
||||
- `NodeFsStorageOptionsSchema` — Zod schema for `NodeFsStorageOptions`. Throws on missing `rootDir`, empty `rootDir`/`fileExtension`/`metadataExtension`, non-positive `maxAttempts`/`maxDelayMs`, or negative `baseDelayMs`.
|
||||
- `NodeFsStorageRetryOptionsSchema` — Zod schema for the retry sub-bag.
|
||||
- `TimersLike` — interface; `setTimer(handler, ms)` and `clearTimer(id)`. Compatible with `globalThis.setTimeout`/`clearTimeout` (the default).
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `rootDir` is validated `min(1)` — empty strings are rejected to avoid ambiguous "current directory" semantics.
|
||||
- `fileExtension` and `metadataExtension` are also `min(1)` so callers cannot accidentally collide every key into the same path.
|
||||
- The adapter validates `fs`/`timers`/`clock` only structurally — passing `{} as never` will surface as a missing-method TypeError on the first call. This is intentional: forcing every option to have a Zod-describable shape would prevent injecting real callable functions.
|
||||
- The schema accepts `retry: undefined` and `retry: {}` — both produce default retry behaviour.
|
||||
@@ -0,0 +1,56 @@
|
||||
import type { Clock, TimersLike } from "@suaveplan/types";
|
||||
import z from "zod";
|
||||
import type { NodeFsLike } from "./fs-adapter.types.js";
|
||||
|
||||
// Re-export so consumers of `@suaveplan/storage-file/types` get the canonical
|
||||
// shared `TimersLike` contract without importing `@suaveplan/types` directly.
|
||||
export type { TimersLike };
|
||||
|
||||
export interface NodeFsStorageRetryOptions {
|
||||
maxAttempts?: number;
|
||||
baseDelayMs?: number;
|
||||
maxDelayMs?: number;
|
||||
}
|
||||
|
||||
export interface NodeFsStorageOptions {
|
||||
/** Root directory under which all keys are stored. Required. */
|
||||
rootDir: string;
|
||||
/** Whether `set` automatically creates missing parent directories. Default: true. */
|
||||
mkdirp?: boolean;
|
||||
/** Extension appended to data files. Default: `.data`. */
|
||||
fileExtension?: string;
|
||||
/** Extension appended to metadata sidecar files. Default: `.meta`. */
|
||||
metadataExtension?: string;
|
||||
/** Retry configuration applied to transient filesystem errors. */
|
||||
retry?: NodeFsStorageRetryOptions;
|
||||
/** Injectable filesystem adapter — defaults to `defaultNodeFs()`. */
|
||||
fs?: NodeFsLike;
|
||||
/** Injectable timers pair used by the retry loop. */
|
||||
timers?: TimersLike;
|
||||
/** Injectable clock used for metadata timestamps. */
|
||||
clock?: Clock;
|
||||
}
|
||||
|
||||
export const NodeFsStorageRetryOptionsSchema = z.object({
|
||||
maxAttempts: z.number().int().positive().optional(),
|
||||
baseDelayMs: z.number().int().nonnegative().optional(),
|
||||
maxDelayMs: z.number().int().positive().optional(),
|
||||
});
|
||||
|
||||
// fs / timers / clock are validated structurally — Zod cannot fully describe
|
||||
// them without losing the structural shape. We accept `unknown` and rely on
|
||||
// TypeScript to enforce the contract at the call site.
|
||||
const looseAdapter = z.unknown().optional();
|
||||
|
||||
export const NodeFsStorageOptionsSchema = z.object({
|
||||
rootDir: z
|
||||
.string()
|
||||
.min(1, { message: "rootDir must be a non-empty string" }),
|
||||
mkdirp: z.boolean().optional(),
|
||||
fileExtension: z.string().min(1).optional(),
|
||||
metadataExtension: z.string().min(1).optional(),
|
||||
retry: NodeFsStorageRetryOptionsSchema.optional(),
|
||||
fs: looseAdapter,
|
||||
timers: looseAdapter,
|
||||
clock: looseAdapter,
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
// Polyfills for test environment
|
||||
12
packages/persistence/storage/storage-file/tsconfig.json
Normal file
12
packages/persistence/storage/storage-file/tsconfig.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": ".",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts", "tests/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
12
packages/persistence/storage/storage-file/tsconfig.lib.json
Normal file
12
packages/persistence/storage/storage-file/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
39
packages/persistence/storage/storage-file/vite.config.ts
Normal file
39
packages/persistence/storage/storage-file/vite.config.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/storage-file",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@opentelemetry/api",
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/retry",
|
||||
/^@suaveplan\/storage/,
|
||||
"@suaveplan/types",
|
||||
"tslib",
|
||||
"zod",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
28
packages/persistence/storage/storage-file/vitest.config.ts
Normal file
28
packages/persistence/storage/storage-file/vitest.config.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.spec-conformance.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user