Initial Commit
This commit is contained in:
361
openspec/archive/2026-04-23-enterprise-storage/proposal.md
Normal file
361
openspec/archive/2026-04-23-enterprise-storage/proposal.md
Normal file
@@ -0,0 +1,361 @@
|
||||
# OpenSpec Change: enterprise-storage
|
||||
|
||||
- **Change ID**: enterprise-storage
|
||||
- **Package**: `@suaveplan/storage`
|
||||
- **Category**: universal
|
||||
- **Target path**: `universal/storage/storage`
|
||||
- **Status**: IN PROGRESS
|
||||
- **Date**: 2026-04-23
|
||||
- **Reference spec**: `/mnt/s/web/suaveplan/enterprise/storage.md`
|
||||
|
||||
## Summary
|
||||
|
||||
Bootstrap `@suaveplan/storage` on genesis from scratch. This package provides the core key-value storage abstractions used by all downstream packages that need persistent or in-process state: `@suaveplan/drive`, `@suaveplan/idempotency`, `@suaveplan/logger` (StorageLogger), `@suaveplan/crypto-keys` (Keychain serialized-JSON adapter), and others.
|
||||
|
||||
The genesis rewrite preserves the full public API surface from the main branch and adds enterprise-grade enhancements: `StorageLike` implementation on `AbstractStorage`, streaming primitives (`createReadStream`/`createWriteStream`/`getPartial`/`getItemSize`) required by the drive layer, structured errors extending `@suaveplan/error` with stable `STORAGE.*` codes, Zod schemas for every config and options shape, OTel counters and histograms on all public operations, `EventBusLike` integration replacing the old `EventEmitter` re-export, `AbortSignal` on every async method, property-based tests (fast-check), and a mitata benchmark suite.
|
||||
|
||||
## Current-state audit (genesis)
|
||||
|
||||
The package does not exist on genesis yet — this is a fresh build.
|
||||
|
||||
## Available workspace deps on genesis
|
||||
|
||||
| Dependency | Catalog / pin | Available |
|
||||
|---|---|---|
|
||||
| `@suaveplan/error` | `workspace:^` | ✅ |
|
||||
| `@suaveplan/types` | `workspace:^` | ✅ (includes `StorageLike`) |
|
||||
| `@suaveplan/events` | `workspace:^` | ✅ |
|
||||
| `zod` | `catalog:forms` | ✅ |
|
||||
| `@opentelemetry/api` | `catalog:observability` | ✅ |
|
||||
| `tslib` | `catalog:build` | ✅ |
|
||||
| `vitest` | `catalog:testing` (devDep) | ✅ |
|
||||
| `@vitest/coverage-istanbul` | `catalog:testing` (devDep) | ✅ |
|
||||
| `fast-check` | `catalog:testing` (devDep) | ✅ |
|
||||
| `mitata` | `catalog:testing` (devDep) | ✅ |
|
||||
|
||||
## Preserved public API surface (from main branch)
|
||||
|
||||
### Errors (`src/errors/`)
|
||||
|
||||
```ts
|
||||
// Extend @suaveplan/error via defineCodes — not ad-hoc subclasses
|
||||
class StorageError extends SuaveplanError // code: STORAGE.GENERIC
|
||||
class StorageNotFoundError extends StorageError // code: STORAGE.NOT_FOUND
|
||||
class StoragePermissionError extends StorageError // code: STORAGE.PERMISSION_DENIED
|
||||
class StorageQuotaError extends StorageError // code: STORAGE.QUOTA_EXCEEDED
|
||||
// New on genesis:
|
||||
class StorageInvalidKeyError extends StorageError // code: STORAGE.INVALID_KEY
|
||||
class StorageBatchError extends StorageError // code: STORAGE.BATCH_ERROR
|
||||
```
|
||||
|
||||
### Types (`src/types/`)
|
||||
|
||||
```ts
|
||||
// All from main branch — Zod-validated on genesis
|
||||
interface StorageOptions {
|
||||
ttl?: number;
|
||||
metadata?: Record<string, unknown>;
|
||||
compress?: boolean;
|
||||
encrypt?: boolean;
|
||||
}
|
||||
|
||||
interface StorageMetadata {
|
||||
createdAt: number;
|
||||
modifiedAt: number;
|
||||
expiresAt?: number;
|
||||
size: number;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface StorageResult<T> {
|
||||
value: T;
|
||||
metadata: StorageMetadata;
|
||||
}
|
||||
|
||||
interface StorageListOptions {
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
prefix?: string;
|
||||
continuationToken?: string;
|
||||
}
|
||||
|
||||
interface StorageListResult {
|
||||
keys: string[];
|
||||
total: number;
|
||||
hasMore: boolean;
|
||||
continuationToken?: string;
|
||||
}
|
||||
|
||||
interface StoragePatternOptions extends StorageListOptions {
|
||||
pattern?: string;
|
||||
}
|
||||
|
||||
type StorageBatchOperation =
|
||||
| { type: 'get'; key: string }
|
||||
| { type: 'set'; key: string; value: Uint8Array; options?: StorageOptions }
|
||||
| { type: 'delete'; key: string };
|
||||
|
||||
interface StorageBatchResult {
|
||||
operation: StorageBatchOperation;
|
||||
success: boolean;
|
||||
value?: Uint8Array | null;
|
||||
error?: Error;
|
||||
}
|
||||
|
||||
interface StorageStats {
|
||||
reads: number;
|
||||
writes: number;
|
||||
deletes: number;
|
||||
hits: number;
|
||||
misses: number;
|
||||
bytesRead: number;
|
||||
bytesWritten: number;
|
||||
keyCount?: number;
|
||||
totalGets?: number;
|
||||
totalSets?: number;
|
||||
totalDeletes?: number;
|
||||
totalErrors?: number;
|
||||
totalSize?: number;
|
||||
}
|
||||
|
||||
// Event types — replaces EventEmitter-based surface with EventBusLike
|
||||
type StorageEventType = 'set' | 'get' | 'delete' | 'clear' | 'error';
|
||||
|
||||
interface StorageEventData {
|
||||
key?: string;
|
||||
value?: Uint8Array | null;
|
||||
error?: Error;
|
||||
}
|
||||
|
||||
type StorageEventHandler = (data: StorageEventData) => void;
|
||||
|
||||
// Hook types (unchanged from main)
|
||||
interface StoragePreSetHook {
|
||||
name?: string;
|
||||
priority?: number;
|
||||
transform(key: string, value: Uint8Array, options?: StorageOptions):
|
||||
Promise<{ key: string; value: Uint8Array; options?: StorageOptions }>;
|
||||
}
|
||||
|
||||
interface StoragePostGetHook {
|
||||
name?: string;
|
||||
priority?: number;
|
||||
transform(key: string, value: Uint8Array): Promise<Uint8Array>;
|
||||
}
|
||||
```
|
||||
|
||||
### Factory types (`src/factory/`)
|
||||
|
||||
```ts
|
||||
// Module-augmentation pattern preserved
|
||||
interface StorageProviderRegistry {
|
||||
memory: MemoryStorage;
|
||||
// downstream packages augment this
|
||||
}
|
||||
|
||||
type StorageProviderType = keyof StorageProviderRegistry;
|
||||
type StorageProviderImplementation<T extends StorageProviderType> = StorageProviderRegistry[T];
|
||||
|
||||
interface StorageProviderConfig {
|
||||
memory?: MemoryStorageConfig;
|
||||
}
|
||||
|
||||
interface MemoryStorageConfig {
|
||||
maxSize?: number;
|
||||
cleanupIntervalMs?: number; // surfaced from MemoryStorage constructor
|
||||
}
|
||||
|
||||
type StorageProviderConfigFor<T extends StorageProviderType> = ...;
|
||||
|
||||
interface StorageFactoryConfig<T extends StorageProviderType> {
|
||||
type: T;
|
||||
config?: StorageProviderConfigFor<T>;
|
||||
}
|
||||
|
||||
// Alias export (backwards compat)
|
||||
type InMemory = MemoryStorage;
|
||||
```
|
||||
|
||||
### AbstractStorage (`src/abstract/`)
|
||||
|
||||
All existing public methods preserved, with `AbortSignal` added and `StorageLike` conformance:
|
||||
|
||||
```ts
|
||||
abstract class AbstractStorage implements StorageLike {
|
||||
// StorageLike contract (adapted: get returns Uint8Array | undefined, delete returns boolean)
|
||||
abstract get(key: string, signal?: AbortSignal): Promise<Uint8Array | undefined>;
|
||||
abstract set(key: string, value: Uint8Array, options?: StorageOptions, signal?: AbortSignal): Promise<void>;
|
||||
abstract delete(key: string, signal?: AbortSignal): Promise<boolean>;
|
||||
abstract exists(key: string, signal?: AbortSignal): Promise<boolean>;
|
||||
abstract list(options?: StorageListOptions, signal?: AbortSignal): Promise<StorageListResult>;
|
||||
abstract clear(signal?: AbortSignal): Promise<void>;
|
||||
|
||||
// Extended beyond StorageLike
|
||||
abstract getWithMetadata(key: string, signal?: AbortSignal): Promise<StorageResult<Uint8Array> | null>;
|
||||
abstract getMetadata(key: string, signal?: AbortSignal): Promise<StorageMetadata | null>;
|
||||
abstract getSize(signal?: AbortSignal): Promise<number>;
|
||||
|
||||
// Streaming (new on genesis — required by @suaveplan/drive)
|
||||
abstract getPartial(key: string, offset: number, length: number, signal?: AbortSignal): Promise<Uint8Array | null>;
|
||||
abstract getItemSize(key: string, signal?: AbortSignal): Promise<number | null>;
|
||||
abstract createReadStream(key: string, signal?: AbortSignal): ReadableStream<Uint8Array>;
|
||||
abstract createWriteStream(key: string, signal?: AbortSignal): WritableStream<Uint8Array>;
|
||||
|
||||
// Concrete helpers on AbstractStorage (all accept optional AbortSignal)
|
||||
batch(operations: StorageBatchOperation[], signal?: AbortSignal): Promise<StorageBatchResult[]>;
|
||||
getdel(key: string, signal?: AbortSignal): Promise<Uint8Array | null>;
|
||||
mdelete(keys: string[], signal?: AbortSignal): Promise<number>;
|
||||
listWithPattern(options?: StoragePatternOptions, signal?: AbortSignal): Promise<StorageListResult>;
|
||||
namespace(prefix: string): NamespacedStorage;
|
||||
|
||||
// Size limits
|
||||
setMaxValueSize(bytes: number): void;
|
||||
setMaxTotalSize(bytes?: number): void;
|
||||
|
||||
// Stats
|
||||
getStats(): StorageStats;
|
||||
resetStats(): void;
|
||||
|
||||
// Event hooks (EventBusLike-backed on genesis)
|
||||
on(event: StorageEventType, handler: StorageEventHandler): void;
|
||||
off(event: StorageEventType, handler: StorageEventHandler): void;
|
||||
|
||||
// Transform hooks
|
||||
addPreSetHook(hook: StoragePreSetHook): void;
|
||||
addPostGetHook(hook: StoragePostGetHook): void;
|
||||
removeHook(name?: string): void;
|
||||
}
|
||||
```
|
||||
|
||||
### NamespacedStorage (`src/abstract/`)
|
||||
|
||||
```ts
|
||||
class NamespacedStorage extends AbstractStorage {
|
||||
constructor(storage: AbstractStorage, prefix: string);
|
||||
// Delegates all operations to underlying storage with prefix: key → `${prefix}:${key}`
|
||||
// clear() only removes keys in this namespace
|
||||
// getSize() returns scoped byte count (not total — fixed from main branch approximation)
|
||||
}
|
||||
```
|
||||
|
||||
### MemoryStorage (`src/memory/`)
|
||||
|
||||
```ts
|
||||
class MemoryStorage extends AbstractStorage {
|
||||
constructor(options?: MemoryStorageConfig);
|
||||
|
||||
// Implements all AbstractStorage abstracts
|
||||
get size(): number; // number of live (non-expired) keys
|
||||
destroy(): void; // stops cleanup interval
|
||||
|
||||
// New on genesis
|
||||
getPartial(key, offset, length, signal?): Promise<Uint8Array | null>;
|
||||
getItemSize(key, signal?): Promise<number | null>;
|
||||
createReadStream(key, signal?): ReadableStream<Uint8Array>;
|
||||
createWriteStream(key, signal?): WritableStream<Uint8Array>;
|
||||
}
|
||||
```
|
||||
|
||||
### Factory functions (`src/factory/`)
|
||||
|
||||
```ts
|
||||
function registerStorageProvider<T extends string>(
|
||||
type: T,
|
||||
implementation: new (config?: unknown) => AbstractStorage
|
||||
): void;
|
||||
|
||||
function createStorage<T extends StorageProviderType>(
|
||||
options: StorageFactoryConfig<T>
|
||||
): StorageProviderImplementation<T>;
|
||||
|
||||
function createStorageWithConfig<T extends StorageProviderType>(
|
||||
type: T,
|
||||
config?: StorageProviderConfigFor<T>
|
||||
): StorageProviderImplementation<T>;
|
||||
|
||||
function hasStorageProvider(type: string): boolean;
|
||||
function getStorageProviders(): string[];
|
||||
function unregisterStorageProvider(type: string): void;
|
||||
function clearStorageProviders(): void; // resets to memory-only
|
||||
|
||||
// Object facade (legacy compat — preserved, not deprecated on genesis)
|
||||
const StorageFactory: {
|
||||
register: typeof registerStorageProvider;
|
||||
create: typeof createStorage;
|
||||
createWithConfig: typeof createStorageWithConfig;
|
||||
hasProvider: typeof hasStorageProvider;
|
||||
getProviders: typeof getStorageProviders;
|
||||
unregister: typeof unregisterStorageProvider;
|
||||
clearProviders: typeof clearStorageProviders;
|
||||
};
|
||||
```
|
||||
|
||||
## Dropped from main branch
|
||||
|
||||
| Symbol | Reason |
|
||||
|---|---|
|
||||
| `export { EventEmitter } from './utils/event-emitter.js'` | `EventEmitter` was already just a re-export of `@suaveplan/events` on main. Genesis drops the pass-through; callers import directly from `@suaveplan/events`. `AbstractStorage` now accepts an optional injected `EventBusLike`. |
|
||||
|
||||
## New additions for genesis
|
||||
|
||||
### Structured errors (`src/errors/`)
|
||||
|
||||
- `defineCodes` from `@suaveplan/error` for stable `STORAGE.*` code registry.
|
||||
- All error classes extend `SuaveplanError` (cause-chain, OTel span-event, wire format, redaction).
|
||||
- Added: `StorageInvalidKeyError` (was an inline `StorageError` with code `"INVALID_KEY"` on main), `StorageBatchError`.
|
||||
|
||||
### Zod schemas (`src/types/`)
|
||||
|
||||
- `StorageOptionsSchema`, `StorageMetadataSchema`, `StorageResultSchema`, `StorageListOptionsSchema`, `StorageListResultSchema`, `StoragePatternOptionsSchema`, `StorageBatchOperationSchema`, `StorageBatchResultSchema`, `StorageStatsSchema`, `MemoryStorageConfigSchema`.
|
||||
- Used for runtime boundary validation (factory config, hook inputs) and wire-format safety.
|
||||
|
||||
### OTel instrumentation
|
||||
|
||||
- Counters: `storage.ops` (labelled by `operation`, `provider`, `status`).
|
||||
- Histograms: `storage.op.duration_ms` (per operation latency), `storage.value.bytes` (read/write sizes).
|
||||
- Applied at the `AbstractStorage` concrete-method layer, not inside subclasses.
|
||||
|
||||
### EventBusLike integration
|
||||
|
||||
- `AbstractStorage` constructor accepts an optional `EventBusLike` (from `@suaveplan/types`).
|
||||
- Default: an internal minimal bus backed by `@suaveplan/events` `EventBus`.
|
||||
- `on`/`off` delegate to the bus; existing `StorageEventType`/`StorageEventHandler` surface preserved.
|
||||
|
||||
### AbortSignal propagation
|
||||
|
||||
- All async abstract methods and concrete helpers accept an optional trailing `signal?: AbortSignal`.
|
||||
- `MemoryStorage` checks `signal.aborted` at the top of each method and throws `StorageError` with code `STORAGE.ABORTED`.
|
||||
- `batch()` checks signal between operations and rolls back on abort.
|
||||
|
||||
### Streaming primitives (required by `@suaveplan/drive`)
|
||||
|
||||
- `getPartial(key, offset, length, signal?)` — byte-range read.
|
||||
- `getItemSize(key, signal?)` — size probe without data transfer.
|
||||
- `createReadStream(key, signal?)` — `ReadableStream<Uint8Array>` backed by `get()`.
|
||||
- `createWriteStream(key, signal?)` — `WritableStream<Uint8Array>` that accumulates chunks then calls `set()`.
|
||||
- Default implementations on `AbstractStorage` using `get`/`set` for subclasses that don't override.
|
||||
- `MemoryStorage` overrides all four with zero-copy slices from the internal `Uint8Array`.
|
||||
|
||||
### Property-based tests (fast-check)
|
||||
|
||||
- Key-validation invariants (arbitrary strings, path-traversal, length boundaries).
|
||||
- `batch()` rollback correctness under arbitrary failure injection.
|
||||
- Namespace roundtrip: `namespace(p).set(k, v)` then `namespace(p).get(k)` equals `v`.
|
||||
- TTL expiry: for any `ttl > 0`, a read after `ttl` ms must return `undefined`.
|
||||
- `listWithPattern` result consistency against brute-force `list()` + JS filter.
|
||||
|
||||
### Benchmarks (`src/__benchmarks__/`)
|
||||
|
||||
- `memory-ops.bench.ts` — throughput for `get`/`set`/`delete`/`list` on `MemoryStorage`.
|
||||
- `batch-ops.bench.ts` — batch vs sequential write throughput.
|
||||
- `namespace-ops.bench.ts` — namespace overhead vs direct access.
|
||||
|
||||
### NamespacedStorage `getSize()` fix
|
||||
|
||||
Main branch `getSize()` returned the total storage size rather than the scoped namespace size. Genesis implements a correct scoped byte count by iterating only keys under `this.prefix`.
|
||||
|
||||
## Impact
|
||||
|
||||
- **Unblocks**: `@suaveplan/drive` (needs `getPartial`/`createReadStream`/`createWriteStream`), `@suaveplan/idempotency` (MemoryStore adapter), `@suaveplan/crypto-keys` (Keychain serialized-JSON), `@suaveplan/logger` (StorageLogger sink).
|
||||
- **No breaking changes** to the main-branch public API surface beyond the `EventEmitter` re-export removal and the `get()` return type narrowing from `null` to `undefined` (to match `StorageLike`).
|
||||
- **`sideEffects: false`** confirmed — provider registration happens lazily inside `factory.ts`, not at module scope before the factory module is imported.
|
||||
394
openspec/archive/2026-04-23-enterprise-storage/tasks.md
Normal file
394
openspec/archive/2026-04-23-enterprise-storage/tasks.md
Normal file
@@ -0,0 +1,394 @@
|
||||
# Tasks: enterprise-storage
|
||||
|
||||
Change ID: `enterprise-storage`
|
||||
Package: `@suaveplan/storage`
|
||||
Path: `universal/storage/storage`
|
||||
|
||||
Work tasks in order. Each section is a logical gate — do not start the next section until all checkboxes in the current section are green and `bunx turbo lint typecheck test --filter=@suaveplan/storage` passes.
|
||||
|
||||
---
|
||||
|
||||
## 1. Scaffold
|
||||
|
||||
- [x] Create `universal/storage/storage/` directory tree:
|
||||
```
|
||||
universal/storage/storage/
|
||||
src/
|
||||
abstract/
|
||||
errors/
|
||||
factory/
|
||||
memory/
|
||||
types/
|
||||
__benchmarks__/
|
||||
tests/
|
||||
package.json
|
||||
tsconfig.json
|
||||
tsconfig.build.json
|
||||
vite.config.ts
|
||||
vitest.config.ts
|
||||
biome.json (extend root)
|
||||
```
|
||||
- [x] Write `package.json`:
|
||||
- `name`: `@suaveplan/storage`, `version`: `0.1.0`, `private`: false
|
||||
- `"sideEffects": false`
|
||||
- `type`: `"module"`
|
||||
- `main`/`module`/`types` pointing at `dist/` output
|
||||
- Subpath exports:
|
||||
- `"."` → `./dist/index.js`
|
||||
- `"./abstract"` → `./dist/abstract/index.js`
|
||||
- `"./errors"` → `./dist/errors/index.js`
|
||||
- `"./factory"` → `./dist/factory/index.js`
|
||||
- `"./memory"` → `./dist/memory/index.js`
|
||||
- `"./types"` → `./dist/types/index.js`
|
||||
- Runtime deps: `@suaveplan/error workspace:^`, `@suaveplan/types workspace:^`, `@suaveplan/events workspace:^`, `zod catalog:forms`, `@opentelemetry/api catalog:observability`, `tslib catalog:build`
|
||||
- Dev deps: `vitest catalog:testing`, `@vitest/coverage-istanbul catalog:testing`, `fast-check catalog:testing`, `mitata catalog:testing`
|
||||
- Scripts: `build`, `test`, `test:coverage`, `test:bench`, `lint`, `typecheck`, `docs:check`
|
||||
- [x] Write `tsconfig.json` extending `@suaveplan/typescript-config/library.json`
|
||||
- [x] Write `tsconfig.build.json` (excludes test files and `__benchmarks__`)
|
||||
- [x] Write `vite.config.ts` for library build (entry `src/index.ts`, formats `es`/`cjs`, `dts: true`)
|
||||
- [x] Write `vitest.config.ts` using `createTestConfig()` from `@suaveplan/testing/config`; set `provider: 'istanbul'`, `thresholds` at 100/100/100/100
|
||||
- [x] Add package to `turbo.json` pipeline (inherits global tasks — confirm no extra config needed)
|
||||
- [x] Add `@suaveplan/storage` to root `bun.lockb` workspace list and run `bun install`
|
||||
- [x] Write guard test `tests/tree-shake.test.ts`:
|
||||
- Assert the built `dist/index.js` contains no `import ... from 'node:*'` (universal runtime constraint)
|
||||
- Assert `dist/index.js` byte size is tracked (soft limit alert at 50 KB)
|
||||
- [x] Confirm `bunx turbo build --filter=@suaveplan/storage` exits 0 (even with empty `src/index.ts`)
|
||||
|
||||
---
|
||||
|
||||
## 2. Types + Error Codes
|
||||
|
||||
### 2a. Error codes (`src/errors/`)
|
||||
|
||||
- [x] Create `src/errors/errors.ts`:
|
||||
- Import `SuaveplanError`, `defineCodes` from `@suaveplan/error`
|
||||
- Define code registry with `defineCodes`:
|
||||
```
|
||||
STORAGE.GENERIC
|
||||
STORAGE.NOT_FOUND
|
||||
STORAGE.PERMISSION_DENIED
|
||||
STORAGE.QUOTA_EXCEEDED
|
||||
STORAGE.INVALID_KEY
|
||||
STORAGE.BATCH_ERROR
|
||||
STORAGE.ABORTED
|
||||
```
|
||||
- Implement `StorageError extends SuaveplanError` — constructor accepts `(message, code, key?, cause?)`; preserves `key` as a typed public field; emits OTel span event on construction
|
||||
- Implement `StorageNotFoundError extends StorageError` — constructor `(key, cause?)`
|
||||
- Implement `StoragePermissionError extends StorageError` — constructor `(key, operation, cause?)`
|
||||
- Implement `StorageQuotaError extends StorageError` — constructor `(key, cause?)`
|
||||
- Implement `StorageInvalidKeyError extends StorageError` — constructor `(key, reason, cause?)`
|
||||
- Implement `StorageBatchError extends StorageError` — constructor `(message, cause?)`
|
||||
- [x] Create `src/errors/errors.types.ts` with any shared error-related type aliases
|
||||
- [x] Create `src/errors/errors.md` (purpose, features, usage examples, API reference, implementation notes — ≥ 200 words)
|
||||
- [x] Create `src/errors/index.ts` barrel
|
||||
- [x] Create `src/errors/errors.test.ts`:
|
||||
- Instantiate each error class and assert `name`, `code`, `key`, `cause` fields
|
||||
- Assert each is `instanceof SuaveplanError`
|
||||
- Assert `StorageError` with each `STORAGE.*` code round-trips through `@suaveplan/error` wire format
|
||||
- `{ timeout: 5000 }` on all tests
|
||||
|
||||
### 2b. Types and Zod schemas (`src/types/`)
|
||||
|
||||
- [x] Create `src/types/types.ts` — all interface definitions from the preserved surface (see proposal § Preserved public API):
|
||||
- `StorageOptions`, `StorageMetadata`, `StorageResult<T>`, `StorageListOptions`, `StorageListResult`, `StoragePatternOptions`, `StorageBatchOperation`, `StorageBatchResult`, `StorageStats`, `StorageEventType`, `StorageEventData`, `StorageEventHandler`, `StoragePreSetHook`, `StoragePostGetHook`
|
||||
- All `metadata` fields typed `Record<string, unknown>` (not `any`)
|
||||
- [x] Create `src/types/schemas.ts` — Zod schemas:
|
||||
- `StorageOptionsSchema`, `StorageMetadataSchema`, `StorageResultSchema` (generic via `z.object` + type param helper), `StorageListOptionsSchema`, `StorageListResultSchema`, `StoragePatternOptionsSchema`, `StorageBatchOperationSchema`, `StorageBatchResultSchema`, `StorageStatsSchema`
|
||||
- Export inferred types alongside schemas (e.g. `export type StorageOptions = z.infer<typeof StorageOptionsSchema>`)
|
||||
- [x] Create `src/types/types.md` and `src/types/schemas.md` (≥ 200 words each)
|
||||
- [x] Create `src/types/index.ts` barrel
|
||||
- [x] Create `src/types/types.test.ts`:
|
||||
- Parse valid and invalid objects through every Zod schema
|
||||
- Assert discriminated union `StorageBatchOperationSchema` correctly narrows on `type`
|
||||
- `{ timeout: 5000 }` on all tests
|
||||
|
||||
---
|
||||
|
||||
## 3. AbstractStorage Core
|
||||
|
||||
### 3a. Implementation (`src/abstract/abstract.ts`)
|
||||
|
||||
- [x] Declare `AbstractStorage implements StorageLike` (import from `@suaveplan/types`)
|
||||
- [x] Constructor accepts optional `eventBus?: EventBusLike` (from `@suaveplan/types`); defaults to `new EventBus()` from `@suaveplan/events`
|
||||
- [x] Internal `protected stats: StorageStats` initialized with all counters at 0
|
||||
- [x] Internal `protected preSetHooks: StoragePreSetHook[]` and `protected postGetHooks: StoragePostGetHook[]`
|
||||
- [x] Internal `protected maxValueSize: number` defaulting to `10 * 1024 * 1024`
|
||||
- [x] Internal `protected maxTotalSize: number | undefined`
|
||||
- [x] Implement `protected validateKey(key: string): void` — checks: non-empty, no `..` or `\`, no absolute path, length ≤ 1024, no `<>"|*?`; throws `StorageInvalidKeyError`
|
||||
- [x] Implement `protected async validateValueSize(key, value, signal?)` — throws `StorageQuotaError` on single-value or total-quota breach
|
||||
- [x] Declare all abstract methods with `signal?: AbortSignal` on every async signature:
|
||||
- `get`, `set`, `delete`, `exists`, `list`, `clear` (StorageLike conformance)
|
||||
- `getWithMetadata`, `getMetadata`, `getSize`
|
||||
- `getPartial`, `getItemSize`, `createReadStream`, `createWriteStream`
|
||||
- [x] Provide default implementations for streaming methods that delegate to `get`/`set` (subclasses may override for efficiency):
|
||||
- `getPartial`: calls `get`, slices the resulting `Uint8Array`
|
||||
- `getItemSize`: calls `getMetadata`, returns `metadata.size`
|
||||
- `createReadStream`: enqueues single chunk from `get()` then closes
|
||||
- `createWriteStream`: accumulates all chunks, calls `set()` on close
|
||||
- [x] Implement concrete helpers: `batch`, `getdel`, `mdelete`, `listWithPattern` — all accept optional `signal`, check `signal?.aborted` at start and between async steps; throw `StorageError(STORAGE.ABORTED)` when aborted
|
||||
- [x] `batch()` rollback: on error or abort, reverse completed `set`/`delete` ops; use structured `console.warn` only if OTel is unavailable at that point, otherwise emit span event
|
||||
- [x] Implement `setMaxValueSize`, `setMaxTotalSize`
|
||||
- [x] Implement `getStats()`, `resetStats()`
|
||||
- [x] Implement `on(event, handler)` and `off(event, handler)` delegating to the injected `EventBusLike`
|
||||
- [x] Implement `addPreSetHook`, `addPostGetHook`, `removeHook` with priority sort
|
||||
- [x] Implement `protected async applyPreSetHooks(key, value, options?)` and `protected async applyPostGetHooks(key, value)`
|
||||
- [x] Implement `protected emitEvent(event, data)` — publishes to `EventBusLike`
|
||||
- [x] Implement `namespace(prefix): NamespacedStorage`
|
||||
|
||||
### 3b. OTel instrumentation (`src/abstract/abstract.ts` or extracted `src/abstract/telemetry.ts`)
|
||||
|
||||
- [x] Import `metrics` from `@opentelemetry/api`
|
||||
- [x] Create meter `@suaveplan/storage` once (module-level singleton, lazy init)
|
||||
- [x] Counter `storage.ops` — attributes: `operation` (get/set/delete/exists/list/clear/batch/getdel/mdelete), `provider` (class name), `status` (success/error/aborted)
|
||||
- [x] Histogram `storage.op.duration_ms` — same attributes
|
||||
- [x] Histogram `storage.value.bytes` — attributes: `operation` (get/set), `provider`
|
||||
- [x] Instrument all concrete helper methods and wrap the OTel recording around the abstract method calls within those helpers; individual subclasses do not need to add OTel (they inherit it)
|
||||
|
||||
### 3c. NamespacedStorage (`src/abstract/namespace.ts`)
|
||||
|
||||
- [x] `class NamespacedStorage extends AbstractStorage`
|
||||
- [x] Constructor: `(storage: AbstractStorage, prefix: string)` — validate prefix is non-empty, no `:`
|
||||
- [x] `prefixKey(key)` → `${prefix}:${key}`; `unprefixKey(key)` strips prefix
|
||||
- [x] Implement all abstract methods by delegating to underlying `storage` with prefixed key
|
||||
- [x] `list()` filters by `prefix:` prefix, strips prefix from returned keys, applies pagination correctly against filtered total
|
||||
- [x] `clear()` lists all keys in this namespace and calls `mdelete` on prefixed keys — scoped only, does not touch other namespaces
|
||||
- [x] `getSize()` iterates only namespace keys and sums `metadata.size` — fixes the main-branch approximate-total bug
|
||||
- [x] `getPartial`, `getItemSize`, `createReadStream`, `createWriteStream` delegate via `prefixKey`
|
||||
|
||||
### 3d. Docs and tests
|
||||
|
||||
- [x] Create `src/abstract/abstract.md` (≥ 200 words, both basic and advanced examples)
|
||||
- [x] Create `src/abstract/namespace.md` (≥ 200 words)
|
||||
- [x] Create `src/abstract/index.ts` barrel
|
||||
- [x] Create `src/abstract/abstract.test.ts` — test via a minimal `ConcreteStorage` implementation (not `MemoryStorage`; keeps unit tests isolated):
|
||||
- `validateKey`: empty key, path-traversal variants, absolute paths, over-1024-chars, forbidden chars, valid key
|
||||
- `validateValueSize`: single-value limit, total-quota limit
|
||||
- `batch()`: happy path, rollback on mid-batch failure, abort signal mid-batch
|
||||
- `getdel()`: value returned and deleted, null when absent
|
||||
- `mdelete()`: returns count of actually-deleted keys
|
||||
- `listWithPattern()`: glob `*`, `?`, no-pattern passthrough, pagination
|
||||
- `on/off/emitEvent`: handler called on emit, removed after `off`
|
||||
- `addPreSetHook`/`addPostGetHook`/`removeHook`: priority ordering, named removal
|
||||
- `setMaxValueSize`/`setMaxTotalSize`: rejects invalid values
|
||||
- `getStats()`/`resetStats()`: snapshot copy, reset to zeros
|
||||
- `namespace()`: returns `NamespacedStorage` with correct prefix
|
||||
- AbortSignal: each helper aborts before work starts
|
||||
- `{ timeout: 5000 }` on sync tests; `{ timeout: 30000 }` on async tests
|
||||
- [x] Create `src/abstract/namespace.test.ts`:
|
||||
- Prefix isolation: keys in `ns('a')` never visible via `ns('b')`
|
||||
- `clear()`: only removes namespace-scoped keys
|
||||
- `getSize()`: returns scoped bytes only
|
||||
- `list()` pagination against namespace-filtered total
|
||||
- `{ timeout: 5000 }` / `{ timeout: 30000 }` as appropriate
|
||||
|
||||
---
|
||||
|
||||
## 4. MemoryStorage
|
||||
|
||||
### 4a. Implementation (`src/memory/memory.ts`)
|
||||
|
||||
- [x] `class MemoryStorage extends AbstractStorage`
|
||||
- [x] Internal `Map<string, { value: Uint8Array; metadata: StorageMetadata }>` store
|
||||
- [x] Constructor accepts `MemoryStorageConfig` (Zod-validated): `maxSize?`, `cleanupIntervalMs?` (default 60000; 0 disables timer)
|
||||
- [x] Background TTL cleanup loop using `setInterval`; call `.unref()` in Node/Bun to avoid blocking exit
|
||||
- [x] `get(key, signal?)`:
|
||||
- Check `signal?.aborted`; throw `StorageError(STORAGE.ABORTED)`
|
||||
- Return `undefined` (not `null`) for missing/expired keys — matches `StorageLike`
|
||||
- Expire-on-read for TTL entries; update stats
|
||||
- Apply `postGetHooks`; emit `get` event
|
||||
- [x] `set(key, value, options?, signal?)`:
|
||||
- Check signal; `validateKey`; `validateValueSize`; apply `preSetHooks`
|
||||
- Write entry with metadata (`createdAt`, `modifiedAt`, `expiresAt`, `size`)
|
||||
- On error: roll back stats increments
|
||||
- [x] `delete(key, signal?)`: returns `boolean` (true if existed) — matches `StorageLike`; update stats; emit event
|
||||
- [x] `exists(key, signal?)`: expire-on-read; return boolean
|
||||
- [x] `list(options?, signal?)`: filter expired, filter by `prefix`, paginate, return `StorageListResult`
|
||||
- [x] `getWithMetadata(key, signal?)`: returns `StorageResult<Uint8Array> | null`
|
||||
- [x] `getMetadata(key, signal?)`: returns `StorageMetadata | null` without retrieving value bytes
|
||||
- [x] `clear(signal?)`: clears map; resets `keyCount`/`totalSize` stats; emits `clear` event
|
||||
- [x] `getSize(signal?)`: sums live (non-expired) entry byte lengths
|
||||
- [x] `get size(): number` — count of live (non-expired) entries
|
||||
- [x] `destroy()`: clears the cleanup interval; idempotent
|
||||
- [x] Streaming overrides (zero-copy from internal `Uint8Array`):
|
||||
- `getPartial(key, offset, length, signal?)`: `entry.value.subarray(offset, offset + length)` — handles out-of-range gracefully (clamp to actual size)
|
||||
- `getItemSize(key, signal?)`: `entry.metadata.size` without extra allocation
|
||||
- `createReadStream(key, signal?)`: `ReadableStream` that enqueues the stored `Uint8Array` (defensive copy so callers cannot mutate the store)
|
||||
- `createWriteStream(key, signal?)`: `WritableStream` that accumulates typed-array chunks then calls `this.set()` on close; pipes abort signal through `WritableStreamDefaultController`
|
||||
|
||||
### 4b. Types (`src/memory/memory.types.ts`)
|
||||
|
||||
- [x] `MemoryEntry` — internal store entry shape
|
||||
- [x] `MemoryStorageConfig` — exported, used by `MemoryStorageConfigSchema` in `src/types/schemas.ts`
|
||||
|
||||
### 4c. Docs and tests
|
||||
|
||||
- [x] Create `src/memory/memory.md` (≥ 200 words, includes TTL and streaming examples)
|
||||
- [x] Create `src/memory/index.ts` barrel
|
||||
- [x] Create `src/memory/memory.test.ts` (core operations):
|
||||
- `get`/`set`/`delete`/`exists`/`list`/`clear`/`getSize` happy paths
|
||||
- TTL: entry expires after `ttl` ms, returns `undefined` on late read
|
||||
- `getWithMetadata` / `getMetadata` field correctness
|
||||
- Stats accuracy across all operations including error paths
|
||||
- Size limit (`maxSize`) enforced on `set`
|
||||
- `destroy()` stops the cleanup timer
|
||||
- `get size` property reflects live key count
|
||||
- AbortSignal: each method aborts correctly
|
||||
- `{ timeout: 5000 }` / `{ timeout: 30000 }` as appropriate
|
||||
- [x] Create `src/memory/memory-streaming.test.ts`:
|
||||
- `getPartial`: byte-range slices, out-of-range clamping, missing key returns null
|
||||
- `getItemSize`: matches stored byte length
|
||||
- `createReadStream`: full round-trip matches original bytes
|
||||
- `createWriteStream`: multi-chunk write round-trips correctly via `get()`
|
||||
- AbortSignal on streaming methods
|
||||
- `{ timeout: 30000 }` on all
|
||||
- [x] Create `src/memory/memory-ttl-cleanup.test.ts`:
|
||||
- Expired entries removed by background cleanup (use `FakeClock` via injectable clock or mock `Date.now`)
|
||||
- Stats (`keyCount`, `totalSize`) correctly decremented after cleanup
|
||||
- `cleanupIntervalMs: 0` disables background timer
|
||||
- `{ timeout: 30000 }` on all
|
||||
- [x] Create `src/memory/memory-edge-cases.test.ts`:
|
||||
- Overwrite: `modifiedAt` updates, `createdAt` unchanged
|
||||
- Large value exactly at `maxValueSize` accepted; one byte over rejected
|
||||
- Empty `Uint8Array` stored and retrieved correctly
|
||||
- `clear()` then `getSize()` returns 0
|
||||
- Concurrent async `set`s to the same key settle deterministically (last writer wins in single-threaded JS)
|
||||
- `{ timeout: 30000 }` on all
|
||||
- [x] Create `src/memory/memory-atomic.test.ts`:
|
||||
- `getdel()` is logically atomic: value returned and key absent after call
|
||||
- `mdelete()` returns correct count for mix of existing and missing keys
|
||||
- `{ timeout: 5000 }` / `{ timeout: 30000 }` as appropriate
|
||||
|
||||
---
|
||||
|
||||
## 5. NamespacedStorage (integration tests)
|
||||
|
||||
- [x] Create `tests/namespace-integration.test.ts` (in `tests/` sibling, cross-cutting):
|
||||
- Full round-trip using `MemoryStorage` as the backing store: `set`→`get`→`getWithMetadata`→`delete`→`exists`
|
||||
- `list()` with prefix filter, pagination
|
||||
- Verify no key leakage: `ns('a').set('k', v)` is not visible via `ns('b').get('k')`
|
||||
- `getSize()` returns bytes for the namespace only
|
||||
- Nested namespaces: `ns1.namespace('sub')` works correctly
|
||||
- `{ timeout: 30000 }` on all
|
||||
|
||||
---
|
||||
|
||||
## 6. StorageFactory + Provider Registry
|
||||
|
||||
### 6a. Implementation (`src/factory/factory.ts`)
|
||||
|
||||
- [x] Module-level `providers = new Map<string, new (config?: unknown) => AbstractStorage>()`
|
||||
- [x] Auto-register `"memory"` → `MemoryStorage` at module initialisation (this is the only side effect; it is isolated to the factory module and does not affect tree-shaking of other exports)
|
||||
- [x] Implement all factory functions from the preserved surface (see proposal):
|
||||
- `registerStorageProvider<T extends string>(type, implementation): void`
|
||||
- `createStorage<T extends StorageProviderType>(options): StorageProviderImplementation<T>` — throws `StorageError(STORAGE.GENERIC)` if provider not found (not raw `Error`)
|
||||
- `createStorageWithConfig<T extends StorageProviderType>(type, config?): StorageProviderImplementation<T>`
|
||||
- `hasStorageProvider(type): boolean`
|
||||
- `getStorageProviders(): string[]`
|
||||
- `unregisterStorageProvider(type): void`
|
||||
- `clearStorageProviders(): void` — resets to memory-only
|
||||
- [x] `const StorageFactory = { register, create, createWithConfig, hasProvider, getProviders, unregister, clearProviders } as const`
|
||||
|
||||
### 6b. Factory types (`src/factory/factory.types.ts`)
|
||||
|
||||
- [x] `StorageProviderRegistry` interface with `memory: MemoryStorage`
|
||||
- [x] `type InMemory = MemoryStorage` — alias export for backwards compat
|
||||
- [x] `StorageProviderType`, `StorageProviderImplementation<T>`, `StorageProviderConfig`, `MemoryStorageConfig` (re-export from `memory.types.ts`), `StorageProviderConfigFor<T>`, `StorageFactoryConfig<T>`
|
||||
|
||||
### 6c. Docs and tests
|
||||
|
||||
- [x] Create `src/factory/factory.md` (≥ 200 words, module-augmentation registration pattern example)
|
||||
- [x] Create `src/factory/index.ts` barrel
|
||||
- [x] Create `src/factory/factory.test.ts`:
|
||||
- `createStorage({ type: 'memory' })` returns a `MemoryStorage` instance
|
||||
- `createStorage` with unregistered type throws `StorageError` with code `STORAGE.GENERIC`
|
||||
- `registerStorageProvider` + `createStorage` with custom provider works
|
||||
- `hasStorageProvider` / `getStorageProviders` / `unregisterStorageProvider` / `clearStorageProviders` correctness
|
||||
- `StorageFactory` facade delegates to the same functions
|
||||
- `InMemory` type alias is exported (type-level test using `satisfies`)
|
||||
- `{ timeout: 5000 }` on all
|
||||
|
||||
---
|
||||
|
||||
## 7. OTel Instrumentation Verification
|
||||
|
||||
- [x] Create `tests/otel-instrumentation.test.ts`:
|
||||
- Use a `TestSpanExporter` or in-memory OTel SDK setup from `@suaveplan/opentelemetry` (if available) or a minimal no-op; verify that span event / metric recording code paths are exercised without throwing
|
||||
- Assert `storage.ops` counter is incremented by 1 for each of: `get`, `set`, `delete`, `exists`, `list`, `clear`
|
||||
- Assert `storage.op.duration_ms` histogram is recorded for each operation
|
||||
- Assert `storage.value.bytes` histogram is recorded for `get` and `set` with the correct byte count
|
||||
- `{ timeout: 30000 }` on all
|
||||
|
||||
---
|
||||
|
||||
## 8. Property-Based Tests
|
||||
|
||||
- [x] Create `tests/property-based.test.ts`:
|
||||
- **Key validation**: `fc.string()` — `validateKey` accepts all valid keys; rejects all strings containing `..`, `\`, `/`, `<>"|*?`, or longer than 1024 chars
|
||||
- **Set/get roundtrip**: `fc.tuple(fc.string({ minLength: 1, maxLength: 100 }).filter(k => !k.includes('..')), fc.uint8Array())` — `set(k, v)` then `get(k)` returns equal `Uint8Array`
|
||||
- **Namespace roundtrip**: `fc.tuple(fc.string(...), fc.string(...), fc.uint8Array())` — `namespace(p).set(k, v)` then `namespace(p).get(k)` equals `v`; `namespace(p2).get(k)` returns `undefined` for `p2 !== p`
|
||||
- **listWithPattern consistency**: for any store state and glob pattern, `listWithPattern({ pattern })` result is a subset of `list()` result and matches the pattern on every key
|
||||
- **Batch rollback**: inject a failure at a random operation index; assert all keys revert to pre-batch state
|
||||
- **TTL expiry**: `fc.nat({ max: 100 })` for ttl; advance time past ttl; `get` must return `undefined`
|
||||
- `{ timeout: 30000 }` on the `describe` block
|
||||
|
||||
---
|
||||
|
||||
## 9. Benchmarks
|
||||
|
||||
- [x] Create `src/__benchmarks__/memory-ops.bench.ts`:
|
||||
- Benchmark `MemoryStorage` `get`, `set`, `delete` individually (1 000-key warm store)
|
||||
- Benchmark `list()` on a 10 000-key store with and without prefix
|
||||
- Use `mitata` `bench` and `group` API
|
||||
- [x] Create `src/__benchmarks__/batch-ops.bench.ts`:
|
||||
- Benchmark `batch()` of 100 mixed set/get/delete operations vs equivalent sequential calls
|
||||
- Measure rollback overhead by injecting a failure at index 50
|
||||
- [x] Create `src/__benchmarks__/namespace-ops.bench.ts`:
|
||||
- Compare `MemoryStorage.get/set` direct vs through `NamespacedStorage` wrapper
|
||||
- `list()` with namespace prefix vs without
|
||||
- [x] Add `"test:bench": "vitest bench"` script to `package.json`
|
||||
- [x] Confirm `bun run test:bench --filter=@suaveplan/storage` runs without error and prints numbers
|
||||
|
||||
---
|
||||
|
||||
## 10. Docs Coverage Pass
|
||||
|
||||
- [x] Ensure every `.ts` implementation file has a sibling `.md` (no co-located `.types.ts`, `.test.ts`, barrel `index.ts`, or `__benchmarks__` files require `.md`):
|
||||
- `src/errors/errors.md` ✓ (Task 2a)
|
||||
- `src/types/types.md` ✓ (Task 2b)
|
||||
- `src/types/schemas.md` ✓ (Task 2b)
|
||||
- `src/abstract/abstract.md` ✓ (Task 3d)
|
||||
- `src/abstract/namespace.md` ✓ (Task 3d)
|
||||
- `src/abstract/telemetry.md` (if telemetry extracted to its own file)
|
||||
- `src/memory/memory.md` ✓ (Task 4c)
|
||||
- `src/factory/factory.md` ✓ (Task 6c)
|
||||
- [x] Write root `README.md` at `universal/storage/storage/README.md`:
|
||||
- Package purpose (2+ sentences)
|
||||
- Installation snippet
|
||||
- Quick-start example (MemoryStorage + namespace + TTL)
|
||||
- Streaming example (createReadStream / createWriteStream)
|
||||
- Factory / custom provider registration example
|
||||
- Error handling example (StorageNotFoundError, StorageQuotaError)
|
||||
- Links to each co-located `.md` file
|
||||
- Dependency table
|
||||
- No category-level README inside `src/` subdirectories
|
||||
- [x] Run `bun run docs:check --filter=@suaveplan/storage` and fix any missing doc symbols
|
||||
|
||||
---
|
||||
|
||||
## 11. Verify and Publish
|
||||
|
||||
- [x] Run the full pre-publish gate — all four commands must exit 0:
|
||||
```bash
|
||||
bunx turbo lint --filter=@suaveplan/storage
|
||||
bunx turbo typecheck --filter=@suaveplan/storage
|
||||
bunx turbo test --filter=@suaveplan/storage
|
||||
bunx turbo test:coverage --filter=@suaveplan/storage
|
||||
```
|
||||
- [x] Confirm coverage thresholds 100/100/100/100 are met on all touched modules (istanbul reporter)
|
||||
- [x] Confirm `bun run docs:check --filter=@suaveplan/storage` reports 100%
|
||||
- [x] Confirm `bun run test:bench --filter=@suaveplan/storage` runs to completion and outputs benchmark numbers
|
||||
- [x] Confirm `sideEffects: false` is set in `package.json` and tree-shake guard test passes
|
||||
- [x] Review `src/` for any `TODO`, `FIXME`, `@deprecated`, `any`, `ts-ignore`, or `it.skip` / `describe.skip` — fix all before proceeding
|
||||
- [x] Add `@suaveplan/storage` to `CLAUDE.md` package listing (under Universal packages) with its full description and dep list
|
||||
- [x] Publish: `bun scripts/publish.ts --package @suaveplan/storage`
|
||||
- [x] Archive this OpenSpec change: move `openspec/changes/enterprise-storage/` to `openspec/archive/2026-04-23-enterprise-storage/` and fold spec deltas into `openspec/specs/storage/spec.md`
|
||||
Reference in New Issue
Block a user