Initial Commit

docs/drive/dehydrated-drives.md

@@ -0,0 +1,470 @@
+# Dehydrated Drives — Templates, Policies, and CDN Fallback
+
+A **dehydrated drive** is a drive whose folder structure and file metadata are pre-configured by a server-signed template, while the actual file bytes arrive later from peers, the CDN, or local writes. The user sees the full directory tree on day one — files have correct names, sizes, and lock states — and chunks populate transparently as they're requested.
+
+This is the answer to "ship a 2 GB game asset bundle to N players" and "subscribe a user to a curated drive without copying everything up front." It is the resolution of [`system-review.md`](../system-review.md) §3, §4, §5, §8, §9, §12, §13, §16. The spec belongs in `openspec/specs/drive-template/spec.md` once the proposal lands.
+
+---
+
+## 1. The lifecycle
+
+```text
+┌──────────────────────────────────────────────────────────────────┐
+│ Server                                                           │
+│  - Curates the drive: which files, which folders, which chunks   │
+│  - Computes hashes, sizes, chunk counts                          │
+│  - Signs DriveTemplate with server Ed25519 key                   │
+│  - Publishes template (via API, share link, or peer broadcast)   │
+└──────────────────────────┬───────────────────────────────────────┘
+                           │ signed DriveTemplate (~10-100 KB)
+                           ▼
+┌──────────────────────────────────────────────────────────────────┐
+│ Client                                                           │
+│  drive.hydrateFrom(template)                                     │
+│   ├─ verify server signature                                     │
+│   ├─ instantiate DirIndex from FolderTemplate tree               │
+│   ├─ instantiate FileLocationIndex from FileTemplate stubs       │
+│   │   (every entry's `storageHints` starts with kind:'mesh' and  │
+│   │    optionally kind:'cdn')                                    │
+│   ├─ unwrap inline keys → KeyRing (lock icon flips per file)     │
+│   ├─ for each file with policy.autoDownload === true:            │
+│   │   schedule a fetch via mesh-protocol (see mesh-protocol.md)  │
+│   │   on miss, fall through to CDN                               │
+│   └─ emit `drive:hydrated` event                                 │
+└──────────────────────────────────────────────────────────────────┘
+```
+
+After hydration, the drive looks identical to one that was populated by writes. The user calls `read("/photo.jpg")` the same way; the read pipeline transparently fetches missing chunks. The dehydrated state is invisible at the API surface — only the `presence` field on DirEntries hints that bytes haven't arrived.
+
+---
+
+## 2. The `DriveTemplate` shape
+
+```ts
+interface DriveTemplate {
+  schemaVersion: 1;                    // pinned; future versions are new packages
+  driveId: string;                     // unique drive identity (32-byte hex)
+  rootFolder: FolderTemplate;
+  cdnFallback?: CdnFallbackConfig;
+  policy: GlobalPolicy;
+  issuer: string;                      // server pubkey (Ed25519, 32-byte hex)
+  issuedAt: number;                    // unix ms
+  expiresAt: number;                   // unix ms
+  nonce: Uint8Array;                   // 16 bytes; replay protection
+  serverSignature: Uint8Array;         // Ed25519 over canonicalSerialize(this without sig)
+}
+
+interface FolderTemplate {
+  name: string;                        // single path component
+  children: (FolderTemplate | FileTemplate)[];
+  // depth is capped at 32 (validated by DriveTemplateSchema)
+}
+
+interface FileTemplate {
+  name: string;
+  fileLocId: string;                   // primary key into FileLocationIndex
+  fileId: string;                      // 32-byte hex; stable across versions
+  size: number;                        // plaintext bytes
+  totalChunks: number;
+  chunkSize: number;
+  checksum: string;                    // 'blake3:<hex>' over plaintext
+  flags: PerFilePolicy;
+  encryption?: FileEncryption;
+  manifestKey: string;                 // where the chunk manifest lives
+  manifestSig: Uint8Array;             // detached Ed25519 sig over the manifest's content
+}
+
+interface PerFilePolicy {
+  autoDownload: boolean;               // eagerly fetch on hydrate?
+  meshAllowed: boolean;                // may we cd:request from peers?
+  cdnAllowed: boolean;                 // may we hit the CDN?
+  retention?: RetentionPolicy;
+}
+
+type RetentionPolicy =
+  | 'permanent'                        // never evict
+  | 'lru'                              // global byte-budget LRU
+  | { maxAgeMs: number };              // chunks older than N ms eligible for eviction
+
+interface FileEncryption {
+  algorithm: 'xchacha20-poly1305';
+  keyDistribution:
+    | { kind: 'inline'; wrappedKey: Uint8Array }       // shipped in template, see §7
+    | { kind: 'retroactive'; keyAuthority: string };   // peer pubkey will issue grant later
+}
+
+interface GlobalPolicy {
+  defaultPolicy: PerFilePolicy;        // applied to files whose `flags` is partial
+  maxLocalCacheBytes?: number;         // global LRU budget (excludes 'permanent' files)
+  maxConcurrentFetches?: number;       // backpressure on hydrate
+  trustedKeyAuthorities?: string[];    // pubkeys whose key:grant we accept
+}
+
+interface CdnFallbackConfig {
+  baseUrl: string;                     // e.g. 'https://chunks.example.com'
+  pathScheme: 'reverse-id';            // see §4 for the URL convention
+  levels: number;                      // directory depth (typically 3)
+  padding: number;                     // chunk ID zero-pad width (typically 9)
+  uploadEndpoint?: string;             // for write-back of newly-fetched chunks
+}
+```
+
+---
+
+## 3. Signature, replay protection, and validation
+
+### Canonical serialization
+
+The server computes the signature over a deterministic byte representation:
+
+```text
+canonicalSerialize(template) =
+  schemaVersion(varint) ‖
+  driveId(32) ‖
+  rootFolder(canonical-folder) ‖
+  cdnFallback(canonical-cdn-or-empty) ‖
+  policy(canonical-policy) ‖
+  issuer(32) ‖
+  issuedAt(8) ‖
+  expiresAt(8) ‖
+  nonce(16)
+```
+
+Encoded via `@suaveplan/bitpacker-schema` with deterministic field order. JSON is **not** used — it doesn't have a canonical form across implementations.
+
+### Validation order
+
+`drive.hydrateFrom(template)` validates in this exact order; any failure aborts before instantiating anything:
+
+1. `schemaVersion === 1` (else `DRIVE.TEMPLATE_VERSION_UNSUPPORTED`).
+2. `Date.now() < expiresAt` (else `DRIVE.TEMPLATE_EXPIRED`).
+3. `Date.now() - issuedAt < 24*60*60*1000` (24-hour issuance window — `DRIVE.TEMPLATE_STALE`).
+4. `nonce` not seen in last 1024 templates (LRU cache; `DRIVE.TEMPLATE_REPLAY`).
+5. `issuer` ∈ `globalPolicy.trustedKeyAuthorities` (or matches drive-creation key — `DRIVE.TEMPLATE_UNTRUSTED_ISSUER`).
+6. Ed25519 verify `serverSignature` over `canonicalSerialize(template - sig)` (`DRIVE.TEMPLATE_SIG_INVALID`).
+7. Recursive folder depth ≤ 32 (`DRIVE.TEMPLATE_TOO_DEEP`).
+8. Per-file: `chunkSize > 164` (header overhead), `totalChunks * chunkSize >= size`, `manifestKey` non-empty (`DRIVE.TEMPLATE_MALFORMED`).
+
+Only after all eight pass does hydration begin.
+
+### Why nonce + issuedAt
+
+A captured template, even one within its `expiresAt`, cannot be replayed against the same drive because the nonce is recorded. A client moving between devices must request a fresh template; this is by design — it gives the server a chance to update policy.
+
+---
+
+## 4. CDN fallback URL convention
+
+When `policy.cdnAllowed === true` and a chunk is not available from the mesh, the chunks plugin falls back to HTTPS via `drive-transport-https`. The URL convention is **reversed-ID** with configurable depth and padding:
+
+```text
+<baseUrl>/<reversed[0]>/<reversed[1]>/...<reversed[levels-1]>/<fileId>_<chunkIdPadded>.chunk
+```
+
+### Worked example
+
+```text
+fileId       = "abcdef1234567890fedcba0987654321abcdef1234567890fedcba0987654321"
+reversed     = "1234567890abcdef1234567890abcdef1234567890fedcba0987654321fedcba"
+levels       = 3
+padding      = 9
+chunkId      = 47
+
+URL = https://chunks.example.com/1/2/3/<full-fileId>_000000047.chunk
+```
+
+### Why reversed
+
+File IDs in practice cluster by prefix:
+
+- Sequential issuance (`f-000001`, `f-000002`, ...) makes prefix-based hashing put millions of files in the same directory.
+- Timestamp-prefixed IDs (`1729a4f72c3b...` from a sortable UUID) cluster by issuance time.
+- Hash prefixes are uniform but humans tend to name files in consistent ways (creation tools embed magic bytes, etc.).
+
+Reversing distributes leaves randomly across the directory tree regardless of how IDs were generated. The first three reversed characters give 16³ = 4,096 leaf directories — enough to keep any single directory under 250 entries even at 1 M files.
+
+### Configuration trade-offs
+
+| `levels` | Leaf dirs | Chunks/dir at 1M files | When to use |
+|---|---|---|---|
+| 2 | 256 | 3,906 | <100 K files; minimize HTTP path length |
+| 3 | 4,096 | 244 | **default — good balance** |
+| 4 | 65,536 | 15 | >10 M files; inode budget allows |
+
+Padding (9 digits) supports up to 999,999,999 chunks per file (~64 PB at 64 KiB chunks). Not the bottleneck.
+
+### Read protocol
+
+```text
+GET https://chunks.example.com/1/2/3/<fileId>_000000047.chunk
+Range: bytes=0-                  (optional — supports partial chunk fetch)
+Accept: application/octet-stream
+```
+
+Response body is the locked v0x0001 chunk format (`metadata(100) ‖ sig(64) ‖ ciphertext`). Same bytes the mesh would have returned. The drive's `chunks` plugin verifies the Ed25519 sig and AEAD tag identically — the CDN never sees plaintext.
+
+### CDN as cache vs CDN as authority
+
+The CDN is treated as a **cache, not an authority**. If a chunk's CDN copy fails signature verification, the drive:
+
+1. Discards the bytes.
+2. Falls back to the mesh.
+3. Optionally reports the corruption to a configured endpoint.
+
+This means a compromised CDN cannot poison a drive — it can only deny service.
+
+---
+
+## 5. CDN write-back
+
+`CdnFallbackConfig.uploadEndpoint` (optional) lets a drive contribute newly-fetched chunks back to the CDN. This is the path that lets a small group of users seed a dehydrated drive without needing every chunk on the CDN at issuance time.
+
+```text
+POST <baseUrl>/upload
+Content-Type: application/octet-stream
+Authorization: Bearer <CdnUploadToken>
+Body: <chunk bytes — same locked v0x0001 format>
+```
+
+```ts
+interface CdnUploadToken {
+  fileId: string;
+  chunkId: number;
+  chunkHash: string;             // BLAKE3 of the chunk bytes (the CDN will verify)
+  uploaderPubkey: string;
+  expiresAt: number;             // 60-second expiry by default
+  serverSignature: Uint8Array;   // Ed25519 by the drive's issuing server
+}
+```
+
+The CDN:
+
+1. Validates the token signature against the drive's known issuer key.
+2. Hashes the body, compares to `chunkHash` — reject on mismatch.
+3. Writes to the reversed-ID URL.
+
+A CDN that doesn't validate signatures can decline writes (read-only mode); the drive falls back to peer-only seeding for that file.
+
+### Why server-issued tokens
+
+Direct PUT (no token) lets anyone poison the CDN. CORS won't help — attackers bypass it. The token couples the upload to a server-attested intent and a specific `(fileId, chunkId, chunkHash)`, and expires fast enough that a stolen token isn't useful.
+
+---
+
+## 6. Per-file policies in detail
+
+### `autoDownload`
+
+- `true` → on `hydrateFrom()`, schedule a fetch for all chunks not present locally. Honors `maxConcurrentFetches`.
+- `false` → wait until first `read()`. The read triggers the same fetch path; the user pays the latency on first access.
+
+UX: a drive showing a directory listing renders progress for `autoDownload=true` files even if the user hasn't opened them yet.
+
+### `meshAllowed`
+
+- `true` → `drive-mesh` may emit `cd:request` for this file's chunks.
+- `false` → mesh layer skipped entirely; only local + CDN paths attempted.
+
+Use `meshAllowed=false` for confidential files where peers shouldn't even *know* the file exists. The drive's gossip stops at hydration; a peer scraping the network bus never sees a request for this file.
+
+### `cdnAllowed`
+
+- `true` → `drive-transport-https` may fetch from the CDN.
+- `false` → CDN path skipped. Use for files restricted to peer-only distribution (e.g., user-shared content that the CDN doesn't host).
+
+### `retention`
+
+The cache eviction policy. See [`../system-review.md`](../system-review.md) §9.
+
+| Mode | Behavior |
+|---|---|
+| `'permanent'` | Never evict. The drive's job is to keep these chunks forever. |
+| `'lru'` | Eligible for global LRU eviction. When `maxLocalCacheBytes` is exceeded, evict oldest-touched chunks (NOT whole files) until under budget. The file remains visible with reduced `chunksPresent`; re-fetches on next read. |
+| `{ maxAgeMs }` | Chunks older than N ms eligible for eviction at the next GC pass. Useful for ephemeral content (tournaments, event streams). |
+
+Eviction is at chunk granularity, never file granularity. The DirEntry, FileLocation, and manifest stay; only the bytes go. This means a "dehydrated" file gracefully transitions back to "needs-fetch" without index churn.
+
+### Defaults via `GlobalPolicy.defaultPolicy`
+
+A `FileTemplate.flags` may omit any field; the missing fields fill from `defaultPolicy`. This keeps the template compact for drives where every file shares the same policy.
+
+---
+
+## 7. Encryption keys — inline vs retroactive
+
+### Inline (key shipped in template)
+
+```ts
+encryption: {
+  algorithm: 'xchacha20-poly1305',
+  keyDistribution: {
+    kind: 'inline',
+    wrappedKey: Uint8Array,   // wrapped to recipient's X25519 pubkey via crypto_box_seal
+  },
+}
+```
+
+On `hydrateFrom()`, the drive unwraps `wrappedKey` with the recipient's secret key (held in `@suaveplan/crypto-vault`). On success, the unwrapped `fileKey` lands in the drive's KeyRing under `fileId`. The lock icon flips from 🔒 to 🔓 immediately.
+
+This is the simple case: drive operator decides at template-issuance time who gets the key, and the recipient has it from day one.
+
+### Retroactive (key arrives later)
+
+```ts
+encryption: {
+  algorithm: 'xchacha20-poly1305',
+  keyDistribution: {
+    kind: 'retroactive',
+    keyAuthority: '<peer-pubkey>',   // the peer that will issue grants
+  },
+}
+```
+
+On `hydrateFrom()`, the drive:
+
+1. Records the file as encrypted-locked.
+2. Subscribes to `bus:key:grant` events filtered by `fileId`.
+3. Renders 🔒 in the UI.
+
+When the key authority decides to grant access (or the user requests it via a side channel — e.g., paying for content, joining a group), it issues:
+
+```ts
+interface KeyGrantEnvelope extends MessageEnvelope {
+  type: 'key:grant';
+  payload: KeyGrantPayload;
+}
+
+interface KeyGrantPayload {
+  fileId: string;
+  recipient: string;            // recipient pubkey
+  wrappedKey: Uint8Array;       // wrapped to recipient via crypto_box_seal
+  issuer: string;               // keyAuthority pubkey from the template
+  issuerSignature: Uint8Array;  // Ed25519 over (fileId, recipient, wrappedKey, expiresAt)
+  expiresAt: number;            // optional — for time-limited grants
+}
+```
+
+The drive validates:
+
+1. `issuer === template.encryption.keyAuthority` for this file.
+2. `recipient === ourPubkey`.
+3. Ed25519 sig.
+4. `Date.now() < expiresAt` (if set).
+
+On success, unwrap, store in KeyRing, flip 🔒 → 🔓, emit `drive:key:granted` event. UI updates.
+
+### Revocation
+
+A retroactive key grant can carry an `expiresAt`. After that timestamp:
+
+- The drive evicts the unwrapped key from the KeyRing.
+- The lock icon flips back 🔓 → 🔒.
+- A new grant can be issued to extend access.
+
+This is forward revocation only — anything the recipient already decrypted, they have. (See [`mesh-and-sharing.md`](./mesh-and-sharing.md) §"What revocation actually does".) For destructive revocation, rotate the `fileKey` server-side and re-issue the template with a new `manifestSig`.
+
+### Trust roots
+
+`GlobalPolicy.trustedKeyAuthorities` is the set of issuer pubkeys whose `key:grant` envelopes the drive will accept. Typically just `[server-pubkey]`, but multi-issuer setups (a game with multiple content vendors) can list several. The set is fixed at template issuance — adding new issuers requires a fresh template.
+
+---
+
+## 8. The `Drive.hydrateFrom()` API
+
+```ts
+class Drive {
+  hydrateFrom(template: DriveTemplate, options?: HydrateOptions): Promise<HydrateResult>;
+}
+
+interface HydrateOptions {
+  signal?: AbortSignal;
+  onProgress?: (event: HydrateProgressEvent) => void;
+  preFetch?: 'none' | 'metadata-only' | 'auto-download-files';
+}
+
+interface HydrateResult {
+  filesAdded: number;
+  bytesScheduled: number;        // sum of sizes of autoDownload files
+  unlocked: number;              // count of inline-key files unlocked
+  locked: number;                // count of files awaiting retroactive grant
+}
+
+type HydrateProgressEvent =
+  | { kind: 'verified' }
+  | { kind: 'index-built'; entries: number }
+  | { kind: 'fetch-scheduled'; fileId: string }
+  | { kind: 'key-unlocked'; fileId: string };
+```
+
+`preFetch` controls hydration aggressiveness:
+
+- `'none'` — instantiate indexes only; fetch on first read.
+- `'metadata-only'` — fetch all manifests but no chunks. Lets the drive verify file integrity claims without the byte cost.
+- `'auto-download-files'` (default) — honor each file's `policy.autoDownload`.
+
+---
+
+## 9. Worked example — game asset bundle
+
+A game wants to ship a 2 GB asset bundle to 10K players. The server:
+
+1. Chunks every asset file at 64 KiB. ~32K chunks across 200 files.
+2. Computes Ed25519-signed chunk format for each chunk; uploads to the CDN at `/chunks.game.example.com/<reversed-id>/<file>_<chunk>.chunk`.
+3. Computes a `DriveTemplate` listing every asset path, fileId, size, manifest, encryption (none for assets, inline for premium DLC).
+4. Signs the template; expiry 30 days; nonce per player.
+5. Distributes the template via the game launcher.
+
+On the player's machine:
+
+```ts
+const drive = new Drive(opfsStorage)
+  .usePlugin(chunksPlugin({ ... }))
+  .usePlugin(meshPlugin({ networkNode }))
+  .usePlugin(transportWebrtcPlugin({ ... }))
+  .usePlugin(transportHttpsPlugin({ baseUrl: 'https://chunks.game.example.com' }));
+
+await drive.initialize();
+const result = await drive.hydrateFrom(template, {
+  preFetch: 'auto-download-files',
+  onProgress: (e) => updateLoadingScreen(e),
+});
+// result.filesAdded === 200; result.bytesScheduled === 2_000_000_000;
+```
+
+What happens next:
+
+- For files with `autoDownload=true` and `meshAllowed=true`: drive-mesh broadcasts `cd:request`. Other players already running the game (with chunks cached) respond. Bandwidth comes from peers, not the CDN.
+- For files no peer has yet (newest player): drive-transport-https hits the CDN.
+- For files with `autoDownload=false`: deferred until the player navigates to the corresponding game zone; first-access pays a fetch.
+- For DLC with inline encryption: keys unwrap immediately; UI shows 🔓.
+- For event-bound DLC (retroactive keys, e.g., "this skin unlocks during the seasonal event"): UI shows 🔒. When the season starts, the server broadcasts `key:grant`; UIs flip to 🔓.
+
+The CDN's bandwidth bill drops dramatically once the player base has any seed. The mesh handles the cold-start of new geographic clusters. The drive UX is identical regardless of where bytes came from.
+
+---
+
+## 10. Test contract
+
+The `drive-template-v1` change folder will ship:
+
+- `tests/template-roundtrip.test.ts` — canonicalSerialize / sign / verify cycle.
+- `tests/template-replay.test.ts` — second hydrate with same nonce rejected.
+- `tests/template-expired.test.ts` — expired template rejected with correct error code.
+- `tests/template-tampered.test.ts` — bit flip anywhere in the template breaks the sig.
+- `tests/policy-meshallowed-false.test.ts` — `meshAllowed: false` skips bus emit (verified via test bus interceptor).
+- `tests/policy-cdnallowed-false.test.ts` — same for CDN.
+- `tests/retention-lru.test.ts` — eviction respects `maxLocalCacheBytes`; permanent files never evicted.
+- `tests/inline-key-unlock.test.ts` — hydrate flips lock icon; `read()` returns plaintext.
+- `tests/retroactive-key-grant.test.ts` — incoming `key:grant` flips lock icon; `read()` works.
+- `tests/cdn-reverse-id-url.test.ts` — URL builder produces correct path for various `levels`/`padding` combos.
+- `tests/cdn-bad-sig.test.ts` — corrupted CDN response falls back to mesh.
+
+---
+
+## 11. Reading further
+
+- [`mesh-protocol.md`](./mesh-protocol.md) — the chunk-request/response/advertise envelopes that hydration uses.
+- [`encryption.md`](./encryption.md) — chunk format that the templates reference.
+- [`mesh-and-sharing.md`](./mesh-and-sharing.md) — partition-based sharing as the alternative to dehydrated drives.
+- [`../system-review.md`](../system-review.md) — the review this doc resolves.
+- [`../drive-storage-network.md`](../drive-storage-network.md) — the whole composition map; hydration plugs into the read pipeline at the `mesh` layer.