Initial Commit

scripts/openspec-conformance.ts

@@ -0,0 +1,582 @@
+#!/usr/bin/env bun
+/**
+ * openspec-conformance — assert that a package's implementation actually
+ * matches its OpenSpec contract.
+ *
+ * The hygiene gates (lint, typecheck, coverage, docs:check) verify that
+ * something exists and compiles. They cannot tell whether the spec's
+ * MUST clauses are reflected in code. This script closes that gap with
+ * five static checks:
+ *
+ *   1. **Spec parsing** — every `### CAP-NN —` requirement in the
+ *      relevant spec(s) must carry a `**Verified by:** path::"name"` line.
+ *   2. **Error codes thrown** — every code in `src/error-codes.ts` is
+ *      thrown from at least one non-test source file.
+ *   3. **Schemas at boundaries** — every schema exported from
+ *      `src/schemas/` is invoked via `.parse(` / `.safeParse(` at a
+ *      non-test call site.
+ *   4. **No `: unknown` in public d.ts** — public types must not leak
+ *      `unknown` (often a sign of a generic that wasn't threaded
+ *      through). Limited to the package's own `dist/`, ignoring
+ *      bundled vendor `.d.ts` files.
+ *   5. **Side-effect code registration** — if `src/error-codes.ts`
+ *      exists, `src/index.ts` MUST side-effect-import it.
+ *   6. **No retroactive ticking** — `tasks.md` of any change folder
+ *      pointing at this package must not contain the literal string
+ *      "retroactively ticked".
+ *
+ * Usage:
+ *   bun scripts/openspec-conformance.ts <pkg-name|pkg-dir>
+ *   bun scripts/openspec-conformance.ts --all
+ *   bun scripts/openspec-conformance.ts --json <pkg>
+ *
+ * Exits 0 on full pass, 1 on any failure.
+ */
+
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { basename, join, relative, resolve } from "node:path";
+
+const ROOT = resolve(import.meta.dir, "..");
+const ARCHIVE_DIR = join(ROOT, "openspec", "archive");
+const CHANGES_DIR = join(ROOT, "openspec", "changes");
+
+type Severity = "error" | "warn" | "info";
+
+interface Finding {
+    severity: Severity;
+    rule: string;
+    message: string;
+    evidence?: string;
+}
+
+interface Report {
+    pkgName: string;
+    pkgDir: string;
+    findings: Finding[];
+    skipped: string[];
+}
+
+// ── Argument parsing ──────────────────────────────────────────────────────────
+
+function parseArgs(): { targets: string[]; json: boolean; all: boolean } {
+    const args = process.argv.slice(2);
+    let json = false;
+    let all = false;
+    const targets: string[] = [];
+    for (const a of args) {
+        if (a === "--json") json = true;
+        else if (a === "--all") all = true;
+        else targets.push(a);
+    }
+    return { targets, json, all };
+}
+
+// ── Package resolution ────────────────────────────────────────────────────────
+
+function findPackageDir(target: string): string {
+    // Allow absolute / relative path
+    const asPath = resolve(target);
+    if (existsSync(join(asPath, "package.json"))) return asPath;
+
+    // Strip @suaveplan/ prefix if present
+    const bare = target.replace(/^@suaveplan\//, "");
+    const pkgsRoot = join(ROOT, "packages");
+    const groups = readdirSync(pkgsRoot, { withFileTypes: true })
+        .filter((d) => d.isDirectory())
+        .map((d) => d.name);
+    for (const g of groups) {
+        const candidate = join(pkgsRoot, g, bare);
+        if (existsSync(join(candidate, "package.json"))) return candidate;
+    }
+    throw new Error(`package not found: ${target}`);
+}
+
+function readPkgName(pkgDir: string): string {
+    const pkg = JSON.parse(readFileSync(join(pkgDir, "package.json"), "utf8"));
+    return pkg.name as string;
+}
+
+function listAllPackages(): string[] {
+    const pkgsRoot = join(ROOT, "packages");
+    const out: string[] = [];
+    for (const g of readdirSync(pkgsRoot, { withFileTypes: true })) {
+        if (!g.isDirectory()) continue;
+        const groupDir = join(pkgsRoot, g.name);
+        for (const p of readdirSync(groupDir, { withFileTypes: true })) {
+            if (!p.isDirectory()) continue;
+            if (existsSync(join(groupDir, p.name, "package.json"))) {
+                out.push(join(groupDir, p.name));
+            }
+        }
+    }
+    return out;
+}
+
+// ── File walker (src/) ────────────────────────────────────────────────────────
+
+function walkSrc(dir: string, out: string[] = []): string[] {
+    if (!existsSync(dir)) return out;
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        // Avoid descending into vendored deps or build outputs that may
+        // appear under src/ in some packages.
+        if (
+            entry.name === "node_modules" ||
+            entry.name === "dist" ||
+            entry.name === ".turbo"
+        )
+            continue;
+        const p = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walkSrc(p, out);
+        } else if (
+            /\.tsx?$/.test(entry.name) &&
+            !/\.(test|spec|bench|stories)\.tsx?$/.test(entry.name)
+        ) {
+            out.push(p);
+        }
+    }
+    return out;
+}
+
+// Cache per-package src reads — checkErrorCodes and checkSchemas both want
+// the full source list, and readFileSync over many files is the dominant
+// per-package cost on `--all` runs.
+const srcCache = new Map<string, { path: string; content: string }[]>();
+function readAllSrc(pkgDir: string): { path: string; content: string }[] {
+    const cached = srcCache.get(pkgDir);
+    if (cached) return cached;
+    const files = walkSrc(join(pkgDir, "src"));
+    const out = files.map((f) => ({
+        path: f,
+        content: readFileSync(f, "utf8"),
+    }));
+    srcCache.set(pkgDir, out);
+    return out;
+}
+
+// ── Check: error codes thrown ─────────────────────────────────────────────────
+
+function checkErrorCodes(pkgDir: string, findings: Finding[]): void {
+    const codesPath = join(pkgDir, "src", "error-codes.ts");
+    if (!existsSync(codesPath)) return;
+    const content = readFileSync(codesPath, "utf8");
+    const m = content.match(
+        /defineCodes\(\s*["'][^"']+["']\s*,\s*\[([\s\S]*?)\]\s*as const\s*\)/
+    );
+    if (!m) {
+        findings.push({
+            severity: "warn",
+            rule: "error-codes",
+            message:
+                "src/error-codes.ts present but defineCodes(...) call not parseable",
+            evidence: relative(ROOT, codesPath),
+        });
+        return;
+    }
+    const codeNames = [...(m[1] as string).matchAll(/["']([A-Z0-9_]+)["']/g)]
+        .map((c) => c[1] as string)
+        .filter(Boolean);
+    if (codeNames.length === 0) return;
+
+    const sources = readAllSrc(pkgDir).filter(
+        (s) => !s.path.endsWith("error-codes.ts")
+    );
+    for (const code of codeNames) {
+        const re = new RegExp(`\\b${code}\\b`);
+        const hit = sources.find((s) => re.test(s.content));
+        if (!hit) {
+            findings.push({
+                severity: "error",
+                rule: "error-codes-thrown",
+                message: `error code ${code} is registered but never used in src/`,
+            });
+        }
+    }
+}
+
+// ── Check: schemas at boundaries ──────────────────────────────────────────────
+
+function checkSchemas(pkgDir: string, findings: Finding[]): void {
+    const schemasDir = join(pkgDir, "src", "schemas");
+    if (!existsSync(schemasDir)) return;
+    const schemaFiles = walkSrc(schemasDir);
+    if (schemaFiles.length === 0) return;
+
+    const schemaNames: string[] = [];
+    for (const f of schemaFiles) {
+        const content = readFileSync(f, "utf8");
+        for (const m of content.matchAll(
+            /\bexport\s+const\s+([A-Z]\w*Schema)\b/g
+        )) {
+            schemaNames.push(m[1] as string);
+        }
+    }
+
+    const sources = readAllSrc(pkgDir).filter(
+        (s) => !s.path.startsWith(schemasDir)
+    );
+    for (const name of schemaNames) {
+        const re = new RegExp(`\\b${name}\\.(safe)?[Pp]arse\\(`);
+        const hit = sources.find((s) => re.test(s.content));
+        if (!hit) {
+            findings.push({
+                severity: "warn",
+                rule: "schemas-at-boundaries",
+                message: `${name} is exported but never invoked outside src/schemas/. If it is consumer-facing only, mark with a "// schema: boundary-only" comment in the schema file to suppress.`,
+            });
+        }
+    }
+}
+
+// ── Check: side-effect import in index ────────────────────────────────────────
+
+function checkSideEffectImport(pkgDir: string, findings: Finding[]): void {
+    const codes = join(pkgDir, "src", "error-codes.ts");
+    if (!existsSync(codes)) return;
+    const idx = join(pkgDir, "src", "index.ts");
+    if (!existsSync(idx)) return;
+    const content = readFileSync(idx, "utf8");
+    const sideEffect = /^\s*import\s+["']\.\/error-codes(?:\.js)?["']\s*;?$/m;
+    if (!sideEffect.test(content)) {
+        findings.push({
+            severity: "error",
+            rule: "side-effect-codes-import",
+            message:
+                'src/index.ts must side-effect-import "./error-codes.js" so subpath consumers register codes too. Add: import "./error-codes.js";',
+        });
+    }
+}
+
+// ── Check: : unknown in public d.ts ───────────────────────────────────────────
+
+function checkPublicUnknown(pkgDir: string, findings: Finding[]): void {
+    const dist = join(pkgDir, "dist");
+    if (!existsSync(dist)) {
+        findings.push({
+            severity: "info",
+            rule: "public-unknown",
+            message:
+                "dist/ not found — run `bunx turbo build` to enable public-type leakage check",
+        });
+        return;
+    }
+    const dts: string[] = [];
+    walkDts(dist, dts);
+    let leakCount = 0;
+    const samples: string[] = [];
+    for (const f of dts) {
+        // Skip bundled vendor d.ts files (the build pulls in @opentelemetry etc.)
+        if (f.includes("/node_modules/")) continue;
+        const lines = readFileSync(f, "utf8").split("\n");
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i] as string;
+            // Skip private/protected members and comment lines
+            if (/^\s*(private|protected|\/\/|\/\*)/.test(line)) continue;
+            // Match `: unknown` followed by `;`, `,`, `)`, `=`, `|`, end-of-line
+            // but not as part of `Record<string, unknown>` / `unknown[]`.
+            // Allow standard ES `Error.cause` shape (`cause?: unknown`),
+            // which is the spec-mandated type for Error.cause and not
+            // a sign of generics drift.
+            if (/\bcause\??\s*:\s*unknown\b/.test(line)) continue;
+            if (/:\s*unknown\s*[;,)|=]?\s*$/.test(line)) {
+                leakCount++;
+                if (samples.length < 5) {
+                    samples.push(
+                        `${relative(ROOT, f)}:${i + 1}: ${line.trim()}`
+                    );
+                }
+            }
+        }
+    }
+    if (leakCount > 0) {
+        findings.push({
+            severity: "warn",
+            rule: "public-unknown",
+            message: `${leakCount} public position(s) in dist/*.d.ts resolve to "unknown". This often means a generic was declared in the spec but not threaded through the public API.`,
+            evidence: samples.join("\n  "),
+        });
+    }
+}
+
+function walkDts(dir: string, out: string[]): void {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        // Skip vendored deps — they cost the bulk of the wall-clock for
+        // packages that bundle large dependency graphs (@opentelemetry,
+        // libsodium, sharp). The leak check filters these on the path
+        // anyway; pruning here saves the readdirSync recursion.
+        if (entry.name === "node_modules") continue;
+        const p = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walkDts(p, out);
+        } else if (entry.name.endsWith(".d.ts")) {
+            out.push(p);
+        }
+    }
+}
+
+// ── Check: spec requirements have Verified-by ────────────────────────────────
+
+function findRelatedChangeFolders(pkgName: string): string[] {
+    const bare = pkgName.replace(/^@suaveplan\//, "");
+    const out: string[] = [];
+    // Authoritative match: a change folder is "related" to this package
+    // only if its proposal.md declares `**Package**: @suaveplan/<bare>`
+    // in the frontmatter (first 30 lines). Folder-name matching is
+    // unreliable — e.g. `enterprise-testing-dom`, `enterprise-ab-testing`,
+    // and `enterprise-network-testing` all contain the `testing` token
+    // but belong to different packages/capabilities.
+    // Match list-item or bare frontmatter line:
+    //   "- **Package**: `@suaveplan/<bare>`"
+    //   "**Package**: @suaveplan/<bare>"
+    const frontmatterRe = new RegExp(
+        `(?:^|\\n)\\s*-?\\s*\\*\\*Package\\*\\*:\\s*\`?@suaveplan/${escapeRe(bare)}\`?\\s*(?:$|\\n)`
+    );
+    for (const root of [ARCHIVE_DIR, CHANGES_DIR]) {
+        if (!existsSync(root)) continue;
+        for (const entry of readdirSync(root, { withFileTypes: true })) {
+            if (!entry.isDirectory()) continue;
+            const folder = join(root, entry.name);
+            const proposal = join(folder, "proposal.md");
+            if (!existsSync(proposal)) continue;
+            const content = readFileSync(proposal, "utf8");
+            const head = content.split("\n").slice(0, 30).join("\n");
+            if (frontmatterRe.test(head)) out.push(folder);
+        }
+    }
+    return out;
+}
+
+function escapeRe(s: string): string {
+    return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function findRelatedSpecCapabilities(pkgName: string): string[] {
+    // Map related change folders → capability names referenced in their
+    // specs/<cap>/ subdirs. The folded source-of-truth specs live under
+    // `openspec/specs/<cap>/spec.md`.
+    const folders = findRelatedChangeFolders(pkgName);
+    const caps = new Set<string>();
+    for (const folder of folders) {
+        const specsDir = join(folder, "specs");
+        if (!existsSync(specsDir)) continue;
+        for (const cap of readdirSync(specsDir, { withFileTypes: true })) {
+            if (cap.isDirectory()) caps.add(cap.name);
+        }
+    }
+    // Fallback: package basename as capability name (common pattern).
+    const bare = pkgName.replace(/^@suaveplan\//, "");
+    if (existsSync(join(ROOT, "openspec", "specs", bare))) caps.add(bare);
+    return [...caps];
+}
+
+function checkSpecVerifiedBy(
+    pkgName: string,
+    pkgDir: string,
+    findings: Finding[]
+): void {
+    const caps = findRelatedSpecCapabilities(pkgName);
+    if (caps.length === 0) {
+        findings.push({
+            severity: "info",
+            rule: "spec-verified-by",
+            message:
+                "no related OpenSpec capability found; spec-conformance skipped",
+        });
+        return;
+    }
+    // Authoritative spec sources: the folded `openspec/specs/<cap>/spec.md`
+    // PLUS any active change in `openspec/changes/<id>/specs/<cap>/spec.md`
+    // whose proposal.md frontmatter explicitly names THIS package. Active
+    // changes that touch the same capability but belong to a different
+    // package (e.g. `enterprise-testing-integration` writes deltas under
+    // `specs/testing/` while owning a sibling capability) MUST NOT be
+    // pulled in here — that would force this package's audit to verify
+    // requirements another change is on the hook for.
+    const relatedFolders = new Set(findRelatedChangeFolders(pkgName));
+    const specPaths: string[] = [];
+    for (const cap of caps) {
+        const folded = join(ROOT, "openspec", "specs", cap, "spec.md");
+        if (existsSync(folded)) specPaths.push(folded);
+    }
+    for (const folder of relatedFolders) {
+        if (!folder.startsWith(CHANGES_DIR)) continue;
+        for (const cap of caps) {
+            const spec = join(folder, "specs", cap, "spec.md");
+            if (existsSync(spec)) specPaths.push(spec);
+        }
+    }
+
+    let totalReqs = 0;
+    let unverified = 0;
+    const samples: string[] = [];
+    for (const spec of specPaths) {
+        const content = readFileSync(spec, "utf8");
+        const blocks = splitByHeading(content, /^###\s+(.+?)\s*$/m);
+        for (const { heading, body } of blocks) {
+            // Accept two requirement-heading styles:
+            //   "CAP-NN — <name>"        (e.g. "I18N-01 — Structured errors")
+            //   "Requirement: <name>"    (events / prose style)
+            const isStructured = /^[A-Z][A-Z0-9]*-\d+\b/.test(heading);
+            const isProse = /^Requirement:\s+/i.test(heading);
+            if (!isStructured && !isProse) continue;
+            totalReqs++;
+            if (!/\*\*Verified by:\*\*/i.test(body)) {
+                unverified++;
+                if (samples.length < 5) {
+                    samples.push(`${relative(ROOT, spec)}: ${heading}`);
+                }
+            }
+        }
+    }
+    if (totalReqs === 0) {
+        findings.push({
+            severity: "info",
+            rule: "spec-verified-by",
+            message:
+                "no `### CAP-NN —` requirements parsed in folded specs/ or active changes/ for this package",
+        });
+        return;
+    }
+    if (unverified > 0) {
+        findings.push({
+            severity: "error",
+            rule: "spec-verified-by",
+            message: `${unverified}/${totalReqs} requirement(s) lack a **Verified by:** line in folded spec or active change. Add a line naming the test that asserts the requirement.`,
+            evidence: samples.join("\n  "),
+        });
+    }
+    void pkgDir;
+}
+
+function splitByHeading(
+    content: string,
+    re: RegExp
+): { heading: string; body: string }[] {
+    const lines = content.split("\n");
+    const out: { heading: string; body: string }[] = [];
+    let current: { heading: string; body: string[] } | null = null;
+    for (const line of lines) {
+        const m = line.match(re);
+        if (m) {
+            if (current)
+                out.push({
+                    heading: current.heading,
+                    body: current.body.join("\n"),
+                });
+            current = { heading: m[1] as string, body: [] };
+        } else if (current) {
+            current.body.push(line);
+        }
+    }
+    if (current)
+        out.push({
+            heading: current.heading,
+            body: current.body.join("\n"),
+        });
+    return out;
+}
+
+// ── Check: no retroactive ticking ─────────────────────────────────────────────
+
+function checkNoRetroactiveTicking(pkgName: string, findings: Finding[]): void {
+    const bare = pkgName.replace(/^@suaveplan\//, "");
+    const folders = findRelatedChangeFolders(pkgName);
+    // A *-spec-conformance change (active or archived) rectifies prior drift.
+    const hasRectification = folders.some((f) =>
+        /spec-conformance|conformance/.test(basename(f))
+    );
+    for (const folder of folders) {
+        const tasks = join(folder, "tasks.md");
+        if (!existsSync(tasks)) continue;
+        const content = readFileSync(tasks, "utf8");
+        if (!/retroactively ticked/i.test(content)) continue;
+        const isHistorical = folder.startsWith(ARCHIVE_DIR);
+        if (isHistorical && hasRectification) {
+            findings.push({
+                severity: "info",
+                rule: "no-retroactive-ticking",
+                message: `${relative(ROOT, tasks)} was retroactively ticked, but a *-spec-conformance change has rectified it.`,
+            });
+        } else {
+            findings.push({
+                severity: "error",
+                rule: "no-retroactive-ticking",
+                message: `${relative(ROOT, tasks)} contains the phrase "retroactively ticked" — open a ${bare}-spec-conformance change before more migrations.`,
+            });
+        }
+    }
+}
+
+// ── Run ───────────────────────────────────────────────────────────────────────
+
+function runOne(pkgDir: string): Report {
+    const pkgName = readPkgName(pkgDir);
+    const findings: Finding[] = [];
+    const skipped: string[] = [];
+    checkErrorCodes(pkgDir, findings);
+    checkSchemas(pkgDir, findings);
+    checkSideEffectImport(pkgDir, findings);
+    checkPublicUnknown(pkgDir, findings);
+    checkSpecVerifiedBy(pkgName, pkgDir, findings);
+    checkNoRetroactiveTicking(pkgName, findings);
+    return { pkgName, pkgDir, findings, skipped };
+}
+
+function printReport(report: Report): boolean {
+    const errors = report.findings.filter((f) => f.severity === "error");
+    const warns = report.findings.filter((f) => f.severity === "warn");
+    const infos = report.findings.filter((f) => f.severity === "info");
+
+    const status = errors.length === 0 ? "PASS" : "FAIL";
+    const summary = `${status}  ${report.pkgName}  (${errors.length} error, ${warns.length} warn, ${infos.length} info)`;
+    process.stdout.write(`${summary}\n`);
+    for (const f of report.findings) {
+        const tag =
+            f.severity === "error" ? "✖" : f.severity === "warn" ? "!" : "·";
+        process.stdout.write(`  ${tag} [${f.rule}] ${f.message}\n`);
+        if (f.evidence) {
+            for (const line of f.evidence.split("\n")) {
+                process.stdout.write(`      ${line}\n`);
+            }
+        }
+    }
+    return errors.length === 0;
+}
+
+function main(): void {
+    const { targets, json, all } = parseArgs();
+    const dirs = all
+        ? listAllPackages()
+        : targets.map((t) => findPackageDir(t));
+    if (dirs.length === 0) {
+        process.stderr.write(
+            "usage: bun scripts/openspec-conformance.ts <pkg-name|pkg-dir> [--json] | --all\n"
+        );
+        process.exit(2);
+    }
+    // Stream per-package: run + print as we go so `--all` produces output
+    // immediately instead of buffering until every package has been scanned.
+    const reports: Report[] = [];
+    let failedCount = 0;
+    for (const dir of dirs) {
+        const report = runOne(dir);
+        reports.push(report);
+        if (!json) {
+            printReport(report);
+        }
+        if (report.findings.some((f) => f.severity === "error")) {
+            failedCount++;
+        }
+    }
+    if (json) {
+        process.stdout.write(`${JSON.stringify(reports, null, 2)}\n`);
+    }
+    if (!json && all) {
+        process.stdout.write(
+            `\n=== ${reports.length - failedCount}/${reports.length} packages pass ===\n`
+        );
+    }
+    process.exit(failedCount === 0 ? 0 : 1);
+}
+
+main();