Initial Commit

scripts/lint/check-conformance-tests.ts

@@ -0,0 +1,190 @@
+#!/usr/bin/env bun
+// §16 lint: conformance harnesses live with the contract
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+const ROOT = resolve(import.meta.dir, "../..");
+const PACKAGES_DIR = join(ROOT, "packages");
+
+interface Pkg {
+    readonly name: string;
+    readonly basename: string;
+    readonly dir: string;
+    readonly deps: ReadonlySet<string>;
+    readonly skip: ReadonlySet<string>;
+    readonly hasTesting: boolean;
+}
+
+function findPackages(dir: string, out: Pkg[], depth = 0): void {
+    if (depth > 5) return;
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+        if (
+            e.name === "node_modules" ||
+            e.name === "dist" ||
+            e.name.startsWith(".")
+        )
+            continue;
+        const full = join(dir, e.name);
+        if (!e.isDirectory()) continue;
+        const pj = join(full, "package.json");
+        try {
+            if (statSync(pj).isFile()) {
+                const raw = JSON.parse(readFileSync(pj, "utf-8"));
+                const name = raw.name as string | undefined;
+                if (name?.startsWith("@suaveplan/")) {
+                    out.push({
+                        name,
+                        basename: name.slice("@suaveplan/".length),
+                        dir: full,
+                        deps: new Set(
+                            Object.keys(raw.dependencies ?? {}).filter(
+                                (n: string) => n.startsWith("@suaveplan/")
+                            )
+                        ),
+                        skip: new Set(raw.suaveplan?.conformanceSkip ?? []),
+                        hasTesting: Object.keys(raw.exports ?? {}).includes(
+                            "./testing"
+                        ),
+                    });
+                }
+                continue;
+            }
+        } catch {
+            /* not a pkg */
+        }
+        findPackages(full, out, depth + 1);
+    }
+}
+
+function harnessExportsOf(pkg: Pkg): readonly string[] {
+    if (!pkg.hasTesting) return [];
+    const root = join(pkg.dir, "src", "testing");
+    try {
+        statSync(root);
+    } catch {
+        return [];
+    }
+    const names = new Set<string>();
+    function scan(d: string): void {
+        for (const e of readdirSync(d, { withFileTypes: true })) {
+            const f = join(d, e.name);
+            if (e.isDirectory()) scan(f);
+            else if (e.isFile() && f.endsWith(".ts")) {
+                const text = readFileSync(f, "utf-8");
+                const re =
+                    /export\s+(?:async\s+)?function\s+(run\w+Conformance)\s*\(/g;
+                let m: RegExpExecArray | null;
+                while ((m = re.exec(text)) !== null) {
+                    if (m[1]) names.add(m[1]);
+                }
+            }
+        }
+    }
+    scan(root);
+    return [...names];
+}
+
+function collectTestFiles(root: string, out: string[], depth = 0): void {
+    if (depth > 8) return;
+    let entries: ReturnType<typeof readdirSync>;
+    try {
+        entries = readdirSync(root, { withFileTypes: true });
+    } catch {
+        return;
+    }
+    for (const e of entries) {
+        if (
+            e.name === "node_modules" ||
+            e.name === "dist" ||
+            e.name.startsWith(".")
+        )
+            continue;
+        const full = join(root, e.name);
+        if (e.isDirectory()) {
+            collectTestFiles(full, out, depth + 1);
+            continue;
+        }
+        if (!e.isFile()) continue;
+        if (!full.endsWith(".test.ts") && !full.endsWith(".spec.ts")) continue;
+        out.push(full);
+    }
+}
+
+function hasConformanceTest(
+    pkg: Pkg,
+    contractBasename: string,
+    harnesses: readonly string[]
+): boolean {
+    const candidates: string[] = [];
+    collectTestFiles(join(pkg.dir, "tests"), candidates);
+    collectTestFiles(join(pkg.dir, "src"), candidates);
+    const harnessSet = new Set(harnesses);
+    for (const file of candidates) {
+        const base = file.split("/").pop() ?? "";
+        if (!/conformance/i.test(base)) continue;
+        let text: string;
+        try {
+            text = readFileSync(file, "utf-8");
+        } catch {
+            continue;
+        }
+        if (text.includes(`@suaveplan/${contractBasename}/testing`))
+            return true;
+        for (const h of harnessSet) {
+            // function-call invocation, possibly multiline whitespace before `(`
+            const re = new RegExp(`\\b${h}\\s*\\(`);
+            if (re.test(text)) return true;
+        }
+    }
+    return false;
+}
+
+const pkgs: Pkg[] = [];
+findPackages(PACKAGES_DIR, pkgs);
+const byName = new Map(pkgs.map((p) => [p.name, p]));
+
+interface V {
+    readonly consumer: string;
+    readonly contract: string;
+    readonly harnesses: readonly string[];
+}
+const violations: V[] = [];
+
+for (const consumer of pkgs) {
+    for (const dep of consumer.deps) {
+        const contract = byName.get(dep);
+        if (!contract || contract.basename === consumer.basename) continue;
+        const harnesses = harnessExportsOf(contract);
+        if (harnesses.length === 0) continue;
+        if (consumer.skip.has(contract.basename)) continue;
+        if (!hasConformanceTest(consumer, contract.basename, harnesses)) {
+            violations.push({
+                consumer: consumer.name,
+                contract: contract.name,
+                harnesses,
+            });
+        }
+    }
+}
+
+if (violations.length === 0) {
+    console.log(
+        `✓ check-conformance-tests: 0 violations across ${pkgs.length} packages`
+    );
+    process.exit(0);
+}
+console.error(`✗ check-conformance-tests: ${violations.length} violation(s)`);
+console.error(
+    "Per openspec/AGENTS.md §16, every package consuming a contract MUST run"
+);
+console.error(
+    "the contract's conformance harness. Add tests/conformance/<contract>.test.ts.\n"
+);
+for (const v of violations) {
+    console.error(`  ${v.consumer}`);
+    console.error(`    consumes contract: ${v.contract}`);
+    console.error(`    harnesses available: ${v.harnesses.join(", ")}`);
+    const base = v.contract.slice("@suaveplan/".length);
+    console.error(`    fix: add tests/conformance/${base}.test.ts`);
+}
+process.exit(1);