Initial Commit

packages/testkit/testing/ab-testing/tests/integration.test.ts

@@ -0,0 +1,168 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { createExperimentManager } from "../src/core/experiment-manager/experiment-manager.js";
+import {
+    ForceVariantProvider,
+    LoggingProvider,
+} from "../src/providers/provider.js";
+import type { ExperimentDefinition } from "../src/types/common.js";
+
+const runningExp = (
+    id: string,
+    overrides?: Partial<ExperimentDefinition>
+): ExperimentDefinition => ({
+    id,
+    name: `Experiment ${id}`,
+    status: "running",
+    variants: [
+        { id: "ctrl", name: "Control", weight: 1, isControl: true },
+        { id: "treat", name: "Treatment", weight: 1 },
+    ],
+    primaryMetric: "conversion",
+    ...overrides,
+});
+
+describe("end-to-end flow", () => {
+    it("full happy path: define → assign → track → results", {
+        timeout: 5000,
+    }, () => {
+        const manager = createExperimentManager();
+        manager.defineExperiment(runningExp("e2e"));
+
+        // Assign 10 users
+        const assignments = Array.from({ length: 10 }, (_, i) =>
+            manager.assign("e2e", `user-${i}`)
+        );
+        expect(assignments).toHaveLength(10);
+        expect(assignments.every((a) => a.inExperiment)).toBe(true);
+
+        // Track conversions for first 5 users
+        for (let i = 0; i < 5; i++) {
+            manager.track("e2e", `user-${i}`, {
+                name: "conversion",
+                value: 10,
+            });
+        }
+
+        const results = manager.getResults("e2e");
+        expect(results.totalAssignments).toBe(10);
+        expect(results.totalConversions).toBe(5);
+        expect(results.variants.length).toBeGreaterThan(0);
+    });
+
+    it("targeting excludes ineligible users", { timeout: 5000 }, () => {
+        const manager = createExperimentManager();
+        manager.defineExperiment(
+            runningExp("targeted", {
+                targeting: [
+                    { attribute: "plan", operator: "eq", value: "premium" },
+                ],
+            })
+        );
+
+        const included = manager.assign("targeted", "premium-user", {
+            plan: "premium",
+        });
+        const excluded = manager.assign("targeted", "free-user", {
+            plan: "free",
+        });
+
+        expect(included.inExperiment).toBe(true);
+        expect(excluded.inExperiment).toBe(false);
+        expect(excluded.variantId).toBe("");
+    });
+
+    it("sample rate excludes portion of users", { timeout: 5000 }, () => {
+        // Use rng that always returns 0 → always included
+        const alwaysIn = createExperimentManager({
+            rng: { nextFloat: () => 0, nextUint32: () => 0 },
+        });
+        alwaysIn.defineExperiment(runningExp("sample", { sampleRate: 0.5 }));
+        const a1 = alwaysIn.assign("sample", "u1");
+        expect(a1.inExperiment).toBe(true);
+
+        // Use rng that always returns 1 → always excluded
+        const alwaysOut = createExperimentManager({
+            rng: { nextFloat: () => 1, nextUint32: () => 0xffffffff },
+        });
+        alwaysOut.defineExperiment(runningExp("sample", { sampleRate: 0.5 }));
+        const a2 = alwaysOut.assign("sample", "u1");
+        expect(a2.inExperiment).toBe(false);
+    });
+
+    it("ForceVariantProvider overrides bucketing", { timeout: 5000 }, () => {
+        const manager = createExperimentManager();
+        manager.defineExperiment(runningExp("force-test"));
+        manager.use(
+            new ForceVariantProvider({ "force-test": { "qa-user": "treat" } })
+        );
+
+        const a = manager.assign("force-test", "qa-user");
+        expect(a.variantId).toBe("treat");
+        expect(a.isControl).toBe(false);
+    });
+
+    it("LoggingProvider receives assignment events", { timeout: 5000 }, () => {
+        const received: string[] = [];
+        const manager = createExperimentManager();
+        manager.defineExperiment(runningExp("logging-test"));
+        manager.use(
+            new LoggingProvider({
+                onAssignment: (a) => {
+                    received.push(a.variantId);
+                },
+            })
+        );
+
+        manager.assign("logging-test", "u1");
+        expect(received).toHaveLength(1);
+    });
+
+    it("LoggingProvider receives conversion events", { timeout: 5000 }, () => {
+        const received: string[] = [];
+        const manager = createExperimentManager();
+        manager.defineExperiment(runningExp("conv-test"));
+        manager.use(
+            new LoggingProvider({
+                onConversion: (expId) => {
+                    received.push(expId);
+                },
+            })
+        );
+
+        manager.track("conv-test", "u1", { name: "click" });
+        expect(received).toHaveLength(1);
+        expect(received[0]).toBe("conv-test");
+    });
+
+    it("multiple experiments run independently", { timeout: 5000 }, () => {
+        const manager = createExperimentManager();
+        manager.defineExperiment(runningExp("exp-a"));
+        manager.defineExperiment(runningExp("exp-b"));
+
+        const aA = manager.assign("exp-a", "user-shared");
+        const aB = manager.assign("exp-b", "user-shared");
+
+        // Same user can be in different variants in different experiments
+        expect(aA.experimentId).toBe("exp-a");
+        expect(aB.experimentId).toBe("exp-b");
+    });
+
+    it("hashSeed changes variant assignment", { timeout: 5000 }, () => {
+        const m1 = createExperimentManager({ hashSeed: 0 });
+        const m2 = createExperimentManager({ hashSeed: 999999 });
+
+        m1.defineExperiment(runningExp("seed-test"));
+        m2.defineExperiment(runningExp("seed-test"));
+
+        // Run many users and verify the distributions differ (not just one user)
+        const results1 = new Set<string>();
+        const results2 = new Set<string>();
+        for (let i = 0; i < 100; i++) {
+            results1.add(m1.assign("seed-test", `s-${i}`).variantId);
+            results2.add(m2.assign("seed-test", `s-${i}`).variantId);
+        }
+        // Both should have multiple variants
+        expect(results1.size).toBeGreaterThan(0);
+        expect(results2.size).toBeGreaterThan(0);
+    });
+});

packages/testkit/testing/ab-testing/tests/property.test.ts

@@ -0,0 +1,125 @@
+import { describe, it } from "@suaveplan/testing/runner";
+import * as fc from "fast-check";
+import {
+    bucketUser,
+    normaliseWeights,
+    selectVariant,
+} from "../src/bucketing/bucketing.js";
+import { evaluateTargeting } from "../src/targeting/targeting.js";
+
+describe("property: bucketUser", () => {
+    it("always returns [0,1) for arbitrary inputs", { timeout: 30000 }, () => {
+        fc.assert(
+            fc.property(
+                fc.string(),
+                fc.string(),
+                fc.integer({ min: -1_000_000, max: 1_000_000 }),
+                (userId, experimentId, seed) => {
+                    const b = bucketUser(userId, experimentId, seed);
+                    return b >= 0 && b < 1;
+                }
+            ),
+            { numRuns: 1000 }
+        );
+    });
+
+    it("is deterministic for the same inputs", { timeout: 30000 }, () => {
+        fc.assert(
+            fc.property(fc.string(), fc.string(), (userId, experimentId) => {
+                const a = bucketUser(userId, experimentId);
+                const b = bucketUser(userId, experimentId);
+                return a === b;
+            }),
+            { numRuns: 1000 }
+        );
+    });
+});
+
+describe("property: normaliseWeights + selectVariant", () => {
+    const positiveVariantArb = fc.record({
+        id: fc.string({ minLength: 1 }),
+        name: fc.string({ minLength: 1 }),
+        weight: fc.float({
+            min: Math.fround(0.001),
+            max: Math.fround(1000),
+            noNaN: true,
+        }),
+    });
+
+    it("selected variant always belongs to input", { timeout: 30000 }, () => {
+        fc.assert(
+            fc.property(
+                fc.array(positiveVariantArb, { minLength: 1, maxLength: 10 }),
+                fc.float({
+                    min: Math.fround(0),
+                    max: Math.fround(0.9999),
+                    noNaN: true,
+                }),
+                (rawVariants, bucket) => {
+                    const normalised = normaliseWeights(rawVariants);
+                    if (normalised.length === 0) return true; // skip degenerate
+                    const selected = selectVariant(normalised, bucket);
+                    return rawVariants.some((v) => v.id === selected.id);
+                }
+            ),
+            { numRuns: 500 }
+        );
+    });
+
+    it("last normalised weight is always exactly 1.0", {
+        timeout: 30000,
+    }, () => {
+        fc.assert(
+            fc.property(
+                fc.array(positiveVariantArb, { minLength: 1, maxLength: 20 }),
+                (rawVariants) => {
+                    const normalised = normaliseWeights(rawVariants);
+                    if (normalised.length === 0) return true;
+                    return (
+                        normalised[normalised.length - 1]?.normalizedWeight ===
+                        1.0
+                    );
+                }
+            ),
+            { numRuns: 500 }
+        );
+    });
+});
+
+describe("property: evaluateTargeting", () => {
+    it("empty rules always returns true", { timeout: 30000 }, () => {
+        fc.assert(
+            fc.property(
+                fc.dictionary(
+                    fc.string(),
+                    fc.oneof(fc.string(), fc.integer(), fc.boolean())
+                ),
+                (attrs) => {
+                    return evaluateTargeting([], attrs) === true;
+                }
+            ),
+            { numRuns: 500 }
+        );
+    });
+
+    it("eq rule is reflexive for present attributes", {
+        timeout: 30000,
+    }, () => {
+        fc.assert(
+            fc.property(
+                fc.string({ minLength: 1 }),
+                fc.oneof(fc.string(), fc.integer()),
+                (key, value) => {
+                    const attrs = { [key]: value };
+                    const rule = {
+                        attribute: key,
+                        operator: "eq" as const,
+                        value,
+                    };
+                    return evaluateTargeting([rule], attrs) === true;
+                }
+            ),
+            { numRuns: 500 }
+        );
+    });
+});

packages/testkit/testing/ab-testing/tests/setup/polyfills.ts

@@ -0,0 +1,8 @@
+// Ensure `globalThis.crypto` is available in bare-Node vitest runs.
+import { webcrypto } from "node:crypto";
+
+type WritableGlobal = { crypto?: Crypto };
+const g = globalThis as WritableGlobal;
+if (g.crypto === undefined) {
+    g.crypto = webcrypto as unknown as Crypto;
+}