suaveplan/packages/testkit/testing/ab-testing/tests/integration.test.ts

import { describe, expect, it } from "@suaveplan/testing/runner";
import { createExperimentManager } from "../src/core/experiment-manager/experiment-manager.js";
import {
    ForceVariantProvider,
    LoggingProvider,
} from "../src/providers/provider.js";
import type { ExperimentDefinition } from "../src/types/common.js";

const runningExp = (
    id: string,
    overrides?: Partial<ExperimentDefinition>
): ExperimentDefinition => ({
    id,
    name: `Experiment ${id}`,
    status: "running",
    variants: [
        { id: "ctrl", name: "Control", weight: 1, isControl: true },
        { id: "treat", name: "Treatment", weight: 1 },
    ],
    primaryMetric: "conversion",
    ...overrides,
});

describe("end-to-end flow", () => {
    it("full happy path: define → assign → track → results", {
        timeout: 5000,
    }, () => {
        const manager = createExperimentManager();
        manager.defineExperiment(runningExp("e2e"));

        // Assign 10 users
        const assignments = Array.from({ length: 10 }, (_, i) =>
            manager.assign("e2e", `user-${i}`)
        );
        expect(assignments).toHaveLength(10);
        expect(assignments.every((a) => a.inExperiment)).toBe(true);

        // Track conversions for first 5 users
        for (let i = 0; i < 5; i++) {
            manager.track("e2e", `user-${i}`, {
                name: "conversion",
                value: 10,
            });
        }

        const results = manager.getResults("e2e");
        expect(results.totalAssignments).toBe(10);
        expect(results.totalConversions).toBe(5);
        expect(results.variants.length).toBeGreaterThan(0);
    });

    it("targeting excludes ineligible users", { timeout: 5000 }, () => {
        const manager = createExperimentManager();
        manager.defineExperiment(
            runningExp("targeted", {
                targeting: [
                    { attribute: "plan", operator: "eq", value: "premium" },
                ],
            })
        );

        const included = manager.assign("targeted", "premium-user", {
            plan: "premium",
        });
        const excluded = manager.assign("targeted", "free-user", {
            plan: "free",
        });

        expect(included.inExperiment).toBe(true);
        expect(excluded.inExperiment).toBe(false);
        expect(excluded.variantId).toBe("");
    });

    it("sample rate excludes portion of users", { timeout: 5000 }, () => {
        // Use rng that always returns 0 → always included
        const alwaysIn = createExperimentManager({
            rng: { nextFloat: () => 0, nextUint32: () => 0 },
        });
        alwaysIn.defineExperiment(runningExp("sample", { sampleRate: 0.5 }));
        const a1 = alwaysIn.assign("sample", "u1");
        expect(a1.inExperiment).toBe(true);

        // Use rng that always returns 1 → always excluded
        const alwaysOut = createExperimentManager({
            rng: { nextFloat: () => 1, nextUint32: () => 0xffffffff },
        });
        alwaysOut.defineExperiment(runningExp("sample", { sampleRate: 0.5 }));
        const a2 = alwaysOut.assign("sample", "u1");
        expect(a2.inExperiment).toBe(false);
    });

    it("ForceVariantProvider overrides bucketing", { timeout: 5000 }, () => {
        const manager = createExperimentManager();
        manager.defineExperiment(runningExp("force-test"));
        manager.use(
            new ForceVariantProvider({ "force-test": { "qa-user": "treat" } })
        );

        const a = manager.assign("force-test", "qa-user");
        expect(a.variantId).toBe("treat");
        expect(a.isControl).toBe(false);
    });

    it("LoggingProvider receives assignment events", { timeout: 5000 }, () => {
        const received: string[] = [];
        const manager = createExperimentManager();
        manager.defineExperiment(runningExp("logging-test"));
        manager.use(
            new LoggingProvider({
                onAssignment: (a) => {
                    received.push(a.variantId);
                },
            })
        );

        manager.assign("logging-test", "u1");
        expect(received).toHaveLength(1);
    });

    it("LoggingProvider receives conversion events", { timeout: 5000 }, () => {
        const received: string[] = [];
        const manager = createExperimentManager();
        manager.defineExperiment(runningExp("conv-test"));
        manager.use(
            new LoggingProvider({
                onConversion: (expId) => {
                    received.push(expId);
                },
            })
        );

        manager.track("conv-test", "u1", { name: "click" });
        expect(received).toHaveLength(1);
        expect(received[0]).toBe("conv-test");
    });

    it("multiple experiments run independently", { timeout: 5000 }, () => {
        const manager = createExperimentManager();
        manager.defineExperiment(runningExp("exp-a"));
        manager.defineExperiment(runningExp("exp-b"));

        const aA = manager.assign("exp-a", "user-shared");
        const aB = manager.assign("exp-b", "user-shared");

        // Same user can be in different variants in different experiments
        expect(aA.experimentId).toBe("exp-a");
        expect(aB.experimentId).toBe("exp-b");
    });

    it("hashSeed changes variant assignment", { timeout: 5000 }, () => {
        const m1 = createExperimentManager({ hashSeed: 0 });
        const m2 = createExperimentManager({ hashSeed: 999999 });

        m1.defineExperiment(runningExp("seed-test"));
        m2.defineExperiment(runningExp("seed-test"));

        // Run many users and verify the distributions differ (not just one user)
        const results1 = new Set<string>();
        const results2 = new Set<string>();
        for (let i = 0; i < 100; i++) {
            results1.add(m1.assign("seed-test", `s-${i}`).variantId);
            results2.add(m2.assign("seed-test", `s-${i}`).variantId);
        }
        // Both should have multiple variants
        expect(results1.size).toBeGreaterThan(0);
        expect(results2.size).toBeGreaterThan(0);
    });
});