Initial Commit
This commit is contained in:
241
openspec/archive/2026-04-24-enterprise-ab-testing/proposal.md
Normal file
241
openspec/archive/2026-04-24-enterprise-ab-testing/proposal.md
Normal file
@@ -0,0 +1,241 @@
|
||||
# OpenSpec Change: enterprise-ab-testing
|
||||
|
||||
- **Change ID**: enterprise-ab-testing
|
||||
- **Package**: `@suaveplan/ab-testing`
|
||||
- **Category**: universal/workflow
|
||||
- **Target path**: `universal/workflow/ab-testing`
|
||||
- **Status**: PROPOSED
|
||||
- **Date**: 2026-04-24
|
||||
- **Reference spec**: `/mnt/s/web/suaveplan/enterprise/ab-testing.md`
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Deliver `@suaveplan/ab-testing` on the genesis branch from scratch at enterprise quality. The package provides an A/B testing engine with deterministic assignment, audience targeting, experiment lifecycle management, metric tracking, and results analysis. Genesis adds structured errors via `@suaveplan/error`, Zod schemas for all experiment types, OTel instrumentation on assignment and conversion events, injectable `Clock` and `Rng` for deterministic bucketing, integration with `@suaveplan/feature-flags` as an optional provider, event-bus emission for assignment/conversion events, and property-based tests for bucketing distribution invariants.
|
||||
|
||||
---
|
||||
|
||||
## Current-state audit (genesis)
|
||||
|
||||
The package does not exist on genesis yet — this is a fresh build.
|
||||
|
||||
---
|
||||
|
||||
## Available workspace deps on genesis
|
||||
|
||||
| Package | Role |
|
||||
|---|---|
|
||||
| `@suaveplan/error` (`workspace:^`) | Base class for `AbTestingError`; stable error codes |
|
||||
| `@suaveplan/types` (`workspace:^`) | `LoggerLike`, `Clock`, `Rng`, `EventBusLike` |
|
||||
| `@suaveplan/events` (`workspace:^`) | Event bus for assignment and conversion events |
|
||||
| `@suaveplan/logger` (`workspace:^`) | Structured logging with OTel trace correlation |
|
||||
| `@suaveplan/feature-flags` (`workspace:^`) | Optional feature-flag provider for experiment gating |
|
||||
| `zod` (`catalog:forms`) | Zod schemas for all public types |
|
||||
| `@opentelemetry/api` (`catalog:observability`) | Spans + counters on assignment/conversion |
|
||||
| `tslib` (`catalog:build`) | Runtime helpers |
|
||||
|
||||
Dev deps: `vitest`, `@vitest/coverage-istanbul`, `fast-check`, `mitata` (all `catalog:testing`).
|
||||
|
||||
---
|
||||
|
||||
## Public API surface
|
||||
|
||||
### Core
|
||||
|
||||
```ts
|
||||
function createExperimentManager(options?: ExperimentManagerOptions): ExperimentManager
|
||||
|
||||
interface ExperimentManager {
|
||||
defineExperiment(experiment: ExperimentDefinition): void
|
||||
getExperiment(id: string): ExperimentDefinition | null
|
||||
listExperiments(): ReadonlyArray<ExperimentDefinition>
|
||||
assign(experimentId: string, userId: string, attributes?: UserAttributes): Assignment
|
||||
track(experimentId: string, userId: string, metric: MetricEvent): void
|
||||
getResults(experimentId: string): ExperimentResults
|
||||
use(provider: ExperimentProvider): this
|
||||
}
|
||||
```
|
||||
|
||||
### Experiment definition
|
||||
|
||||
```ts
|
||||
interface ExperimentDefinition {
|
||||
id: string
|
||||
name: string
|
||||
status: 'draft' | 'running' | 'paused' | 'concluded'
|
||||
variants: ReadonlyArray<Variant>
|
||||
targeting?: TargetingRule[]
|
||||
startDate?: string
|
||||
endDate?: string
|
||||
sampleRate?: number // 0–1, default 1.0
|
||||
primaryMetric?: string
|
||||
secondaryMetrics?: string[]
|
||||
}
|
||||
|
||||
interface Variant {
|
||||
id: string
|
||||
name: string
|
||||
weight: number // relative weight, will be normalised
|
||||
isControl?: boolean
|
||||
}
|
||||
|
||||
interface TargetingRule {
|
||||
attribute: string
|
||||
operator: 'eq' | 'neq' | 'in' | 'notIn' | 'gt' | 'lt' | 'gte' | 'lte'
|
||||
value: unknown
|
||||
}
|
||||
```
|
||||
|
||||
### Assignment
|
||||
|
||||
```ts
|
||||
interface Assignment {
|
||||
readonly experimentId: string
|
||||
readonly userId: string
|
||||
readonly variantId: string
|
||||
readonly isControl: boolean
|
||||
readonly timestamp: number
|
||||
readonly inExperiment: boolean // false if excluded by targeting or sample rate
|
||||
}
|
||||
```
|
||||
|
||||
### Metrics
|
||||
|
||||
```ts
|
||||
interface MetricEvent {
|
||||
name: string
|
||||
value?: number
|
||||
timestamp?: number
|
||||
properties?: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface ExperimentResults {
|
||||
readonly experimentId: string
|
||||
readonly variants: ReadonlyArray<VariantResults>
|
||||
readonly totalAssignments: number
|
||||
readonly totalConversions: number
|
||||
}
|
||||
|
||||
interface VariantResults {
|
||||
readonly variantId: string
|
||||
readonly assignments: number
|
||||
readonly conversions: number
|
||||
readonly conversionRate: number
|
||||
readonly metrics: ReadonlyArray<{ name: string; count: number; sum: number; avg: number }>
|
||||
}
|
||||
```
|
||||
|
||||
### Provider interface
|
||||
|
||||
```ts
|
||||
interface ExperimentProvider {
|
||||
readonly name: string
|
||||
getAssignment?(experimentId: string, userId: string): string | null
|
||||
trackAssignment?(assignment: Assignment): void
|
||||
trackConversion?(experimentId: string, userId: string, metric: MetricEvent): void
|
||||
}
|
||||
|
||||
type UserAttributes = Record<string, string | number | boolean>
|
||||
|
||||
interface ExperimentManagerOptions {
|
||||
logger?: LoggerLike
|
||||
clock?: Clock
|
||||
rng?: Rng
|
||||
eventBus?: EventBusLike
|
||||
hashSeed?: number
|
||||
}
|
||||
```
|
||||
|
||||
### Bucketing utilities
|
||||
|
||||
```ts
|
||||
function bucketUser(userId: string, experimentId: string, seed?: number): number // 0–1
|
||||
function normaliseWeights(variants: Variant[]): Array<Variant & { normalizedWeight: number }>
|
||||
function selectVariant(variants: Variant[], bucket: number): Variant
|
||||
```
|
||||
|
||||
### Error codes
|
||||
|
||||
```ts
|
||||
type AbTestingErrorCode =
|
||||
| 'AB_EXPERIMENT_NOT_FOUND'
|
||||
| 'AB_EXPERIMENT_INACTIVE'
|
||||
| 'AB_INVALID_DEFINITION'
|
||||
| 'AB_VARIANT_NOT_FOUND'
|
||||
| 'AB_METRIC_INVALID'
|
||||
|
||||
function registerAbTestingErrorCodes(): void
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Genesis additions
|
||||
|
||||
1. **Structured `AbTestingError`** extending `SuaveplanError` with stable `AB.*` codes
|
||||
2. **Zod schemas** for `ExperimentDefinition`, `Variant`, `TargetingRule`, `MetricEvent`
|
||||
3. **Deterministic FNV-1a bucketing** — injectable `Rng` for testing; FNV-1a hash by default for consistency with `@suaveplan/feature-flags`
|
||||
4. **Targeting rule evaluation** — 8 operators, evaluated before assignment
|
||||
5. **Provider plugin interface** — `use()` for external integrations
|
||||
6. **`AbortSignal`** available on async provider calls
|
||||
7. **OTel instrumentation** — span `ab.assign`, counter `ab.assignments_total`, counter `ab.conversions_total`, attributes include `experiment_id`, `variant_id`
|
||||
8. **Property-based tests** — bucketing distribution, targeting rule evaluation, weight normalisation
|
||||
|
||||
---
|
||||
|
||||
## File layout
|
||||
|
||||
```text
|
||||
universal/workflow/ab-testing/
|
||||
package.json
|
||||
tsconfig.json
|
||||
vitest.config.ts
|
||||
src/
|
||||
index.ts
|
||||
error-codes.ts
|
||||
constants/
|
||||
index.ts
|
||||
types/
|
||||
common.ts
|
||||
common.md
|
||||
core/
|
||||
experiment-manager/
|
||||
experiment-manager.ts
|
||||
experiment-manager.md
|
||||
experiment-manager.types.ts
|
||||
experiment-manager.test.ts
|
||||
index.ts
|
||||
bucketing/
|
||||
bucketing.ts
|
||||
bucketing.md
|
||||
bucketing.test.ts
|
||||
index.ts
|
||||
targeting/
|
||||
targeting.ts
|
||||
targeting.md
|
||||
targeting.test.ts
|
||||
index.ts
|
||||
results/
|
||||
results.ts
|
||||
results.md
|
||||
results.test.ts
|
||||
index.ts
|
||||
providers/
|
||||
provider.ts
|
||||
provider.md
|
||||
provider.test.ts
|
||||
index.ts
|
||||
__benchmarks__/
|
||||
ab-testing.bench.ts
|
||||
tests/
|
||||
integration.test.ts
|
||||
property.test.ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Impact
|
||||
|
||||
**Downstream packages**: Any application layer that needs experiment assignment. No existing genesis package currently depends on it.
|
||||
|
||||
**Workspace dep prerequisites**: `@suaveplan/error`, `@suaveplan/types`, `@suaveplan/events`, `@suaveplan/logger`, `@suaveplan/feature-flags` must be published.
|
||||
128
openspec/archive/2026-04-24-enterprise-ab-testing/tasks.md
Normal file
128
openspec/archive/2026-04-24-enterprise-ab-testing/tasks.md
Normal file
@@ -0,0 +1,128 @@
|
||||
# Tasks: enterprise-ab-testing
|
||||
|
||||
**Change ID**: enterprise-ab-testing
|
||||
**Package**: `@suaveplan/ab-testing`
|
||||
**Target path**: `universal/workflow/ab-testing`
|
||||
**Prerequisite packages on Verdaccio**: `@suaveplan/error`, `@suaveplan/types`, `@suaveplan/events`, `@suaveplan/logger`, `@suaveplan/feature-flags`
|
||||
|
||||
---
|
||||
|
||||
## 1. Scaffold
|
||||
|
||||
- [x] Create `universal/workflow/ab-testing/` with `src/`, `tests/`, `src/__benchmarks__/`, and feature subdirectories per the layout in `proposal.md`.
|
||||
- [x] Write `package.json` at version `0.1.0`:
|
||||
- `"type": "module"`, `"sideEffects": false`
|
||||
- `exports` map: `"."` barrel + one subpath per feature module
|
||||
- `dependencies`: `@suaveplan/error workspace:^`, `@suaveplan/types workspace:^`, `@suaveplan/events workspace:^`, `@suaveplan/logger workspace:^`, `@suaveplan/feature-flags workspace:^`, `zod catalog:forms`, `@opentelemetry/api catalog:observability`, `tslib catalog:build`
|
||||
- `devDependencies`: `vitest catalog:testing`, `@vitest/coverage-istanbul catalog:testing`, `fast-check catalog:testing`, `mitata catalog:testing`, `@suaveplan/testing workspace:^`
|
||||
- [x] Write `tsconfig.json` extending `@suaveplan/typescript-config/library.json`.
|
||||
- [x] Write `vitest.config.ts` using `createTestConfig()`.
|
||||
- [x] Add `lint`, `typecheck`, `test`, `test:coverage`, `build`, `docs:check`, `bench` scripts.
|
||||
|
||||
---
|
||||
|
||||
## 2. Types and error codes
|
||||
|
||||
- [x] Define all public types in `src/types/common.ts`: `ExperimentDefinition`, `Variant`, `TargetingRule`, `Assignment`, `MetricEvent`, `ExperimentResults`, `VariantResults`, `ExperimentManagerOptions`, `ExperimentProvider`, `UserAttributes`.
|
||||
- [x] Write `src/types/common.md` (≥200 words, simple + advanced examples).
|
||||
- [x] Define error codes in `src/error-codes.ts`: `AbTestingErrorCode` union, metadata, `registerAbTestingErrorCodes()`.
|
||||
- [x] Write Zod schemas in `src/types/schemas.ts` + `src/types/schemas.md`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Bucketing (`src/bucketing/`)
|
||||
|
||||
- [x] Implement `bucketUser(userId, experimentId, seed?)`:
|
||||
- FNV-1a hash of `${experimentId}:${userId}` (with optional seed XOR'd in).
|
||||
- Normalises to `[0, 1)` float.
|
||||
- [x] Implement `normaliseWeights(variants)` — normalises `weight` values to sum to 1.
|
||||
- [x] Implement `selectVariant(variants, bucket)` — binary search over cumulative normalised weights.
|
||||
- [x] Write `bucketing.test.ts` (≤500 lines): determinism for same userId+experimentId, distribution spread across 10k users, weight edge cases (all equal, single variant, zero-weight skipped).
|
||||
- [x] Write `bucketing.md` (≥200 words).
|
||||
|
||||
---
|
||||
|
||||
## 4. Targeting (`src/targeting/`)
|
||||
|
||||
- [x] Implement `evaluateTargeting(rules: TargetingRule[], attributes: UserAttributes): boolean`:
|
||||
- All rules must pass (implicit AND).
|
||||
- Operators: `eq`, `neq`, `in`, `notIn`, `gt`, `lt`, `gte`, `lte`.
|
||||
- Returns `true` when user matches all rules (or no rules defined).
|
||||
- [x] Write `targeting.test.ts`: each operator, multi-rule AND, no rules = true, missing attribute = false.
|
||||
- [x] Write `targeting.md` (≥200 words).
|
||||
|
||||
---
|
||||
|
||||
## 5. Results (`src/results/`)
|
||||
|
||||
- [x] Implement `computeResults(experimentId, assignments, conversions): ExperimentResults`:
|
||||
- Group assignments and conversions by variant.
|
||||
- Compute `conversionRate` = conversions / assignments.
|
||||
- Aggregate metrics (count, sum, avg) per metric name per variant.
|
||||
- [x] Write `results.test.ts`: known data produces expected conversion rate, empty data, single variant.
|
||||
- [x] Write `results.md` (≥200 words).
|
||||
|
||||
---
|
||||
|
||||
## 6. Experiment manager (`src/core/experiment-manager/`)
|
||||
|
||||
- [x] Implement `createExperimentManager(options?)`:
|
||||
- Stores experiments in `Map<string, ExperimentDefinition>`.
|
||||
- Stores assignments and metric events in `Map` keyed by `${experimentId}:${userId}`.
|
||||
- [x] Implement `manager.defineExperiment(def)` — validates with Zod; throws `AbTestingError(AB_INVALID_DEFINITION)` on schema error.
|
||||
- [x] Implement `manager.assign(experimentId, userId, attributes?)`:
|
||||
- Check experiment exists and is `running`; throw `AB_EXPERIMENT_INACTIVE` otherwise.
|
||||
- Evaluate targeting rules; set `inExperiment: false` if excluded.
|
||||
- Apply `sampleRate` check using `rng.next()`.
|
||||
- Use `bucketUser` to select variant.
|
||||
- Store assignment, emit OTel span `ab.assign`, counter `ab.assignments_total`.
|
||||
- Emit event-bus event `ab:assigned`.
|
||||
- Call any registered provider `trackAssignment`.
|
||||
- [x] Implement `manager.track(experimentId, userId, metric)` — appends metric event; counter `ab.conversions_total`.
|
||||
- [x] Implement `manager.getResults(experimentId)` — delegates to `computeResults`.
|
||||
- [x] Implement `manager.use(provider)` — registers provider.
|
||||
- [x] Write `experiment-manager.test.ts` (≤500 lines): full assignment flow, targeting exclusion, sample rate, metric tracking, results, provider called.
|
||||
- [x] Write `experiment-manager.md` (≥200 words).
|
||||
|
||||
---
|
||||
|
||||
## 7. Provider interface (`src/providers/`)
|
||||
|
||||
- [x] Define `ExperimentProvider` interface.
|
||||
- [x] Write `provider.test.ts`: provider registered, `trackAssignment` called on assign.
|
||||
- [x] Write `provider.md`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Property-based tests (`tests/property.test.ts`)
|
||||
|
||||
- [x] Bucketing determinism: same `userId` + `experimentId` always produces same bucket value.
|
||||
- [x] Weight distribution: across 10k users, each variant's assignment count is within 10% of its weight proportion.
|
||||
- [x] Targeting AND semantics: if any rule fails, result is `false`; all rules passing → `true`.
|
||||
|
||||
---
|
||||
|
||||
## 9. Benchmarks (`src/__benchmarks__/ab-testing.bench.ts`)
|
||||
|
||||
- [x] `assign_hot`: 10k `assign` calls for the same experiment (cache warm).
|
||||
- [x] `bucket_hot`: 10k `bucketUser` calls.
|
||||
|
||||
---
|
||||
|
||||
## 10. Docs
|
||||
|
||||
- [x] All per-file `.md` files per layout in `proposal.md`.
|
||||
- [x] `universal/workflow/ab-testing/README.md`.
|
||||
|
||||
---
|
||||
|
||||
## 11. Verify and publish
|
||||
|
||||
- [x] `bunx turbo lint --filter=@suaveplan/ab-testing` — zero errors.
|
||||
- [x] `bunx turbo typecheck --filter=@suaveplan/ab-testing` — zero errors.
|
||||
- [x] `bunx turbo test --filter=@suaveplan/ab-testing` — all tests pass.
|
||||
- [x] `bunx turbo test:coverage --filter=@suaveplan/ab-testing` — 100% on all touched modules.
|
||||
- [x] `bun run docs:check --filter=@suaveplan/ab-testing` — zero errors.
|
||||
- [x] `bun scripts/publish.ts --package @suaveplan/ab-testing`.
|
||||
- [x] Update `CLAUDE.md` package listing.
|
||||
- [x] Archive: move `openspec/changes/enterprise-ab-testing/` to `openspec/archive/2026-04-24-enterprise-ab-testing/`.
|
||||
Reference in New Issue
Block a user