Initial Commit
This commit is contained in:
49
packages/foundation/data/dedup/README.md
Normal file
49
packages/foundation/data/dedup/README.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# @suaveplan/dedup - Entity deduplication engine with pluggable match keys, scoring rules, Jaro-Winkler/Levenshtein similarity, and merge strategies
|
||||
|
||||
## Summary
|
||||
|
||||
Entity deduplication engine for detecting and merging duplicate records in any data set. Given a list of objects with varying representations of the same real-world entity (e.g. `"John Smith"` vs `"J. Smith"`, or the same email in different capitalisation), the `DedupEngine` scores pairs using a weighted combination of field-level similarity algorithms — Jaro-Winkler, Levenshtein, exact match, or phonetic — groups records into connected components via union-find, and merges each group with a pluggable strategy (`latestWins`, `manual`, or `scoredMerge`). Use this package for ETL pipelines, contact deduplication, and data cleansing. For domain-name-specific normalisation and alias resolution use `@suaveplan/domain-normalizer`; for string-similarity primitives only, the scoring module is exported as a standalone subpath.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
<!-- end:auto-docs --> - [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_ - [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_ - [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_ - [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_ - [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_
|
||||
## Modules
|
||||
|
||||
| Module | Description |
|
||||
|---|---|
|
||||
| [scoring](src/core/scoring/similarity.md) | Levenshtein, Jaro, Jaro-Winkler, exact, and phonetic similarity algorithms |
|
||||
| [engine](src/core/engine/dedup-engine.md) | `DedupEngine` — detect duplicate groups and merge them |
|
||||
| [merge](src/core/merge/merge-strategies.md) | Built-in merge strategies: latest-wins, manual, scored |
|
||||
|
||||
## Quick Start
|
||||
|
||||
```ts
|
||||
import { DedupEngine } from "@suaveplan/dedup/engine";
|
||||
import { latestWinsStrategy } from "@suaveplan/dedup/merge";
|
||||
|
||||
const engine = new DedupEngine({
|
||||
matchKeys: [
|
||||
{ field: "name", weight: 0.6, strategy: "jaro-winkler" },
|
||||
{ field: "email", weight: 0.4, strategy: "exact" },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.85,
|
||||
});
|
||||
|
||||
const groups = engine.detect(records);
|
||||
for (const group of groups) {
|
||||
const result = engine.merge(group, latestWinsStrategy);
|
||||
console.log(result.merged);
|
||||
}
|
||||
```
|
||||
101
packages/foundation/data/dedup/package.json
Normal file
101
packages/foundation/data/dedup/package.json
Normal file
@@ -0,0 +1,101 @@
|
||||
{
|
||||
"name": "@suaveplan/dedup",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Entity deduplication engine with pluggable match keys, scoring rules, Jaro-Winkler/Levenshtein similarity, and merge strategies",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./scoring": {
|
||||
"import": {
|
||||
"types": "./dist/core/scoring/similarity.d.ts",
|
||||
"default": "./dist/core/scoring/similarity.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/core/scoring/similarity.d.ts",
|
||||
"default": "./dist/core/scoring/similarity.js"
|
||||
}
|
||||
},
|
||||
"./engine": {
|
||||
"import": {
|
||||
"types": "./dist/core/engine/dedup-engine.d.ts",
|
||||
"default": "./dist/core/engine/dedup-engine.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/core/engine/dedup-engine.d.ts",
|
||||
"default": "./dist/core/engine/dedup-engine.js"
|
||||
}
|
||||
},
|
||||
"./merge": {
|
||||
"import": {
|
||||
"types": "./dist/core/merge/merge-strategies.d.ts",
|
||||
"default": "./dist/core/merge/merge-strategies.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/core/merge/merge-strategies.d.ts",
|
||||
"default": "./dist/core/merge/merge-strategies.js"
|
||||
}
|
||||
},
|
||||
"./errors": {
|
||||
"import": {
|
||||
"types": "./dist/errors/dedup-error.d.ts",
|
||||
"default": "./dist/errors/dedup-error.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/errors/dedup-error.d.ts",
|
||||
"default": "./dist/errors/dedup-error.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"fast-check": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
114
packages/foundation/data/dedup/src/core/engine/dedup-engine.md
Normal file
114
packages/foundation/data/dedup/src/core/engine/dedup-engine.md
Normal file
@@ -0,0 +1,114 @@
|
||||
# dedup-engine
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`DedupEngine` is the central orchestrator for entity deduplication. It accepts a validated `DedupConfig`, performs an O(n²) pairwise similarity scan using pluggable match keys and scoring rules, groups near-duplicate entities into connected components via union-find, and exposes a `merge()` method that delegates to caller-supplied merge strategies. The engine is generic over the entity type `T` so it can operate on any record shape.
|
||||
|
||||
## Features
|
||||
|
||||
- **Schema-validated configuration** — `DedupConfig` is parsed with Zod at construction time; invalid configs throw a `DedupError` with code `DEDUP.INVALID_CONFIG`.
|
||||
- **Weighted field scoring** — each `MatchKey` contributes its `weight × similarity(field_a, field_b)` to the pair score.
|
||||
- **Pluggable scoring rules** — `ScoringRule` instances can override or supplement the default key-based scoring for custom business logic.
|
||||
- **Connected-component grouping** — union-find algorithm aggregates transitive duplicate relationships so A≈B and B≈C implies A, B, C are in the same group.
|
||||
- **`maxGroupSize` guard** — prevents unbounded memory growth by capping the number of duplicates per group.
|
||||
- **OTel instrumentation** — emits `dedup.detect.total` counter, `dedup.detect.duration_ms` histogram, and per-span attributes on every `detect()` and `merge()` call.
|
||||
- **Safe merge delegation** — `merge()` wraps strategy execution in a try/catch and re-throws as `DedupError(STRATEGY_ERROR)` to preserve error code stability.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { DedupEngine } from "@suaveplan/dedup/engine";
|
||||
|
||||
const engine = new DedupEngine({
|
||||
matchKeys: [
|
||||
{ field: "name", weight: 0.6, strategy: "jaro-winkler" },
|
||||
{ field: "email", weight: 0.4, strategy: "exact" },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.85,
|
||||
});
|
||||
|
||||
const records = [
|
||||
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
|
||||
{ id: 2, name: "Alice Smyth", email: "alice@example.com" },
|
||||
{ id: 3, name: "Bob Jones", email: "bob@example.com" },
|
||||
];
|
||||
|
||||
const groups = engine.detect(records);
|
||||
// groups[0].canonical → { id: 1, name: "Alice Smith", email: "alice@example.com" }
|
||||
// groups[0].duplicates → [{ id: 2, name: "Alice Smyth", email: "alice@example.com" }]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { DedupEngine } from "@suaveplan/dedup/engine";
|
||||
import { manualStrategy } from "@suaveplan/dedup/merge";
|
||||
|
||||
const engine = new DedupEngine<{ id: number; name: string; updatedAt: string }>({
|
||||
matchKeys: [
|
||||
{ field: "name", weight: 1, strategy: "jaro-winkler", caseSensitive: false },
|
||||
],
|
||||
scoringRules: [
|
||||
{
|
||||
name: "domain-boost",
|
||||
evaluate(a, b, _keys) {
|
||||
// Boost score when both entities are from the same domain
|
||||
const ra = a as Record<string, string>;
|
||||
const rb = b as Record<string, string>;
|
||||
const domainA = ra["email"]?.split("@")[1] ?? "";
|
||||
const domainB = rb["email"]?.split("@")[1] ?? "";
|
||||
return domainA && domainA === domainB ? 0.95 : 0.0;
|
||||
},
|
||||
},
|
||||
],
|
||||
mergeThreshold: 0.8,
|
||||
maxGroupSize: 50,
|
||||
});
|
||||
|
||||
const groups = engine.detect(records);
|
||||
for (const group of groups) {
|
||||
const result = engine.merge(group, manualStrategy);
|
||||
console.log("merged:", result.merged);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `new DedupEngine<T>(config: DedupConfig)`
|
||||
|
||||
Constructs a new engine. `config` is validated against `DedupConfigSchema`; throws `DedupError(INVALID_CONFIG)` on failure.
|
||||
|
||||
### `detect(entities: T[]): DuplicateGroup<T>[]`
|
||||
|
||||
Scans all entity pairs, builds a similarity graph, finds connected components, and returns one `DuplicateGroup` per multi-entity component. Singletons (no duplicates) are omitted.
|
||||
|
||||
**Params:**
|
||||
|
||||
- `entities` — array of entities to deduplicate
|
||||
|
||||
**Returns:** `DuplicateGroup<T>[]` — possibly empty if no duplicates found
|
||||
|
||||
### `merge(group: DuplicateGroup<T>, strategy: MergeStrategy<T>): MergeResult<T>`
|
||||
|
||||
Applies `strategy` to `group` and returns the merged result. Wraps thrown errors as `DedupError(STRATEGY_ERROR)`.
|
||||
|
||||
**Params:**
|
||||
|
||||
- `group` — a group returned by `detect()`
|
||||
- `strategy` — a `MergeStrategy<T>` function
|
||||
|
||||
**Returns:** `MergeResult<T>`
|
||||
|
||||
**Throws:** `DedupError` with code `DEDUP.STRATEGY_ERROR` if the strategy throws
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The pairwise scan is O(n²) in both time and the number of potential edges. For large corpora (> 10 000 entities) callers should pre-filter candidates using blocking keys (e.g. first three characters of a name) before passing them to `detect()`. A future blocking API may be added as a separate module.
|
||||
|
||||
The union-find implementation uses path-halving rather than full path compression to keep the code concise while still achieving near-linear amortised complexity. For groups larger than `maxGroupSize`, only the first `maxGroupSize − 1` duplicates (by original array index) are retained; this is intentional to bound memory and downstream merge cost.
|
||||
|
||||
When multiple `scoringRules` are present, each successive rule _replaces_ the accumulated score for that pair, not adds to it. This allows a single rule to act as a "veto" or "boost" override. If additive composition is needed, callers should encode that logic within a single rule.
|
||||
@@ -0,0 +1,342 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { DEDUP_CODES, DedupError } from "../../errors/dedup-error.js";
|
||||
import { DedupEngine } from "./dedup-engine.js";
|
||||
import type { DuplicateGroup, MergeStrategy } from "./dedup-engine.types.js";
|
||||
|
||||
type Person = {
|
||||
id: number;
|
||||
name: string;
|
||||
email: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
const baseConfig = {
|
||||
matchKeys: [
|
||||
{ field: "name", weight: 0.6, strategy: "jaro-winkler" as const },
|
||||
{ field: "email", weight: 0.4, strategy: "exact" as const },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.8,
|
||||
};
|
||||
|
||||
describe("DedupEngine constructor", () => {
|
||||
it("accepts a valid config", { timeout: 5000 }, () => {
|
||||
expect(() => new DedupEngine(baseConfig)).not.toThrow();
|
||||
});
|
||||
|
||||
it("throws DedupError(INVALID_CONFIG) for empty matchKeys", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
() =>
|
||||
new DedupEngine({
|
||||
matchKeys: [],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.85,
|
||||
})
|
||||
).toThrow(DedupError);
|
||||
try {
|
||||
new DedupEngine({
|
||||
matchKeys: [],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.85,
|
||||
});
|
||||
} catch (err) {
|
||||
expect((err as DedupError).code).toBe(DEDUP_CODES.INVALID_CONFIG);
|
||||
}
|
||||
});
|
||||
|
||||
it("throws DedupError(INVALID_CONFIG) for weight > 1", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(
|
||||
() =>
|
||||
new DedupEngine({
|
||||
matchKeys: [
|
||||
{
|
||||
field: "name",
|
||||
weight: 2,
|
||||
strategy: "exact" as const,
|
||||
},
|
||||
],
|
||||
scoringRules: [],
|
||||
})
|
||||
).toThrow(DedupError);
|
||||
});
|
||||
});
|
||||
|
||||
describe("DedupEngine.detect", () => {
|
||||
it("returns empty array for empty input", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
expect(engine.detect([])).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns empty array for single entity", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
expect(
|
||||
engine.detect([{ id: 1, name: "Alice", email: "a@example.com" }])
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("detects obvious duplicates", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
const records: Person[] = [
|
||||
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
|
||||
{ id: 2, name: "Alice Smyth", email: "alice@example.com" }, // near-dup of 1
|
||||
{ id: 3, name: "Bob Jones", email: "bob@example.com" },
|
||||
{ id: 4, name: "Charlie Brown", email: "charlie@example.com" },
|
||||
{ id: 5, name: "Alice Smith", email: "alice@example.com" }, // exact dup of 1
|
||||
];
|
||||
|
||||
const groups = engine.detect(records);
|
||||
expect(groups.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const group = groups[0];
|
||||
expect(group).toBeDefined();
|
||||
expect((group as NonNullable<typeof group>).canonical.id).toBe(1);
|
||||
expect(
|
||||
(group as NonNullable<typeof group>).duplicates.length
|
||||
).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("does not group dissimilar entities", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
const records: Person[] = [
|
||||
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
|
||||
{ id: 2, name: "Bob Jones", email: "bob@example.com" },
|
||||
{ id: 3, name: "Charlie Brown", email: "charlie@example.com" },
|
||||
];
|
||||
|
||||
const groups = engine.detect(records);
|
||||
expect(groups).toEqual([]);
|
||||
});
|
||||
|
||||
it("groups transitively connected duplicates into one component", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine<Person>({
|
||||
...baseConfig,
|
||||
mergeThreshold: 0.6,
|
||||
});
|
||||
|
||||
// A ≈ B and B ≈ C → they should all be in one group
|
||||
const records: Person[] = [
|
||||
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
|
||||
{ id: 2, name: "Alice Smyth", email: "alice@example.com" },
|
||||
{ id: 3, name: "Alise Smith", email: "alice@example.com" },
|
||||
];
|
||||
|
||||
const groups = engine.detect(records);
|
||||
expect(groups.length).toBe(1);
|
||||
const group = groups[0] as NonNullable<(typeof groups)[0]>;
|
||||
expect(group.canonical.id).toBe(1);
|
||||
expect(group.duplicates.length).toBe(2);
|
||||
});
|
||||
|
||||
it("scores array is aligned with duplicates array", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
const records: Person[] = [
|
||||
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
|
||||
{ id: 2, name: "Alice Smyth", email: "alice@example.com" },
|
||||
];
|
||||
const groups = engine.detect(records);
|
||||
const group = groups[0];
|
||||
expect(group).toBeDefined();
|
||||
const g = group as NonNullable<typeof group>;
|
||||
expect(g.scores.length).toBe(g.duplicates.length);
|
||||
expect(g.scores[0]).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("respects maxGroupSize", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>({
|
||||
matchKeys: [
|
||||
{ field: "email", weight: 1, strategy: "exact" as const },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.99,
|
||||
maxGroupSize: 3,
|
||||
});
|
||||
|
||||
// 5 entities with the same email will all be exact-matches
|
||||
const records: Person[] = Array.from({ length: 5 }, (_, i) => ({
|
||||
id: i + 1,
|
||||
name: `Person ${i + 1}`,
|
||||
email: "same@example.com",
|
||||
}));
|
||||
|
||||
const groups = engine.detect(records);
|
||||
expect(groups.length).toBeGreaterThanOrEqual(1);
|
||||
const group = groups[0] as NonNullable<(typeof groups)[0]>;
|
||||
// maxGroupSize=3 means canonical + 2 duplicates at most
|
||||
expect(group.duplicates.length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("uses scoring rules when provided", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>({
|
||||
matchKeys: [
|
||||
{ field: "name", weight: 1, strategy: "exact" as const },
|
||||
],
|
||||
scoringRules: [
|
||||
{
|
||||
name: "always-match",
|
||||
evaluate: () => 1.0,
|
||||
},
|
||||
],
|
||||
mergeThreshold: 0.85,
|
||||
});
|
||||
|
||||
const records: Person[] = [
|
||||
{ id: 1, name: "Alice", email: "a@example.com" },
|
||||
{ id: 2, name: "Bob", email: "b@example.com" },
|
||||
];
|
||||
|
||||
// The scoring rule overrides everything with 1.0, so they should match
|
||||
const groups = engine.detect(records);
|
||||
expect(groups.length).toBe(1);
|
||||
});
|
||||
|
||||
it("respects caseSensitive: true and does not lowercase field values", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine<Person>({
|
||||
matchKeys: [
|
||||
{
|
||||
field: "name",
|
||||
weight: 1,
|
||||
strategy: "exact" as const,
|
||||
caseSensitive: true,
|
||||
},
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.99,
|
||||
});
|
||||
|
||||
// "Alice" !== "alice" when caseSensitive is true → no duplicates
|
||||
const records: Person[] = [
|
||||
{ id: 1, name: "Alice", email: "a@example.com" },
|
||||
{ id: 2, name: "alice", email: "a@example.com" },
|
||||
];
|
||||
expect(engine.detect(records)).toEqual([]);
|
||||
|
||||
// Exact same case → duplicate
|
||||
const records2: Person[] = [
|
||||
{ id: 1, name: "Alice", email: "a@example.com" },
|
||||
{ id: 2, name: "Alice", email: "a@example.com" },
|
||||
];
|
||||
expect(engine.detect(records2).length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("DedupEngine.merge", () => {
|
||||
it("delegates to the provided strategy", { timeout: 5000 }, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
const group = {
|
||||
canonical: { id: 1, name: "Alice", email: "a@example.com" },
|
||||
duplicates: [{ id: 2, name: "Alicia", email: "a@example.com" }],
|
||||
scores: [0.9],
|
||||
};
|
||||
|
||||
const strategy: MergeStrategy<Person> = (g) => ({
|
||||
merged: g.canonical,
|
||||
sources: [g.canonical, ...g.duplicates],
|
||||
strategy: "test-strategy",
|
||||
});
|
||||
|
||||
const result = engine.merge(group, strategy);
|
||||
expect(result.strategy).toBe("test-strategy");
|
||||
expect(result.merged).toBe(group.canonical);
|
||||
expect(result.sources).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("wraps strategy errors as DedupError(STRATEGY_ERROR)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
const group: DuplicateGroup<Person> = {
|
||||
canonical: { id: 1, name: "Alice", email: "a@example.com" },
|
||||
duplicates: [],
|
||||
scores: [],
|
||||
};
|
||||
|
||||
const badStrategy: MergeStrategy<Person> = () => {
|
||||
throw new Error("strategy exploded");
|
||||
};
|
||||
|
||||
expect(() => engine.merge(group, badStrategy)).toThrow(DedupError);
|
||||
try {
|
||||
engine.merge(group, badStrategy);
|
||||
} catch (err) {
|
||||
expect((err as DedupError).code).toBe(DEDUP_CODES.STRATEGY_ERROR);
|
||||
expect((err as DedupError).message).toContain("strategy exploded");
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps non-Error thrown values as DedupError(STRATEGY_ERROR)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine<Person>(baseConfig);
|
||||
const group: DuplicateGroup<Person> = {
|
||||
canonical: { id: 1, name: "Alice", email: "a@example.com" },
|
||||
duplicates: [],
|
||||
scores: [],
|
||||
};
|
||||
|
||||
const stringThrowStrategy: MergeStrategy<Person> = () => {
|
||||
// eslint-disable-next-line @typescript-eslint/only-throw-error
|
||||
throw "raw string error";
|
||||
};
|
||||
|
||||
try {
|
||||
engine.merge(group, stringThrowStrategy);
|
||||
} catch (err) {
|
||||
expect((err as DedupError).code).toBe(DEDUP_CODES.STRATEGY_ERROR);
|
||||
expect((err as DedupError).message).toContain("raw string error");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("DedupEngine.detect - edge cases", () => {
|
||||
it("handles null/undefined entity fields gracefully", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine({
|
||||
matchKeys: [
|
||||
{
|
||||
field: "nested.value",
|
||||
weight: 1,
|
||||
strategy: "exact" as const,
|
||||
},
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.99,
|
||||
});
|
||||
|
||||
// entity with null nested path — getField should return undefined, toStr → ""
|
||||
const records = [
|
||||
{ nested: null },
|
||||
{ nested: { value: "hello" } },
|
||||
{ other: "x" }, // no 'nested' key at all
|
||||
];
|
||||
|
||||
// Should not throw; dissimilar so no groups
|
||||
expect(() => engine.detect(records)).not.toThrow();
|
||||
});
|
||||
|
||||
it("handles entity with non-object intermediate field", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const engine = new DedupEngine({
|
||||
matchKeys: [
|
||||
{ field: "a.b.c", weight: 1, strategy: "exact" as const },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.99,
|
||||
});
|
||||
|
||||
// 'a' is a primitive, so path traversal hits typeof !== "object"
|
||||
const records = [{ a: "not-an-object" }, { a: "also-not-an-object" }];
|
||||
expect(() => engine.detect(records)).not.toThrow();
|
||||
});
|
||||
});
|
||||
235
packages/foundation/data/dedup/src/core/engine/dedup-engine.ts
Normal file
235
packages/foundation/data/dedup/src/core/engine/dedup-engine.ts
Normal file
@@ -0,0 +1,235 @@
|
||||
import type { Tracer } from "@opentelemetry/api";
|
||||
import { metrics, trace } from "@opentelemetry/api";
|
||||
import { DEDUP_CODES, DedupError } from "../../errors/dedup-error.js";
|
||||
import {
|
||||
type DedupConfig,
|
||||
DedupConfigSchema,
|
||||
type MatchKey,
|
||||
} from "../../schemas/dedup.schema.js";
|
||||
import { computeSimilarity } from "../scoring/similarity.js";
|
||||
import type {
|
||||
DuplicateGroup,
|
||||
MergeResult,
|
||||
MergeStrategy,
|
||||
} from "./dedup-engine.types.js";
|
||||
|
||||
/**
|
||||
* Retrieve a nested field value from an entity using dot-notation.
|
||||
* Returns `undefined` if any segment along the path is missing.
|
||||
*/
|
||||
function getField(entity: unknown, field: string): unknown {
|
||||
const parts = field.split(".");
|
||||
let current: unknown = entity;
|
||||
for (const part of parts) {
|
||||
if (current === null || current === undefined) return undefined;
|
||||
if (typeof current !== "object") return undefined;
|
||||
current = (current as Record<string, unknown>)[part];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coerce a raw field value to a string for similarity comparison.
|
||||
* `null` / `undefined` become empty string.
|
||||
*/
|
||||
function toStr(value: unknown): string {
|
||||
if (value === null || value === undefined) return "";
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a weighted similarity score between two entities based on match keys.
|
||||
* Each match key contributes `weight * similarity(field_a, field_b)` to the total.
|
||||
*/
|
||||
function pairScore(a: unknown, b: unknown, keys: MatchKey[]): number {
|
||||
let score = 0;
|
||||
for (const key of keys) {
|
||||
let va = toStr(getField(a, key.field));
|
||||
let vb = toStr(getField(b, key.field));
|
||||
if (!key.caseSensitive) {
|
||||
va = va.toLowerCase();
|
||||
vb = vb.toLowerCase();
|
||||
}
|
||||
score += key.weight * computeSimilarity(va, vb, key.strategy);
|
||||
}
|
||||
return score;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build connected components from an adjacency list (union-find).
|
||||
* Returns an array of arrays, each containing the indices of entities in that component.
|
||||
*/
|
||||
function buildComponents(n: number, edges: [number, number][]): number[][] {
|
||||
const parent = Array.from({ length: n }, (_, i) => i);
|
||||
|
||||
function find(x: number): number {
|
||||
while (parent[x] !== x) {
|
||||
// Path-halving: indices are bounded by Array.from initialisation, always valid
|
||||
parent[x] = parent[parent[x]!]!;
|
||||
x = parent[x]!;
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
function union(x: number, y: number): void {
|
||||
const px = find(x);
|
||||
const py = find(y);
|
||||
if (px !== py) parent[px] = py;
|
||||
}
|
||||
|
||||
for (const [i, j] of edges) {
|
||||
union(i, j);
|
||||
}
|
||||
|
||||
const map = new Map<number, number[]>();
|
||||
for (let i = 0; i < n; i++) {
|
||||
const root = find(i);
|
||||
if (!map.has(root)) map.set(root, []);
|
||||
(map.get(root) as number[]).push(i);
|
||||
}
|
||||
|
||||
return [...map.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Entity deduplication engine.
|
||||
*
|
||||
* Accepts a `DedupConfig` describing how to compare entity fields and what
|
||||
* similarity threshold triggers a duplicate. Call `detect()` to find groups of
|
||||
* duplicate entities, then optionally call `merge()` on each group with a
|
||||
* chosen `MergeStrategy`.
|
||||
*/
|
||||
export class DedupEngine<T = Record<string, unknown>> {
|
||||
protected readonly config: DedupConfig;
|
||||
protected readonly tracer: Tracer;
|
||||
|
||||
constructor(config: DedupConfig) {
|
||||
const result = DedupConfigSchema.safeParse(config);
|
||||
if (!result.success) {
|
||||
throw new DedupError(
|
||||
`Invalid DedupConfig: ${result.error.message}`,
|
||||
DEDUP_CODES.INVALID_CONFIG
|
||||
);
|
||||
}
|
||||
this.config = result.data;
|
||||
this.tracer = trace.getTracer("@suaveplan/dedup");
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect groups of duplicate entities.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. For every ordered pair (i, j) where i < j, compute a weighted similarity score.
|
||||
* 2. If the score meets `mergeThreshold`, record an edge between i and j.
|
||||
* 3. Use union-find to build connected components from those edges.
|
||||
* 4. Groups with more than one member are emitted as `DuplicateGroup` records.
|
||||
* 5. Within each group the first entity (lowest index) is the canonical.
|
||||
*
|
||||
* OTel: emits `dedup.detect.total` counter and `dedup.detect.duration_ms` histogram.
|
||||
*/
|
||||
detect(entities: T[]): DuplicateGroup<T>[] {
|
||||
const span = this.tracer.startSpan("dedup.detect");
|
||||
const startMs = Date.now();
|
||||
|
||||
const meter = metrics.getMeter("@suaveplan/dedup");
|
||||
const detectCounter = meter.createCounter("dedup.detect.total");
|
||||
const durationHistogram = meter.createHistogram(
|
||||
"dedup.detect.duration_ms"
|
||||
);
|
||||
|
||||
const { matchKeys, scoringRules, mergeThreshold, maxGroupSize } =
|
||||
this.config;
|
||||
// mergeThreshold and maxGroupSize always have defaults applied by Zod
|
||||
const threshold = mergeThreshold;
|
||||
const maxSize = maxGroupSize;
|
||||
|
||||
const edges: [number, number][] = [];
|
||||
// pairScores[i][j] stores the score for entities i and j (i < j)
|
||||
const pairScores = new Map<string, number>();
|
||||
|
||||
for (let i = 0; i < entities.length; i++) {
|
||||
for (let j = i + 1; j < entities.length; j++) {
|
||||
// Start with key-based weighted score
|
||||
let score = pairScore(entities[i], entities[j], matchKeys);
|
||||
|
||||
// Apply additional scoring rules (they can override or supplement)
|
||||
for (const rule of scoringRules) {
|
||||
const evalFn = rule["evaluate"] as (
|
||||
a: unknown,
|
||||
b: unknown,
|
||||
keys: MatchKey[]
|
||||
) => number;
|
||||
score = evalFn(entities[i], entities[j], matchKeys);
|
||||
}
|
||||
|
||||
if (score >= threshold) {
|
||||
edges.push([i, j]);
|
||||
pairScores.set(`${i}:${j}`, score);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const components = buildComponents(entities.length, edges);
|
||||
const groups: DuplicateGroup<T>[] = [];
|
||||
|
||||
for (const component of components) {
|
||||
if (component.length < 2) continue;
|
||||
|
||||
// Sort to ensure the canonical is the entity with the lowest original index
|
||||
const sorted = [...component].sort((a, b) => a - b);
|
||||
const canonicalIdx = sorted[0] as number;
|
||||
const duplicateIdxs = sorted.slice(1);
|
||||
|
||||
// Respect maxGroupSize — if the component is too large, truncate duplicates
|
||||
const cappedDuplicates = duplicateIdxs.slice(0, maxSize - 1);
|
||||
|
||||
const duplicates: T[] = cappedDuplicates.map(
|
||||
(idx) => entities[idx] as T
|
||||
);
|
||||
const scores: number[] = cappedDuplicates.map((idx) => {
|
||||
const key = `${Math.min(canonicalIdx, idx)}:${Math.max(canonicalIdx, idx)}`;
|
||||
// Key always exists — only pairs that passed threshold are in cappedDuplicates
|
||||
return pairScores.get(key)!;
|
||||
});
|
||||
|
||||
groups.push({
|
||||
canonical: entities[canonicalIdx] as T,
|
||||
duplicates,
|
||||
scores,
|
||||
});
|
||||
}
|
||||
|
||||
const durationMs = Date.now() - startMs;
|
||||
detectCounter.add(groups.length);
|
||||
durationHistogram.record(durationMs);
|
||||
span.setAttribute("dedup.groups_found", groups.length);
|
||||
span.end();
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge a duplicate group using the provided strategy.
|
||||
*
|
||||
* The strategy function receives the group and must return a `MergeResult`.
|
||||
* Throws `DedupError` with code `STRATEGY_ERROR` if the strategy throws.
|
||||
*/
|
||||
merge(
|
||||
group: DuplicateGroup<T>,
|
||||
strategy: MergeStrategy<T>
|
||||
): MergeResult<T> {
|
||||
const span = this.tracer.startSpan("dedup.merge");
|
||||
try {
|
||||
const result = strategy(group);
|
||||
span.setAttribute("dedup.merge.strategy", result.strategy);
|
||||
span.end();
|
||||
return result;
|
||||
} catch (err) {
|
||||
span.end();
|
||||
throw new DedupError(
|
||||
`Merge strategy failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
DEDUP_CODES.STRATEGY_ERROR
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { MatchKey } from "../../schemas/dedup.schema.js";
|
||||
|
||||
/**
|
||||
* A group of duplicate entities. The `canonical` entity is the representative
|
||||
* (first entity found in the connected component). Each entry in `duplicates`
|
||||
* corresponds to a `score` in `scores`.
|
||||
*/
|
||||
export interface DuplicateGroup<T = unknown> {
|
||||
canonical: T;
|
||||
duplicates: T[];
|
||||
/** Similarity score for each duplicate vs the canonical, aligned by index. */
|
||||
scores: number[];
|
||||
/** Name of the merge strategy applied (set after `merge()` is called). */
|
||||
mergeStrategy?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The result of merging a duplicate group.
|
||||
*/
|
||||
export interface MergeResult<T = unknown> {
|
||||
merged: T;
|
||||
sources: T[];
|
||||
strategy: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A merge function that combines a duplicate group into a single entity.
|
||||
*/
|
||||
export type MergeStrategy<T = unknown> = (
|
||||
group: DuplicateGroup<T>
|
||||
) => MergeResult<T>;
|
||||
|
||||
/**
|
||||
* A custom scoring rule that evaluates the similarity of two entities.
|
||||
* Must return a value in [0, 1].
|
||||
*/
|
||||
export interface ScoringRuleInstance {
|
||||
name: string;
|
||||
evaluate(a: unknown, b: unknown, keys: MatchKey[]): number;
|
||||
}
|
||||
7
packages/foundation/data/dedup/src/core/engine/index.ts
Normal file
7
packages/foundation/data/dedup/src/core/engine/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export { DedupEngine } from "./dedup-engine";
|
||||
export type {
|
||||
DuplicateGroup,
|
||||
MergeResult,
|
||||
MergeStrategy,
|
||||
ScoringRuleInstance,
|
||||
} from "./dedup-engine.types";
|
||||
19
packages/foundation/data/dedup/src/core/index.ts
Normal file
19
packages/foundation/data/dedup/src/core/index.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
export {
|
||||
DedupEngine,
|
||||
type DuplicateGroup,
|
||||
type MergeResult,
|
||||
type MergeStrategy,
|
||||
type ScoringRuleInstance,
|
||||
} from "./engine";
|
||||
export {
|
||||
latestWinsStrategy,
|
||||
manualStrategy,
|
||||
scoredMergeStrategy,
|
||||
} from "./merge";
|
||||
export {
|
||||
computeSimilarity,
|
||||
exactMatch,
|
||||
jaroSimilarity,
|
||||
jaroWinklerSimilarity,
|
||||
levenshteinDistance,
|
||||
} from "./scoring";
|
||||
5
packages/foundation/data/dedup/src/core/merge/index.ts
Normal file
5
packages/foundation/data/dedup/src/core/merge/index.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
latestWinsStrategy,
|
||||
manualStrategy,
|
||||
scoredMergeStrategy,
|
||||
} from "./merge-strategies";
|
||||
@@ -0,0 +1,92 @@
|
||||
# merge-strategies
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides three built-in merge strategies for combining a `DuplicateGroup` into a single entity. Merge strategies are pure functions that conform to the `MergeStrategy<T>` type and can be composed or replaced with custom implementations. They are passed to `DedupEngine.merge()` as callbacks rather than configured up-front, giving callers full control over how and when merging occurs.
|
||||
|
||||
## Features
|
||||
|
||||
- **`latestWinsStrategy`** — selects whichever entity (canonical or duplicate) has the most recent `updatedAt` or `createdAt` timestamp; ISO 8601 strings sort correctly as plain strings so no date parsing is needed.
|
||||
- **`manualStrategy`** — no-op that returns the canonical entity unchanged, useful when a human reviewer should approve merges before they are committed to a data store.
|
||||
- **`scoredMergeStrategy`** — assembles a "best-of-breed" entity by choosing, for each field, the value from the entity with the highest similarity score; the canonical entity acts as the fallback with score 1.0.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { DedupEngine } from "@suaveplan/dedup/engine";
|
||||
import { latestWinsStrategy } from "@suaveplan/dedup/merge";
|
||||
|
||||
const engine = new DedupEngine({
|
||||
matchKeys: [{ field: "name", weight: 1, strategy: "jaro-winkler" }],
|
||||
scoringRules: [],
|
||||
});
|
||||
|
||||
const groups = engine.detect([
|
||||
{ name: "Alice", updatedAt: "2024-01-01T00:00:00Z" },
|
||||
{ name: "Alicia", updatedAt: "2024-06-15T12:00:00Z" },
|
||||
]);
|
||||
|
||||
for (const group of groups) {
|
||||
const result = engine.merge(group, latestWinsStrategy);
|
||||
// result.merged → { name: "Alicia", updatedAt: "2024-06-15T12:00:00Z" }
|
||||
// result.strategy → "latest-wins"
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { DedupEngine } from "@suaveplan/dedup/engine";
|
||||
import { scoredMergeStrategy, manualStrategy } from "@suaveplan/dedup/merge";
|
||||
import type { MergeStrategy } from "@suaveplan/dedup";
|
||||
|
||||
type Person = { id: number; name: string; email: string; phone?: string };
|
||||
|
||||
const engine = new DedupEngine<Person>({
|
||||
matchKeys: [
|
||||
{ field: "name", weight: 0.5, strategy: "jaro-winkler" },
|
||||
{ field: "email", weight: 0.5, strategy: "exact" },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.8,
|
||||
});
|
||||
|
||||
const groups = engine.detect(persons);
|
||||
|
||||
// Pick strategy based on group size
|
||||
for (const group of groups) {
|
||||
const strategy: MergeStrategy<Person> =
|
||||
group.duplicates.length > 5 ? manualStrategy : scoredMergeStrategy;
|
||||
const result = engine.merge(group, strategy);
|
||||
console.log(result.strategy, result.merged);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `latestWinsStrategy<T>(group: DuplicateGroup<T>): MergeResult<T>`
|
||||
|
||||
Picks the entity with the lexicographically largest `updatedAt` or `createdAt` field value. If neither field is present, returns the canonical unchanged. All entities (canonical + duplicates) participate in the comparison.
|
||||
|
||||
**Returns:** `MergeResult<T>` with `strategy: "latest-wins"`
|
||||
|
||||
### `manualStrategy<T>(group: DuplicateGroup<T>): MergeResult<T>`
|
||||
|
||||
Returns the canonical as the merged entity with no field-level changes. Suitable as a placeholder when merge logic will be determined externally.
|
||||
|
||||
**Returns:** `MergeResult<T>` with `strategy: "manual"`
|
||||
|
||||
### `scoredMergeStrategy<T>(group: DuplicateGroup<T>): MergeResult<T>`
|
||||
|
||||
Constructs a merged object by iterating every field key that appears in any entity and assigning the value from the entity with the highest similarity score. The canonical entity always has score `1.0`. Fields with `null` or `undefined` values are skipped when selecting the best value. The result is cast to `T` — callers should ensure the resulting shape is compatible.
|
||||
|
||||
**Returns:** `MergeResult<T>` with `strategy: "scored-merge"`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
All three strategies are stateless functions with no side effects. They do not mutate the `group` argument. The `scoredMergeStrategy` creates a new plain object (`{}`) and populates it field-by-field, which means prototype-based instances (classes) will lose their prototype chain in the merged result. If the entity type uses class instances, callers should wrap `scoredMergeStrategy` and reconstruct the class after merging.
|
||||
|
||||
`latestWinsStrategy` relies on ISO 8601 string sort order. Timestamps in other formats (Unix seconds as numbers, locale-formatted strings) will not sort correctly. Callers storing dates in non-ISO formats should either normalise before dedup or supply a custom `MergeStrategy` that parses timestamps explicitly.
|
||||
@@ -0,0 +1,232 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import type { DuplicateGroup } from "../engine/dedup-engine.types.js";
|
||||
import {
|
||||
latestWinsStrategy,
|
||||
manualStrategy,
|
||||
scoredMergeStrategy,
|
||||
} from "./merge-strategies.js";
|
||||
|
||||
type Person = {
|
||||
id: number;
|
||||
name: string;
|
||||
email: string;
|
||||
phone?: string;
|
||||
updatedAt?: string;
|
||||
createdAt?: string;
|
||||
};
|
||||
|
||||
function makeGroup(
|
||||
canonical: Person,
|
||||
duplicates: Person[],
|
||||
scores: number[]
|
||||
): DuplicateGroup<Person> {
|
||||
return { canonical, duplicates, scores };
|
||||
}
|
||||
|
||||
describe("latestWinsStrategy", () => {
|
||||
it("returns the entity with the most recent updatedAt", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const group = makeGroup(
|
||||
{
|
||||
id: 1,
|
||||
name: "Alice",
|
||||
email: "a@example.com",
|
||||
updatedAt: "2023-01-01T00:00:00Z",
|
||||
},
|
||||
[
|
||||
{
|
||||
id: 2,
|
||||
name: "Alicia",
|
||||
email: "a@example.com",
|
||||
updatedAt: "2024-06-15T12:00:00Z",
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
name: "Alise",
|
||||
email: "a@example.com",
|
||||
updatedAt: "2022-03-10T08:00:00Z",
|
||||
},
|
||||
],
|
||||
[0.9, 0.85]
|
||||
);
|
||||
|
||||
const result = latestWinsStrategy(group);
|
||||
expect(result.strategy).toBe("latest-wins");
|
||||
expect((result.merged as Person).id).toBe(2);
|
||||
expect(result.sources).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("falls back to canonical when no date fields present", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const group = makeGroup(
|
||||
{ id: 1, name: "Alice", email: "a@example.com" },
|
||||
[{ id: 2, name: "Alicia", email: "a@example.com" }],
|
||||
[0.9]
|
||||
);
|
||||
|
||||
const result = latestWinsStrategy(group);
|
||||
expect((result.merged as Person).id).toBe(1);
|
||||
});
|
||||
|
||||
it("uses createdAt when updatedAt is absent", { timeout: 5000 }, () => {
|
||||
const group = makeGroup(
|
||||
{
|
||||
id: 1,
|
||||
name: "Alice",
|
||||
email: "a@example.com",
|
||||
createdAt: "2022-01-01T00:00:00Z",
|
||||
},
|
||||
[
|
||||
{
|
||||
id: 2,
|
||||
name: "Alicia",
|
||||
email: "a@example.com",
|
||||
createdAt: "2024-01-01T00:00:00Z",
|
||||
},
|
||||
],
|
||||
[0.9]
|
||||
);
|
||||
|
||||
const result = latestWinsStrategy(group);
|
||||
expect((result.merged as Person).id).toBe(2);
|
||||
});
|
||||
|
||||
it("returns canonical when it has the latest updatedAt", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const group = makeGroup(
|
||||
{
|
||||
id: 1,
|
||||
name: "Alice",
|
||||
email: "a@example.com",
|
||||
updatedAt: "2025-01-01T00:00:00Z",
|
||||
},
|
||||
[
|
||||
{
|
||||
id: 2,
|
||||
name: "Alicia",
|
||||
email: "a@example.com",
|
||||
updatedAt: "2024-01-01T00:00:00Z",
|
||||
},
|
||||
],
|
||||
[0.9]
|
||||
);
|
||||
|
||||
const result = latestWinsStrategy(group);
|
||||
expect((result.merged as Person).id).toBe(1);
|
||||
});
|
||||
|
||||
it("includes all sources", { timeout: 5000 }, () => {
|
||||
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
|
||||
const dup = { id: 2, name: "Alicia", email: "a@example.com" };
|
||||
const result = latestWinsStrategy(makeGroup(canonical, [dup], [0.9]));
|
||||
expect(result.sources).toContain(canonical);
|
||||
expect(result.sources).toContain(dup);
|
||||
});
|
||||
});
|
||||
|
||||
describe("manualStrategy", () => {
|
||||
it("returns the canonical as merged, unchanged", { timeout: 5000 }, () => {
|
||||
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
|
||||
const dup = { id: 2, name: "Alicia", email: "a@example.com" };
|
||||
const group = makeGroup(canonical, [dup], [0.9]);
|
||||
|
||||
const result = manualStrategy(group);
|
||||
expect(result.strategy).toBe("manual");
|
||||
expect(result.merged).toBe(canonical); // same reference
|
||||
expect(result.sources).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("includes canonical and all duplicates in sources", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
|
||||
const dups = [
|
||||
{ id: 2, name: "Alicia", email: "a@example.com" },
|
||||
{ id: 3, name: "Alice", email: "alice@other.com" },
|
||||
];
|
||||
const result = manualStrategy(makeGroup(canonical, dups, [0.9, 0.85]));
|
||||
expect(result.sources).toEqual([canonical, ...dups]);
|
||||
});
|
||||
|
||||
it("works when there are no duplicates", { timeout: 5000 }, () => {
|
||||
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
|
||||
const result = manualStrategy(makeGroup(canonical, [], []));
|
||||
expect(result.merged).toBe(canonical);
|
||||
expect(result.sources).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scoredMergeStrategy", () => {
|
||||
it("picks field values from the highest-scored entity", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const canonical = {
|
||||
id: 1,
|
||||
name: "Alice",
|
||||
email: "a@example.com",
|
||||
phone: undefined,
|
||||
};
|
||||
// dup has a phone number and a higher score (0.95 vs canonical's implicit 1.0)
|
||||
const dup = {
|
||||
id: 2,
|
||||
name: "Alicia",
|
||||
email: "a@example.com",
|
||||
phone: "555-1234",
|
||||
};
|
||||
const group = makeGroup(canonical, [dup], [0.95]);
|
||||
|
||||
const result = scoredMergeStrategy(group);
|
||||
expect(result.strategy).toBe("scored-merge");
|
||||
const merged = result.merged as Person;
|
||||
// canonical has score 1.0 so its non-undefined fields take precedence
|
||||
expect(merged.id).toBe(1);
|
||||
expect(merged.email).toBe("a@example.com");
|
||||
// phone is undefined in canonical so dup's phone should be used
|
||||
expect(merged.phone).toBe("555-1234");
|
||||
});
|
||||
|
||||
it("uses canonical (score 1.0) as fallback for all fields", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const canonical = {
|
||||
id: 1,
|
||||
name: "Alice Smith",
|
||||
email: "alice@example.com",
|
||||
};
|
||||
const dup = { id: 2, name: "Alice Smyth", email: "alice@example.com" };
|
||||
const group = makeGroup(canonical, [dup], [0.85]);
|
||||
|
||||
const result = scoredMergeStrategy(group);
|
||||
const merged = result.merged as Person;
|
||||
// canonical score is 1.0 > 0.85 so all fields come from canonical
|
||||
expect(merged.name).toBe("Alice Smith");
|
||||
});
|
||||
|
||||
it("returns all sources", { timeout: 5000 }, () => {
|
||||
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
|
||||
const dup = { id: 2, name: "Alicia", email: "a@example.com" };
|
||||
const result = scoredMergeStrategy(makeGroup(canonical, [dup], [0.9]));
|
||||
expect(result.sources).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("handles group with no duplicates", { timeout: 5000 }, () => {
|
||||
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
|
||||
const result = scoredMergeStrategy(makeGroup(canonical, [], []));
|
||||
const merged = result.merged as Person;
|
||||
expect(merged.name).toBe("Alice");
|
||||
});
|
||||
|
||||
it("handles non-object entities gracefully", { timeout: 5000 }, () => {
|
||||
// Entities that are primitives or null-like are not objects, no keys to extract
|
||||
const group: DuplicateGroup<unknown> = {
|
||||
canonical: null,
|
||||
duplicates: [null],
|
||||
scores: [0.9],
|
||||
};
|
||||
const result = scoredMergeStrategy(group);
|
||||
expect(result.strategy).toBe("scored-merge");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import type {
|
||||
DuplicateGroup,
|
||||
MergeResult,
|
||||
} from "../engine/dedup-engine.types.js";
|
||||
|
||||
/**
|
||||
* Select the most recently updated entity as the merged result.
|
||||
*
|
||||
* Inspects each entity (canonical + duplicates) for an `updatedAt` or
|
||||
* `createdAt` field. Compares them as strings (ISO 8601 dates sort correctly
|
||||
* lexicographically) and returns the entity with the largest value.
|
||||
* If no date field is found, returns the canonical unchanged.
|
||||
*/
|
||||
export function latestWinsStrategy<T>(
|
||||
group: DuplicateGroup<T>
|
||||
): MergeResult<T> {
|
||||
const all = [group.canonical, ...group.duplicates];
|
||||
|
||||
function getDate(entity: T): string {
|
||||
const e = entity as Record<string, unknown>;
|
||||
const v = e["updatedAt"] ?? e["createdAt"];
|
||||
return typeof v === "string" ? v : "";
|
||||
}
|
||||
|
||||
const winner = all.reduce((best, current) =>
|
||||
getDate(current) > getDate(best) ? current : best
|
||||
);
|
||||
|
||||
return {
|
||||
merged: winner,
|
||||
sources: all,
|
||||
strategy: "latest-wins",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Manual / no-op merge strategy.
|
||||
*
|
||||
* Returns the canonical entity as the merged result and lists all sources
|
||||
* (canonical + duplicates). No field merging is performed; this strategy is
|
||||
* useful when human review is required before committing a merge.
|
||||
*/
|
||||
export function manualStrategy<T>(group: DuplicateGroup<T>): MergeResult<T> {
|
||||
return {
|
||||
merged: group.canonical,
|
||||
sources: [group.canonical, ...group.duplicates],
|
||||
strategy: "manual",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Scored merge strategy.
|
||||
*
|
||||
* For each field present in any entity, picks the value from the entity with
|
||||
* the highest similarity score vs the canonical. The canonical itself is
|
||||
* treated as having score 1.0, so its fields are used as the fallback.
|
||||
*
|
||||
* This strategy produces a "best-of-breed" merged entity assembled from
|
||||
* the most confident field values across all duplicates.
|
||||
*/
|
||||
export function scoredMergeStrategy<T>(
|
||||
group: DuplicateGroup<T>
|
||||
): MergeResult<T> {
|
||||
const all = [group.canonical, ...group.duplicates];
|
||||
const allScores = [1.0, ...group.scores];
|
||||
|
||||
// Collect all field keys across every entity
|
||||
const allKeys = new Set<string>();
|
||||
for (const entity of all) {
|
||||
if (entity !== null && typeof entity === "object") {
|
||||
for (const key of Object.keys(entity as object)) {
|
||||
allKeys.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For each field, find the entity with the highest score that has a non-null value
|
||||
const merged: Record<string, unknown> = {};
|
||||
for (const key of allKeys) {
|
||||
let bestValue: unknown;
|
||||
let bestScore = -1;
|
||||
for (let i = 0; i < all.length; i++) {
|
||||
const entity = all[i] as Record<string, unknown>;
|
||||
const value = entity[key];
|
||||
// allScores is constructed as [1.0, ...group.scores], always aligned with `all`
|
||||
const score = allScores[i]!;
|
||||
if (value !== null && value !== undefined && score > bestScore) {
|
||||
bestScore = score;
|
||||
bestValue = value;
|
||||
}
|
||||
}
|
||||
merged[key] = bestValue;
|
||||
}
|
||||
|
||||
return {
|
||||
merged: merged as T,
|
||||
sources: all,
|
||||
strategy: "scored-merge",
|
||||
};
|
||||
}
|
||||
7
packages/foundation/data/dedup/src/core/scoring/index.ts
Normal file
7
packages/foundation/data/dedup/src/core/scoring/index.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
computeSimilarity,
|
||||
exactMatch,
|
||||
jaroSimilarity,
|
||||
jaroWinklerSimilarity,
|
||||
levenshteinDistance,
|
||||
} from "./similarity";
|
||||
@@ -0,0 +1,94 @@
|
||||
# similarity
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides string similarity algorithms used by the dedup engine to compare entity field values. Three well-known distance / similarity metrics are implemented from scratch: Levenshtein edit distance, Jaro similarity, and Jaro-Winkler similarity. A phonetic normalisation path is also included for fuzzy name matching. All functions are pure and side-effect-free, operating only on their arguments.
|
||||
|
||||
## Features
|
||||
|
||||
- **Levenshtein distance** — classic Wagner–Fischer dynamic-programming algorithm with O(min(|a|, |b|)) space optimisation (two-row rolling array).
|
||||
- **Jaro similarity** — accounts for matching characters within a dynamic window and transposition pairs; returns a score in [0, 1].
|
||||
- **Jaro-Winkler similarity** — extends Jaro by boosting strings that share a common prefix (up to 4 characters) using the standard p = 0.1 scaling factor.
|
||||
- **Exact match** — binary 1 / 0 equality check for deterministic field comparison.
|
||||
- **Phonetic match** — lightweight vowel-removal normalisation followed by Jaro-Winkler, useful for names that sound alike but are spelled differently.
|
||||
- **`computeSimilarity` dispatcher** — unified entry point that routes to the right algorithm based on the `MatchKey.strategy` discriminant.
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { levenshteinDistance, jaroWinklerSimilarity, computeSimilarity } from "@suaveplan/dedup/scoring";
|
||||
|
||||
levenshteinDistance("kitten", "sitting"); // 3
|
||||
jaroWinklerSimilarity("MARTHA", "MARHTA"); // ≈ 0.961
|
||||
|
||||
// Use the dispatcher for engine integration
|
||||
const score = computeSimilarity("Alice", "Alicia", "jaro-winkler"); // ≈ 0.947
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { computeSimilarity } from "@suaveplan/dedup/scoring";
|
||||
import type { MatchKey } from "@suaveplan/dedup";
|
||||
|
||||
const keys: MatchKey[] = [
|
||||
{ field: "name", weight: 0.5, strategy: "jaro-winkler", caseSensitive: false },
|
||||
{ field: "email", weight: 0.3, strategy: "exact", caseSensitive: false },
|
||||
{ field: "city", weight: 0.2, strategy: "phonetic", caseSensitive: false },
|
||||
];
|
||||
|
||||
function pairScore(a: Record<string, string>, b: Record<string, string>): number {
|
||||
return keys.reduce((sum, k) => {
|
||||
const va = (a[k.field] ?? "").toLowerCase();
|
||||
const vb = (b[k.field] ?? "").toLowerCase();
|
||||
return sum + k.weight * computeSimilarity(va, vb, k.strategy);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
const score = pairScore(
|
||||
{ name: "John Smith", email: "j@example.com", city: "New York" },
|
||||
{ name: "Jon Smyth", email: "j@example.com", city: "New York" },
|
||||
);
|
||||
// score ≈ 0.93 — high confidence duplicate
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `levenshteinDistance(a, b): number`
|
||||
|
||||
Returns the minimum number of single-character edits (insertions, deletions, substitutions) required to transform `a` into `b`. Always ≥ 0. Runs in O(|a| × |b|) time with O(min(|a|, |b|)) space.
|
||||
|
||||
### `jaroSimilarity(s1, s2): number`
|
||||
|
||||
Jaro metric. Returns a value in [0, 1]. The matching window is `floor(max(|s1|, |s2|) / 2) - 1`. Identical strings return 1; completely different strings return 0.
|
||||
|
||||
### `jaroWinklerSimilarity(s1, s2): number`
|
||||
|
||||
Jaro-Winkler metric. Applies a prefix bonus: `jaro + prefix * 0.1 * (1 - jaro)` where `prefix` is the length of the shared prefix capped at 4. Returns a value in [0, 1].
|
||||
|
||||
### `exactMatch(a, b): number`
|
||||
|
||||
Returns `1` if `a === b`, `0` otherwise. No normalisation is applied; callers should lower-case inputs if case-insensitive comparison is required.
|
||||
|
||||
### `computeSimilarity(a, b, strategy): number`
|
||||
|
||||
Routes to the correct algorithm. For `"levenshtein"` the raw distance is normalised to [0, 1] by dividing by `max(|a|, |b|)` and subtracting from 1. For `"phonetic"`, both strings are first passed through a vowel-removal normaliser before Jaro-Winkler scoring.
|
||||
|
||||
**Params:**
|
||||
|
||||
- `a` — first string
|
||||
- `b` — second string
|
||||
- `strategy` — `"exact" | "jaro-winkler" | "levenshtein" | "phonetic"`
|
||||
|
||||
**Returns:** `number` in [0, 1]
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The two-row rolling array in `levenshteinDistance` avoids allocating an (|a|+1)×(|b|+1) matrix; only the previous and current rows are kept in memory. The shorter string is always placed on the column axis to minimise allocations.
|
||||
|
||||
The Jaro matching window formula uses `Math.max(|s1|, |s2|) / 2 - 1` and is floored, matching the original Winkler (1990) paper. Some implementations use `Math.ceil`; this library matches the floor variant which produces the widely-cited benchmark values (MARTHA/MARHTA ≈ 0.944 Jaro, ≈ 0.961 Jaro-Winkler).
|
||||
|
||||
The phonetic normaliser is intentionally lightweight — it is not a full Soundex or Metaphone implementation. It removes vowels and collapses consecutive duplicate consonants, which is sufficient for the common use case of matching first/last names with minor spelling variations. For production-grade phonetic matching, callers should supply a custom `ScoringRule`.
|
||||
@@ -0,0 +1,210 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import fc from "fast-check";
|
||||
import {
|
||||
computeSimilarity,
|
||||
exactMatch,
|
||||
jaroSimilarity,
|
||||
jaroWinklerSimilarity,
|
||||
levenshteinDistance,
|
||||
} from "./similarity.js";
|
||||
|
||||
describe("levenshteinDistance", () => {
|
||||
it("returns 0 for identical empty strings", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("", "")).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 0 for identical non-empty strings", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("abc", "abc")).toBe(0);
|
||||
});
|
||||
|
||||
it("returns length of b when a is empty", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("", "abc")).toBe(3);
|
||||
});
|
||||
|
||||
it("returns length of a when b is empty", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("abc", "")).toBe(3);
|
||||
});
|
||||
|
||||
it("kitten → sitting = 3", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("kitten", "sitting")).toBe(3);
|
||||
});
|
||||
|
||||
it("sunday → saturday = 3", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("sunday", "saturday")).toBe(3);
|
||||
});
|
||||
|
||||
it("single substitution", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("a", "b")).toBe(1);
|
||||
});
|
||||
|
||||
it("single insertion", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("ab", "abc")).toBe(1);
|
||||
});
|
||||
|
||||
it("is symmetric", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("abc", "xyz")).toBe(
|
||||
levenshteinDistance("xyz", "abc")
|
||||
);
|
||||
});
|
||||
|
||||
it("property: always non-negative", { timeout: 30000 }, () => {
|
||||
fc.assert(
|
||||
fc.property(fc.string(), fc.string(), (a, b) => {
|
||||
expect(levenshteinDistance(a, b)).toBeGreaterThanOrEqual(0);
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("property: triangle inequality", { timeout: 30000 }, () => {
|
||||
fc.assert(
|
||||
fc.property(fc.string(), fc.string(), fc.string(), (a, b, c) => {
|
||||
const ab = levenshteinDistance(a, b);
|
||||
const bc = levenshteinDistance(b, c);
|
||||
const ac = levenshteinDistance(a, c);
|
||||
expect(ac).toBeLessThanOrEqual(ab + bc);
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("property: distance(a, b) === 0 iff a === b", { timeout: 30000 }, () => {
|
||||
fc.assert(
|
||||
fc.property(fc.string(), fc.string(), (a, b) => {
|
||||
const d = levenshteinDistance(a, b);
|
||||
if (a === b) {
|
||||
expect(d).toBe(0);
|
||||
} else {
|
||||
expect(d).toBeGreaterThan(0);
|
||||
}
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("jaroSimilarity", () => {
|
||||
it("returns 1 for identical strings", { timeout: 5000 }, () => {
|
||||
expect(jaroSimilarity("abc", "abc")).toBe(1);
|
||||
});
|
||||
|
||||
it("returns 0 for empty strings against non-empty", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(jaroSimilarity("", "abc")).toBe(0);
|
||||
expect(jaroSimilarity("abc", "")).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 1 for two empty strings", { timeout: 5000 }, () => {
|
||||
expect(jaroSimilarity("", "")).toBe(1);
|
||||
});
|
||||
|
||||
it("MARTHA vs MARHTA ≈ 0.944", { timeout: 5000 }, () => {
|
||||
const j = jaroSimilarity("MARTHA", "MARHTA");
|
||||
expect(j).toBeGreaterThan(0.94);
|
||||
expect(j).toBeLessThan(0.95);
|
||||
});
|
||||
|
||||
it("completely different strings returns low score", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(jaroSimilarity("abc", "xyz")).toBeLessThan(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("jaroWinklerSimilarity", () => {
|
||||
it("MARTHA vs MARHTA ≈ 0.961", { timeout: 5000 }, () => {
|
||||
const jw = jaroWinklerSimilarity("MARTHA", "MARHTA");
|
||||
expect(Math.abs(jw - 0.961)).toBeLessThan(0.001);
|
||||
});
|
||||
|
||||
it("DIXON vs DICKSONX ≈ 0.813", { timeout: 5000 }, () => {
|
||||
const jw = jaroWinklerSimilarity("DIXON", "DICKSONX");
|
||||
expect(Math.abs(jw - 0.813)).toBeLessThan(0.01);
|
||||
});
|
||||
|
||||
it("identical strings return 1", { timeout: 5000 }, () => {
|
||||
expect(jaroWinklerSimilarity("hello", "hello")).toBe(1);
|
||||
});
|
||||
|
||||
it("empty vs non-empty returns 0", { timeout: 5000 }, () => {
|
||||
expect(jaroWinklerSimilarity("", "abc")).toBe(0);
|
||||
});
|
||||
|
||||
it("result is in [0, 1]", { timeout: 5000 }, () => {
|
||||
const jw = jaroWinklerSimilarity("test", "testing");
|
||||
expect(jw).toBeGreaterThanOrEqual(0);
|
||||
expect(jw).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("shared prefix boosts score vs plain jaro", { timeout: 5000 }, () => {
|
||||
const jw = jaroWinklerSimilarity("prefix", "prefix-extra");
|
||||
const j = jaroSimilarity("prefix", "prefix-extra");
|
||||
expect(jw).toBeGreaterThanOrEqual(j);
|
||||
});
|
||||
});
|
||||
|
||||
describe("exactMatch", () => {
|
||||
it("returns 1 for equal strings", { timeout: 5000 }, () => {
|
||||
expect(exactMatch("abc", "abc")).toBe(1);
|
||||
});
|
||||
|
||||
it("returns 0 for different strings", { timeout: 5000 }, () => {
|
||||
expect(exactMatch("abc", "xyz")).toBe(0);
|
||||
});
|
||||
|
||||
it("is case-sensitive", { timeout: 5000 }, () => {
|
||||
expect(exactMatch("ABC", "abc")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeSimilarity", () => {
|
||||
it("exact strategy: equal strings return 1", { timeout: 5000 }, () => {
|
||||
expect(computeSimilarity("a", "a", "exact")).toBe(1);
|
||||
});
|
||||
|
||||
it("exact strategy: different strings return 0", { timeout: 5000 }, () => {
|
||||
expect(computeSimilarity("a", "b", "exact")).toBe(0);
|
||||
});
|
||||
|
||||
it("jaro-winkler strategy delegates to jaroWinklerSimilarity", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const direct = jaroWinklerSimilarity("MARTHA", "MARHTA");
|
||||
const via = computeSimilarity("MARTHA", "MARHTA", "jaro-winkler");
|
||||
expect(via).toBe(direct);
|
||||
});
|
||||
|
||||
it("levenshtein strategy: identical strings return 1", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(computeSimilarity("abc", "abc", "levenshtein")).toBe(1);
|
||||
});
|
||||
|
||||
it("levenshtein strategy: completely different strings return low score", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const score = computeSimilarity("abc", "xyz", "levenshtein");
|
||||
expect(score).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
it("levenshtein strategy: both empty strings return 1", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(computeSimilarity("", "", "levenshtein")).toBe(1);
|
||||
});
|
||||
|
||||
it("levenshtein strategy: one empty returns 0", { timeout: 5000 }, () => {
|
||||
expect(computeSimilarity("", "abc", "levenshtein")).toBe(0);
|
||||
});
|
||||
|
||||
it("phonetic strategy: similar sounding names score high", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const score = computeSimilarity("smith", "smyth", "phonetic");
|
||||
expect(score).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
it("phonetic strategy: identical strings return 1", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(computeSimilarity("alice", "alice", "phonetic")).toBe(1);
|
||||
});
|
||||
});
|
||||
158
packages/foundation/data/dedup/src/core/scoring/similarity.ts
Normal file
158
packages/foundation/data/dedup/src/core/scoring/similarity.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
import type { MatchKey } from "../../schemas/dedup.schema.js";
|
||||
|
||||
/**
|
||||
* Compute the Levenshtein edit distance between two strings using the
|
||||
* classic Wagner–Fischer dynamic-programming algorithm.
|
||||
*
|
||||
* Time: O(|a| × |b|) Space: O(min(|a|, |b|))
|
||||
*/
|
||||
export function levenshteinDistance(a: string, b: string): number {
|
||||
if (a === b) return 0;
|
||||
if (a.length === 0) return b.length;
|
||||
if (b.length === 0) return a.length;
|
||||
|
||||
// Keep only two rows to save memory
|
||||
let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
|
||||
const curr = new Array<number>(b.length + 1);
|
||||
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
// Indices j and j-1 are always valid within the loop bounds
|
||||
curr[j] = Math.min(
|
||||
prev[j]! + 1, // deletion
|
||||
curr[j - 1]! + 1, // insertion
|
||||
prev[j - 1]! + cost // substitution
|
||||
);
|
||||
}
|
||||
// swap
|
||||
[prev] = [[...curr], prev];
|
||||
}
|
||||
|
||||
// prev[b.length] is always initialised by the loop (or the base-case Array.from)
|
||||
return prev[b.length]!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the Jaro similarity between two strings.
|
||||
*
|
||||
* The Jaro score accounts for common characters within a match radius and
|
||||
* transpositions. Returns a value in [0, 1] where 1 means identical.
|
||||
*/
|
||||
export function jaroSimilarity(s1: string, s2: string): number {
|
||||
if (s1 === s2) return 1;
|
||||
if (s1.length === 0 || s2.length === 0) return 0;
|
||||
|
||||
const matchRadius = Math.max(
|
||||
0,
|
||||
Math.floor(Math.max(s1.length, s2.length) / 2) - 1
|
||||
);
|
||||
|
||||
const s1Matched = new Array<boolean>(s1.length).fill(false);
|
||||
const s2Matched = new Array<boolean>(s2.length).fill(false);
|
||||
|
||||
let matches = 0;
|
||||
|
||||
// Find matches within the matching radius
|
||||
for (let i = 0; i < s1.length; i++) {
|
||||
const start = Math.max(0, i - matchRadius);
|
||||
const end = Math.min(i + matchRadius + 1, s2.length);
|
||||
for (let j = start; j < end; j++) {
|
||||
if (!s2Matched[j] && s1[i] === s2[j]) {
|
||||
s1Matched[i] = true;
|
||||
s2Matched[j] = true;
|
||||
matches++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (matches === 0) return 0;
|
||||
|
||||
// Count transpositions
|
||||
let transpositions = 0;
|
||||
let k = 0;
|
||||
for (let i = 0; i < s1.length; i++) {
|
||||
if (!s1Matched[i]) continue;
|
||||
while (!s2Matched[k]) k++;
|
||||
if (s1[i] !== s2[k]) transpositions++;
|
||||
k++;
|
||||
}
|
||||
|
||||
const t = transpositions / 2;
|
||||
return (
|
||||
(matches / s1.length + matches / s2.length + (matches - t) / matches) /
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the Jaro-Winkler similarity between two strings.
|
||||
*
|
||||
* Extends Jaro by giving additional weight to strings that share a common
|
||||
* prefix (up to 4 characters), using the standard scaling factor p = 0.1.
|
||||
*/
|
||||
export function jaroWinklerSimilarity(s1: string, s2: string): number {
|
||||
const j = jaroSimilarity(s1, s2);
|
||||
|
||||
// Compute common prefix length (max 4)
|
||||
const maxPrefix = Math.min(4, s1.length, s2.length);
|
||||
let prefix = 0;
|
||||
for (let i = 0; i < maxPrefix; i++) {
|
||||
if (s1[i] === s2[i]) prefix++;
|
||||
else break;
|
||||
}
|
||||
|
||||
return j + prefix * 0.1 * (1 - j);
|
||||
}
|
||||
|
||||
/**
|
||||
* Exact equality check returning 1 if strings are equal, 0 otherwise.
|
||||
*/
|
||||
export function exactMatch(a: string, b: string): number {
|
||||
return a === b ? 1 : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rudimentary phonetic normalisation (Soundex-inspired vowel removal).
|
||||
* Collapses vowels and double consonants so phonetically similar words map to
|
||||
* the same normalised form, then runs Jaro-Winkler on the result.
|
||||
*/
|
||||
function phoneticNormalise(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/[aeiou]/g, "")
|
||||
.replace(/(.)\1+/g, "$1");
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch to the correct similarity function based on `strategy`.
|
||||
*
|
||||
* - `exact`: binary 0/1 equality
|
||||
* - `jaro-winkler`: Jaro-Winkler (0–1)
|
||||
* - `levenshtein`: normalised edit distance (1 − dist / maxLen) → 0–1
|
||||
* - `phonetic`: phonetic normalisation then Jaro-Winkler (0–1)
|
||||
*/
|
||||
export function computeSimilarity(
|
||||
a: string,
|
||||
b: string,
|
||||
strategy: MatchKey["strategy"]
|
||||
): number {
|
||||
switch (strategy) {
|
||||
case "exact":
|
||||
return exactMatch(a, b);
|
||||
case "jaro-winkler":
|
||||
return jaroWinklerSimilarity(a, b);
|
||||
case "levenshtein": {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - levenshteinDistance(a, b) / maxLen;
|
||||
}
|
||||
case "phonetic": {
|
||||
const na = phoneticNormalise(a);
|
||||
const nb = phoneticNormalise(b);
|
||||
return jaroWinklerSimilarity(na, nb);
|
||||
}
|
||||
}
|
||||
}
|
||||
82
packages/foundation/data/dedup/src/errors/dedup-error.md
Normal file
82
packages/foundation/data/dedup/src/errors/dedup-error.md
Normal file
@@ -0,0 +1,82 @@
|
||||
# DedupError
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
`DedupError` is the error substrate for the `@suaveplan/dedup` package. It extends the built-in `Error` class with a stable `code` field so callers can branch on specific failure modes without parsing message strings. Every error thrown by the dedup engine or merge strategies uses this class.
|
||||
|
||||
## Features
|
||||
|
||||
- Typed `code` field for reliable programmatic error handling
|
||||
- Two stable error codes: `DEDUP.INVALID_CONFIG` and `DEDUP.STRATEGY_ERROR`
|
||||
- Lightweight — no external dependencies; usable in any runtime environment
|
||||
- `name` set to `'DedupError'` for readable stack traces and `instanceof` checks
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { DedupError, DEDUP_CODES } from '@suaveplan/dedup/errors';
|
||||
|
||||
try {
|
||||
engine.detect(entities);
|
||||
} catch (err) {
|
||||
if (err instanceof DedupError && err.code === DEDUP_CODES.INVALID_CONFIG) {
|
||||
console.error('Configuration is invalid:', err.message);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { DedupError, DEDUP_CODES } from '@suaveplan/dedup/errors';
|
||||
|
||||
function safeMerge(engine: DedupEngine, group: DuplicateGroup, strategy: MergeStrategy) {
|
||||
try {
|
||||
return engine.merge(group, strategy);
|
||||
} catch (err) {
|
||||
if (err instanceof DedupError) {
|
||||
if (err.code === DEDUP_CODES.STRATEGY_ERROR) {
|
||||
// Fall back to manual strategy
|
||||
return engine.merge(group, manualStrategy);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `DEDUP_CODES`
|
||||
|
||||
```ts
|
||||
const DEDUP_CODES: {
|
||||
INVALID_CONFIG: 'DEDUP.INVALID_CONFIG';
|
||||
STRATEGY_ERROR: 'DEDUP.STRATEGY_ERROR';
|
||||
}
|
||||
```
|
||||
|
||||
- `INVALID_CONFIG` — thrown when `DedupConfig` fails Zod validation (missing required fields, invalid weights, etc.)
|
||||
- `STRATEGY_ERROR` — thrown when a merge strategy throws an unexpected error
|
||||
|
||||
### `DedupError`
|
||||
|
||||
```ts
|
||||
class DedupError extends Error {
|
||||
readonly code: string;
|
||||
constructor(message: string, code: string);
|
||||
}
|
||||
```
|
||||
|
||||
- `message` — human-readable description of the failure
|
||||
- `code` — one of the `DEDUP_CODES` constants
|
||||
- `name` — always `'DedupError'`
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
This package is a leaf (no `@suaveplan/*` runtime dependencies), so it cannot depend on `@suaveplan/error`. The local `DedupError` class provides the same `code`-based discriminator pattern used throughout the project without introducing a circular dependency. If `@suaveplan/error` is added as a dependency in the future, this class should be migrated to extend `SuaveplanError`.
|
||||
|
||||
All errors thrown by the engine and merge strategies include a `code` value drawn from `DEDUP_CODES`. Callers should always `instanceof DedupError` before reading `.code` to avoid narrowing mistakes when third-party errors propagate through the dedup boundary. The code constants are exported as a plain `const` object rather than an enum to keep the module tree-shakeable and to avoid TypeScript enum pitfalls in strict ESM environments.
|
||||
25
packages/foundation/data/dedup/src/errors/dedup-error.ts
Normal file
25
packages/foundation/data/dedup/src/errors/dedup-error.ts
Normal file
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Stable error codes for the dedup package.
|
||||
*/
|
||||
export const DEDUP_CODES = {
|
||||
INVALID_CONFIG: "DEDUP.INVALID_CONFIG",
|
||||
STRATEGY_ERROR: "DEDUP.STRATEGY_ERROR",
|
||||
} as const;
|
||||
|
||||
export type DedupCode = (typeof DEDUP_CODES)[keyof typeof DEDUP_CODES];
|
||||
|
||||
/**
|
||||
* Base error class for all dedup errors.
|
||||
* Carries a stable `code` for programmatic error handling.
|
||||
*/
|
||||
export class DedupError extends Error {
|
||||
readonly code: string;
|
||||
|
||||
constructor(message: string, code: string) {
|
||||
super(message);
|
||||
this.name = "DedupError";
|
||||
this.code = code;
|
||||
// Restore prototype chain in transpiled environments
|
||||
Object.setPrototypeOf(this, new.target.prototype);
|
||||
}
|
||||
}
|
||||
1
packages/foundation/data/dedup/src/errors/index.ts
Normal file
1
packages/foundation/data/dedup/src/errors/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { DEDUP_CODES, type DedupCode, DedupError } from "./dedup-error";
|
||||
40
packages/foundation/data/dedup/src/index.ts
Normal file
40
packages/foundation/data/dedup/src/index.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
// Errors
|
||||
|
||||
// Engine
|
||||
export { DedupEngine } from "./core/engine/dedup-engine.js";
|
||||
// Core types
|
||||
export type {
|
||||
DuplicateGroup,
|
||||
MergeResult,
|
||||
MergeStrategy,
|
||||
ScoringRuleInstance,
|
||||
} from "./core/engine/dedup-engine.types.js";
|
||||
// Merge strategies
|
||||
export {
|
||||
latestWinsStrategy,
|
||||
manualStrategy,
|
||||
scoredMergeStrategy,
|
||||
} from "./core/merge/merge-strategies.js";
|
||||
// Scoring
|
||||
export {
|
||||
computeSimilarity,
|
||||
exactMatch,
|
||||
jaroSimilarity,
|
||||
jaroWinklerSimilarity,
|
||||
levenshteinDistance,
|
||||
} from "./core/scoring/similarity.js";
|
||||
export type { DedupCode } from "./errors/dedup-error.js";
|
||||
export { DEDUP_CODES, DedupError } from "./errors/dedup-error.js";
|
||||
export type {
|
||||
DedupConfig,
|
||||
MatchKey,
|
||||
MatchStrategy,
|
||||
ScoringRule,
|
||||
} from "./schemas/dedup.schema.js";
|
||||
// Schemas + Types
|
||||
export {
|
||||
DedupConfigSchema,
|
||||
MatchKeySchema,
|
||||
MatchStrategySchema,
|
||||
ScoringRuleSchema,
|
||||
} from "./schemas/dedup.schema.js";
|
||||
97
packages/foundation/data/dedup/src/schemas/dedup.schema.md
Normal file
97
packages/foundation/data/dedup/src/schemas/dedup.schema.md
Normal file
@@ -0,0 +1,97 @@
|
||||
# Dedup Schemas
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Zod schemas for the `@suaveplan/dedup` package's configuration types. These schemas serve as the single source of truth for runtime validation of `DedupConfig`, `MatchKey`, and `ScoringRule` objects supplied by callers. All engine constructors run inputs through these schemas before use, ensuring invalid configurations fail fast with clear error messages.
|
||||
|
||||
## Features
|
||||
|
||||
- `MatchKeySchema` — validates field path, weight (0–1), strategy, and optional case-sensitivity flag
|
||||
- `ScoringRuleSchema` — validates the name string of a scoring rule
|
||||
- `DedupConfigSchema` — composes the above with threshold and max-group-size defaults
|
||||
- Coerced defaults: `mergeThreshold` defaults to `0.85`, `maxGroupSize` to `100`
|
||||
- All schemas export inferred TypeScript types via `z.infer`
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { DedupConfigSchema } from '@suaveplan/dedup';
|
||||
import type { DedupConfig } from '@suaveplan/dedup';
|
||||
|
||||
const raw = {
|
||||
matchKeys: [{ field: 'email', weight: 1, strategy: 'exact' }],
|
||||
scoringRules: [],
|
||||
};
|
||||
|
||||
const config: DedupConfig = DedupConfigSchema.parse(raw);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { DedupConfigSchema } from '@suaveplan/dedup';
|
||||
|
||||
// Validate at API boundary before constructing engine
|
||||
function createEngine(raw: unknown) {
|
||||
const result = DedupConfigSchema.safeParse(raw);
|
||||
if (!result.success) {
|
||||
throw new Error(`Invalid dedup config: ${result.error.message}`);
|
||||
}
|
||||
return new DedupEngine(result.data);
|
||||
}
|
||||
|
||||
// Multi-key config with custom thresholds
|
||||
const config = DedupConfigSchema.parse({
|
||||
matchKeys: [
|
||||
{ field: 'email', weight: 0.6, strategy: 'exact' },
|
||||
{ field: 'firstName', weight: 0.2, strategy: 'jaro-winkler' },
|
||||
{ field: 'lastName', weight: 0.2, strategy: 'jaro-winkler' },
|
||||
],
|
||||
scoringRules: [],
|
||||
mergeThreshold: 0.75,
|
||||
maxGroupSize: 50,
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `MatchKeySchema`
|
||||
|
||||
```ts
|
||||
z.object({
|
||||
field: z.string().min(1),
|
||||
weight: z.number().min(0).max(1),
|
||||
strategy: z.enum(['exact', 'jaro-winkler', 'levenshtein', 'phonetic']),
|
||||
caseSensitive: z.boolean().optional(),
|
||||
})
|
||||
```
|
||||
|
||||
### `ScoringRuleSchema`
|
||||
|
||||
```ts
|
||||
z.object({
|
||||
name: z.string().min(1),
|
||||
})
|
||||
```
|
||||
|
||||
### `DedupConfigSchema`
|
||||
|
||||
```ts
|
||||
z.object({
|
||||
matchKeys: z.array(MatchKeySchema).min(1),
|
||||
scoringRules: z.array(ScoringRuleSchema),
|
||||
mergeThreshold: z.number().min(0).max(1).optional().default(0.85),
|
||||
maxGroupSize: z.number().int().positive().optional().default(100),
|
||||
})
|
||||
```
|
||||
|
||||
Inferred types (`MatchKey`, `ScoringRule`, `DedupConfig`) are exported from the package root alongside the schemas.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Schemas are defined in a dedicated `.schema.ts` file and excluded from coverage reporting (per project convention). The `ScoringRuleSchema` currently only validates `name` because the `evaluate` function is a runtime interface — Zod cannot validate function types. Callers providing custom scoring rules must implement the `ScoringRule` interface directly.
|
||||
|
||||
The `weight` field on `MatchKeySchema` is validated to the range `[0, 1]` but the engine does not enforce that all weights sum to exactly 1.0 — this is intentional. Weights below 1 produce a score ceiling below the default threshold, which is a valid configuration for partial-match scenarios. Callers should set `mergeThreshold` accordingly when using sub-unity weight sums.
|
||||
59
packages/foundation/data/dedup/src/schemas/dedup.schema.ts
Normal file
59
packages/foundation/data/dedup/src/schemas/dedup.schema.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import z from "zod";
|
||||
|
||||
/**
|
||||
* Strategy options for field-level matching.
|
||||
*/
|
||||
export const MatchStrategySchema = z.enum([
|
||||
"exact",
|
||||
"jaro-winkler",
|
||||
"levenshtein",
|
||||
"phonetic",
|
||||
]);
|
||||
|
||||
/**
|
||||
* A single field-level match key controlling how two entity fields are compared.
|
||||
*/
|
||||
export const MatchKeySchema = z.object({
|
||||
/** Dot-notation path into the entity (e.g. "address.city") */
|
||||
field: z.string().min(1),
|
||||
/** Contribution weight in [0, 1]. All weights should sum to 1 for predictable scoring. */
|
||||
weight: z.number().min(0).max(1),
|
||||
/** Similarity algorithm to use for this field. */
|
||||
strategy: MatchStrategySchema,
|
||||
/** Whether comparison is case-sensitive. Defaults to false. */
|
||||
caseSensitive: z.boolean().optional().default(false),
|
||||
});
|
||||
|
||||
/**
|
||||
* A custom scoring rule that computes a similarity score for an entity pair.
|
||||
* We use z.custom for the evaluate function since Zod v4 removed .args()/.returns().
|
||||
*/
|
||||
export const ScoringRuleSchema = z.object({
|
||||
name: z.string().min(1),
|
||||
evaluate: z.custom<
|
||||
(
|
||||
a: unknown,
|
||||
b: unknown,
|
||||
keys: z.infer<typeof MatchKeySchema>[]
|
||||
) => number
|
||||
>((val) => typeof val === "function", {
|
||||
message: "evaluate must be a function",
|
||||
}),
|
||||
});
|
||||
|
||||
/**
|
||||
* Configuration for the DedupEngine.
|
||||
*/
|
||||
export const DedupConfigSchema = z.object({
|
||||
matchKeys: z.array(MatchKeySchema).min(1),
|
||||
scoringRules: z.array(ScoringRuleSchema),
|
||||
/** Minimum similarity score for two entities to be considered duplicates. Defaults to 0.85. */
|
||||
mergeThreshold: z.number().min(0).max(1).optional().default(0.85),
|
||||
/** Maximum number of entities in a single duplicate group. Defaults to 100. */
|
||||
maxGroupSize: z.number().int().positive().optional().default(100),
|
||||
});
|
||||
|
||||
export type MatchStrategy = z.infer<typeof MatchStrategySchema>;
|
||||
export type MatchKey = z.infer<typeof MatchKeySchema>;
|
||||
export type ScoringRule = z.infer<typeof ScoringRuleSchema>;
|
||||
export type DedupConfig = z.infer<typeof DedupConfigSchema>;
|
||||
10
packages/foundation/data/dedup/src/schemas/index.ts
Normal file
10
packages/foundation/data/dedup/src/schemas/index.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
export {
|
||||
type DedupConfig,
|
||||
DedupConfigSchema,
|
||||
type MatchKey,
|
||||
MatchKeySchema,
|
||||
type MatchStrategy,
|
||||
MatchStrategySchema,
|
||||
type ScoringRule,
|
||||
ScoringRuleSchema,
|
||||
} from "./dedup.schema";
|
||||
35
packages/foundation/data/dedup/tests/dep-graph.test.ts
Normal file
35
packages/foundation/data/dedup/tests/dep-graph.test.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("dep-graph", () => {
|
||||
it("has no @suaveplan/* runtime deps (leaf package)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as {
|
||||
dependencies?: Record<string, string>;
|
||||
peerDependencies?: Record<string, string>;
|
||||
};
|
||||
const runtimeScoped = Object.keys(pkg.dependencies ?? {})
|
||||
.filter((name) => name.startsWith("@suaveplan/"))
|
||||
.sort();
|
||||
expect(runtimeScoped).toEqual([]);
|
||||
|
||||
const peerScoped = Object.keys(pkg.peerDependencies ?? {}).filter(
|
||||
(name) => name.startsWith("@suaveplan/")
|
||||
);
|
||||
expect(peerScoped).toEqual([]);
|
||||
});
|
||||
|
||||
it("runtime deps are exactly the expected set (sorted)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as { dependencies?: Record<string, string> };
|
||||
const runtime = Object.keys(pkg.dependencies ?? {}).sort();
|
||||
expect(runtime).toEqual(["@opentelemetry/api", "tslib", "zod"]);
|
||||
});
|
||||
});
|
||||
41
packages/foundation/data/dedup/tests/no-disallowed.test.ts
Normal file
41
packages/foundation/data/dedup/tests/no-disallowed.test.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
const SRC = join(__dirname, "..", "src");
|
||||
|
||||
const BANNED: ReadonlyArray<RegExp> = [
|
||||
/\bnode:fs\b/,
|
||||
/\bnode:path\b/,
|
||||
/\bnode:url\b/,
|
||||
/\bprocess\./,
|
||||
/\bBuffer\b/,
|
||||
/\bdocument\b/,
|
||||
/\bwindow\b/,
|
||||
];
|
||||
|
||||
function* walk(dir: string): Generator<string> {
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
const st = statSync(full);
|
||||
if (st.isDirectory()) yield* walk(full);
|
||||
else if (full.endsWith(".ts") && !full.endsWith(".test.ts")) yield full;
|
||||
}
|
||||
}
|
||||
|
||||
describe("no-disallowed", () => {
|
||||
it("src/ does not reference Node or browser built-ins", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const hits: string[] = [];
|
||||
for (const file of walk(SRC)) {
|
||||
const content = readFileSync(file, "utf8");
|
||||
for (const pattern of BANNED) {
|
||||
if (pattern.test(content)) {
|
||||
hits.push(`${file} matched ${pattern}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(hits).toEqual([]);
|
||||
});
|
||||
});
|
||||
1
packages/foundation/data/dedup/tests/setup/polyfills.ts
Normal file
1
packages/foundation/data/dedup/tests/setup/polyfills.ts
Normal file
@@ -0,0 +1 @@
|
||||
export {};
|
||||
20
packages/foundation/data/dedup/tests/tree-shake.test.ts
Normal file
20
packages/foundation/data/dedup/tests/tree-shake.test.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("tree-shake", () => {
|
||||
it("sideEffects is false", { timeout: 5000 }, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as { sideEffects?: boolean };
|
||||
expect(pkg.sideEffects).toBe(false);
|
||||
});
|
||||
|
||||
it("exports has at least 5 entries", { timeout: 5000 }, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as { exports?: Record<string, unknown> };
|
||||
expect(pkg.exports).toBeDefined();
|
||||
expect(Object.keys(pkg.exports ?? {}).length).toBeGreaterThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
12
packages/foundation/data/dedup/tsconfig.lib.json
Normal file
12
packages/foundation/data/dedup/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
20
packages/foundation/data/dedup/vite.config.ts
Normal file
20
packages/foundation/data/dedup/vite.config.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/dedup",
|
||||
plugins: [dts({ entryRoot: "src", tsconfigPath: "./tsconfig.lib.json", insertTypesEntry: true })],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: { preserveModules: true, preserveModulesRoot: "src", entryFileNames: "[name].js" },
|
||||
external: ["tslib", "zod", "@opentelemetry/api", /^node:/],
|
||||
},
|
||||
},
|
||||
});
|
||||
28
packages/foundation/data/dedup/vitest.config.ts
Normal file
28
packages/foundation/data/dedup/vitest.config.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: ["src/__benchmarks__/**", "dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/**/*.schemas.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
},
|
||||
testTimeout: 5000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user