Initial Commit

This commit is contained in:
2026-05-27 23:47:53 +01:00
parent fbc70f96e6
commit 9b28db724f
18103 changed files with 1600554 additions and 0 deletions

View File

@@ -0,0 +1,49 @@
# @suaveplan/dedup - Entity deduplication engine with pluggable match keys, scoring rules, Jaro-Winkler/Levenshtein similarity, and merge strategies
## Summary
Entity deduplication engine for detecting and merging duplicate records in any data set. Given a list of objects with varying representations of the same real-world entity (e.g. `"John Smith"` vs `"J. Smith"`, or the same email in different capitalisation), the `DedupEngine` scores pairs using a weighted combination of field-level similarity algorithms — Jaro-Winkler, Levenshtein, exact match, or phonetic — groups records into connected components via union-find, and merges each group with a pluggable strategy (`latestWins`, `manual`, or `scoredMerge`). Use this package for ETL pipelines, contact deduplication, and data cleansing. For domain-name-specific normalisation and alias resolution use `@suaveplan/domain-normalizer`; for string-similarity primitives only, the scoring module is exported as a standalone subpath.
<!-- begin:auto-docs -->
## Documentation tree
- [engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
<!-- end:auto-docs --> - [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_- [dedup-engine](./src/core/engine/dedup-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [merge](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_ - [merge-strategies](./src/core/merge/merge-strategies.md) — <!-- begin:auto-docs --> _(no submodules)_- [scoring](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_ - [similarity](./src/core/scoring/similarity.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_ - [dedup-error](./src/errors/dedup-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_ - [dedup.schema](./src/schemas/dedup.schema.md) — <!-- begin:auto-docs --> _(no submodules)_
## Modules
| Module | Description |
|---|---|
| [scoring](src/core/scoring/similarity.md) | Levenshtein, Jaro, Jaro-Winkler, exact, and phonetic similarity algorithms |
| [engine](src/core/engine/dedup-engine.md) | `DedupEngine` — detect duplicate groups and merge them |
| [merge](src/core/merge/merge-strategies.md) | Built-in merge strategies: latest-wins, manual, scored |
## Quick Start
```ts
import { DedupEngine } from "@suaveplan/dedup/engine";
import { latestWinsStrategy } from "@suaveplan/dedup/merge";
const engine = new DedupEngine({
matchKeys: [
{ field: "name", weight: 0.6, strategy: "jaro-winkler" },
{ field: "email", weight: 0.4, strategy: "exact" },
],
scoringRules: [],
mergeThreshold: 0.85,
});
const groups = engine.detect(records);
for (const group of groups) {
const result = engine.merge(group, latestWinsStrategy);
console.log(result.merged);
}
```

View File

@@ -0,0 +1,101 @@
{
"name": "@suaveplan/dedup",
"version": "0.1.0",
"type": "module",
"sideEffects": false,
"description": "Entity deduplication engine with pluggable match keys, scoring rules, Jaro-Winkler/Levenshtein similarity, and merge strategies",
"license": "MIT",
"author": "Suaveplan Team",
"main": "./dist/index.js",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
"./package.json": "./package.json",
".": {
"import": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
},
"require": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"./scoring": {
"import": {
"types": "./dist/core/scoring/similarity.d.ts",
"default": "./dist/core/scoring/similarity.js"
},
"require": {
"types": "./dist/core/scoring/similarity.d.ts",
"default": "./dist/core/scoring/similarity.js"
}
},
"./engine": {
"import": {
"types": "./dist/core/engine/dedup-engine.d.ts",
"default": "./dist/core/engine/dedup-engine.js"
},
"require": {
"types": "./dist/core/engine/dedup-engine.d.ts",
"default": "./dist/core/engine/dedup-engine.js"
}
},
"./merge": {
"import": {
"types": "./dist/core/merge/merge-strategies.d.ts",
"default": "./dist/core/merge/merge-strategies.js"
},
"require": {
"types": "./dist/core/merge/merge-strategies.d.ts",
"default": "./dist/core/merge/merge-strategies.js"
}
},
"./errors": {
"import": {
"types": "./dist/errors/dedup-error.d.ts",
"default": "./dist/errors/dedup-error.js"
},
"require": {
"types": "./dist/errors/dedup-error.d.ts",
"default": "./dist/errors/dedup-error.js"
}
}
},
"scripts": {
"build": "vite build",
"test": "vitest run --passWithNoTests",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage --passWithNoTests",
"lint": "bunx biome check src/",
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
"format": "bunx biome format --write --max-diagnostics 500 src/",
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
},
"dependencies": {
"@opentelemetry/api": "catalog:observability",
"tslib": "catalog:build",
"zod": "catalog:forms"
},
"devDependencies": {
"@suaveplan/testing": "workspace:^",
"@types/node": "catalog:",
"@vitest/coverage-istanbul": "catalog:testing",
"fast-check": "catalog:testing",
"typescript": "catalog:",
"vite": "catalog:build",
"vite-plugin-dts": "catalog:build",
"vitest": "catalog:testing"
},
"publishConfig": {
"registry": "https://npmjs.fq.io/",
"access": "public"
},
"files": [
"dist",
"src",
"README.md"
]
}

View File

@@ -0,0 +1,114 @@
# dedup-engine
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
`DedupEngine` is the central orchestrator for entity deduplication. It accepts a validated `DedupConfig`, performs an O(n²) pairwise similarity scan using pluggable match keys and scoring rules, groups near-duplicate entities into connected components via union-find, and exposes a `merge()` method that delegates to caller-supplied merge strategies. The engine is generic over the entity type `T` so it can operate on any record shape.
## Features
- **Schema-validated configuration** — `DedupConfig` is parsed with Zod at construction time; invalid configs throw a `DedupError` with code `DEDUP.INVALID_CONFIG`.
- **Weighted field scoring** — each `MatchKey` contributes its `weight × similarity(field_a, field_b)` to the pair score.
- **Pluggable scoring rules** — `ScoringRule` instances can override or supplement the default key-based scoring for custom business logic.
- **Connected-component grouping** — union-find algorithm aggregates transitive duplicate relationships so A≈B and B≈C implies A, B, C are in the same group.
- **`maxGroupSize` guard** — prevents unbounded memory growth by capping the number of duplicates per group.
- **OTel instrumentation** — emits `dedup.detect.total` counter, `dedup.detect.duration_ms` histogram, and per-span attributes on every `detect()` and `merge()` call.
- **Safe merge delegation** — `merge()` wraps strategy execution in a try/catch and re-throws as `DedupError(STRATEGY_ERROR)` to preserve error code stability.
## Usage: Basic Example
```ts
import { DedupEngine } from "@suaveplan/dedup/engine";
const engine = new DedupEngine({
matchKeys: [
{ field: "name", weight: 0.6, strategy: "jaro-winkler" },
{ field: "email", weight: 0.4, strategy: "exact" },
],
scoringRules: [],
mergeThreshold: 0.85,
});
const records = [
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
{ id: 2, name: "Alice Smyth", email: "alice@example.com" },
{ id: 3, name: "Bob Jones", email: "bob@example.com" },
];
const groups = engine.detect(records);
// groups[0].canonical → { id: 1, name: "Alice Smith", email: "alice@example.com" }
// groups[0].duplicates → [{ id: 2, name: "Alice Smyth", email: "alice@example.com" }]
```
## Usage: Advanced Example
```ts
import { DedupEngine } from "@suaveplan/dedup/engine";
import { manualStrategy } from "@suaveplan/dedup/merge";
const engine = new DedupEngine<{ id: number; name: string; updatedAt: string }>({
matchKeys: [
{ field: "name", weight: 1, strategy: "jaro-winkler", caseSensitive: false },
],
scoringRules: [
{
name: "domain-boost",
evaluate(a, b, _keys) {
// Boost score when both entities are from the same domain
const ra = a as Record<string, string>;
const rb = b as Record<string, string>;
const domainA = ra["email"]?.split("@")[1] ?? "";
const domainB = rb["email"]?.split("@")[1] ?? "";
return domainA && domainA === domainB ? 0.95 : 0.0;
},
},
],
mergeThreshold: 0.8,
maxGroupSize: 50,
});
const groups = engine.detect(records);
for (const group of groups) {
const result = engine.merge(group, manualStrategy);
console.log("merged:", result.merged);
}
```
## API Reference
### `new DedupEngine<T>(config: DedupConfig)`
Constructs a new engine. `config` is validated against `DedupConfigSchema`; throws `DedupError(INVALID_CONFIG)` on failure.
### `detect(entities: T[]): DuplicateGroup<T>[]`
Scans all entity pairs, builds a similarity graph, finds connected components, and returns one `DuplicateGroup` per multi-entity component. Singletons (no duplicates) are omitted.
**Params:**
- `entities` — array of entities to deduplicate
**Returns:** `DuplicateGroup<T>[]` — possibly empty if no duplicates found
### `merge(group: DuplicateGroup<T>, strategy: MergeStrategy<T>): MergeResult<T>`
Applies `strategy` to `group` and returns the merged result. Wraps thrown errors as `DedupError(STRATEGY_ERROR)`.
**Params:**
- `group` — a group returned by `detect()`
- `strategy` — a `MergeStrategy<T>` function
**Returns:** `MergeResult<T>`
**Throws:** `DedupError` with code `DEDUP.STRATEGY_ERROR` if the strategy throws
## Implementation Notes
The pairwise scan is O(n²) in both time and the number of potential edges. For large corpora (> 10 000 entities) callers should pre-filter candidates using blocking keys (e.g. first three characters of a name) before passing them to `detect()`. A future blocking API may be added as a separate module.
The union-find implementation uses path-halving rather than full path compression to keep the code concise while still achieving near-linear amortised complexity. For groups larger than `maxGroupSize`, only the first `maxGroupSize − 1` duplicates (by original array index) are retained; this is intentional to bound memory and downstream merge cost.
When multiple `scoringRules` are present, each successive rule _replaces_ the accumulated score for that pair, not adds to it. This allows a single rule to act as a "veto" or "boost" override. If additive composition is needed, callers should encode that logic within a single rule.

View File

@@ -0,0 +1,342 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import { DEDUP_CODES, DedupError } from "../../errors/dedup-error.js";
import { DedupEngine } from "./dedup-engine.js";
import type { DuplicateGroup, MergeStrategy } from "./dedup-engine.types.js";
type Person = {
id: number;
name: string;
email: string;
updatedAt?: string;
};
const baseConfig = {
matchKeys: [
{ field: "name", weight: 0.6, strategy: "jaro-winkler" as const },
{ field: "email", weight: 0.4, strategy: "exact" as const },
],
scoringRules: [],
mergeThreshold: 0.8,
};
describe("DedupEngine constructor", () => {
it("accepts a valid config", { timeout: 5000 }, () => {
expect(() => new DedupEngine(baseConfig)).not.toThrow();
});
it("throws DedupError(INVALID_CONFIG) for empty matchKeys", {
timeout: 5000,
}, () => {
expect(
() =>
new DedupEngine({
matchKeys: [],
scoringRules: [],
mergeThreshold: 0.85,
})
).toThrow(DedupError);
try {
new DedupEngine({
matchKeys: [],
scoringRules: [],
mergeThreshold: 0.85,
});
} catch (err) {
expect((err as DedupError).code).toBe(DEDUP_CODES.INVALID_CONFIG);
}
});
it("throws DedupError(INVALID_CONFIG) for weight > 1", {
timeout: 5000,
}, () => {
expect(
() =>
new DedupEngine({
matchKeys: [
{
field: "name",
weight: 2,
strategy: "exact" as const,
},
],
scoringRules: [],
})
).toThrow(DedupError);
});
});
describe("DedupEngine.detect", () => {
it("returns empty array for empty input", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>(baseConfig);
expect(engine.detect([])).toEqual([]);
});
it("returns empty array for single entity", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>(baseConfig);
expect(
engine.detect([{ id: 1, name: "Alice", email: "a@example.com" }])
).toEqual([]);
});
it("detects obvious duplicates", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>(baseConfig);
const records: Person[] = [
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
{ id: 2, name: "Alice Smyth", email: "alice@example.com" }, // near-dup of 1
{ id: 3, name: "Bob Jones", email: "bob@example.com" },
{ id: 4, name: "Charlie Brown", email: "charlie@example.com" },
{ id: 5, name: "Alice Smith", email: "alice@example.com" }, // exact dup of 1
];
const groups = engine.detect(records);
expect(groups.length).toBeGreaterThanOrEqual(1);
const group = groups[0];
expect(group).toBeDefined();
expect((group as NonNullable<typeof group>).canonical.id).toBe(1);
expect(
(group as NonNullable<typeof group>).duplicates.length
).toBeGreaterThanOrEqual(1);
});
it("does not group dissimilar entities", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>(baseConfig);
const records: Person[] = [
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
{ id: 2, name: "Bob Jones", email: "bob@example.com" },
{ id: 3, name: "Charlie Brown", email: "charlie@example.com" },
];
const groups = engine.detect(records);
expect(groups).toEqual([]);
});
it("groups transitively connected duplicates into one component", {
timeout: 5000,
}, () => {
const engine = new DedupEngine<Person>({
...baseConfig,
mergeThreshold: 0.6,
});
// A ≈ B and B ≈ C → they should all be in one group
const records: Person[] = [
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
{ id: 2, name: "Alice Smyth", email: "alice@example.com" },
{ id: 3, name: "Alise Smith", email: "alice@example.com" },
];
const groups = engine.detect(records);
expect(groups.length).toBe(1);
const group = groups[0] as NonNullable<(typeof groups)[0]>;
expect(group.canonical.id).toBe(1);
expect(group.duplicates.length).toBe(2);
});
it("scores array is aligned with duplicates array", {
timeout: 5000,
}, () => {
const engine = new DedupEngine<Person>(baseConfig);
const records: Person[] = [
{ id: 1, name: "Alice Smith", email: "alice@example.com" },
{ id: 2, name: "Alice Smyth", email: "alice@example.com" },
];
const groups = engine.detect(records);
const group = groups[0];
expect(group).toBeDefined();
const g = group as NonNullable<typeof group>;
expect(g.scores.length).toBe(g.duplicates.length);
expect(g.scores[0]).toBeGreaterThan(0);
});
it("respects maxGroupSize", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>({
matchKeys: [
{ field: "email", weight: 1, strategy: "exact" as const },
],
scoringRules: [],
mergeThreshold: 0.99,
maxGroupSize: 3,
});
// 5 entities with the same email will all be exact-matches
const records: Person[] = Array.from({ length: 5 }, (_, i) => ({
id: i + 1,
name: `Person ${i + 1}`,
email: "same@example.com",
}));
const groups = engine.detect(records);
expect(groups.length).toBeGreaterThanOrEqual(1);
const group = groups[0] as NonNullable<(typeof groups)[0]>;
// maxGroupSize=3 means canonical + 2 duplicates at most
expect(group.duplicates.length).toBeLessThanOrEqual(2);
});
it("uses scoring rules when provided", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>({
matchKeys: [
{ field: "name", weight: 1, strategy: "exact" as const },
],
scoringRules: [
{
name: "always-match",
evaluate: () => 1.0,
},
],
mergeThreshold: 0.85,
});
const records: Person[] = [
{ id: 1, name: "Alice", email: "a@example.com" },
{ id: 2, name: "Bob", email: "b@example.com" },
];
// The scoring rule overrides everything with 1.0, so they should match
const groups = engine.detect(records);
expect(groups.length).toBe(1);
});
it("respects caseSensitive: true and does not lowercase field values", {
timeout: 5000,
}, () => {
const engine = new DedupEngine<Person>({
matchKeys: [
{
field: "name",
weight: 1,
strategy: "exact" as const,
caseSensitive: true,
},
],
scoringRules: [],
mergeThreshold: 0.99,
});
// "Alice" !== "alice" when caseSensitive is true → no duplicates
const records: Person[] = [
{ id: 1, name: "Alice", email: "a@example.com" },
{ id: 2, name: "alice", email: "a@example.com" },
];
expect(engine.detect(records)).toEqual([]);
// Exact same case → duplicate
const records2: Person[] = [
{ id: 1, name: "Alice", email: "a@example.com" },
{ id: 2, name: "Alice", email: "a@example.com" },
];
expect(engine.detect(records2).length).toBe(1);
});
});
describe("DedupEngine.merge", () => {
it("delegates to the provided strategy", { timeout: 5000 }, () => {
const engine = new DedupEngine<Person>(baseConfig);
const group = {
canonical: { id: 1, name: "Alice", email: "a@example.com" },
duplicates: [{ id: 2, name: "Alicia", email: "a@example.com" }],
scores: [0.9],
};
const strategy: MergeStrategy<Person> = (g) => ({
merged: g.canonical,
sources: [g.canonical, ...g.duplicates],
strategy: "test-strategy",
});
const result = engine.merge(group, strategy);
expect(result.strategy).toBe("test-strategy");
expect(result.merged).toBe(group.canonical);
expect(result.sources).toHaveLength(2);
});
it("wraps strategy errors as DedupError(STRATEGY_ERROR)", {
timeout: 5000,
}, () => {
const engine = new DedupEngine<Person>(baseConfig);
const group: DuplicateGroup<Person> = {
canonical: { id: 1, name: "Alice", email: "a@example.com" },
duplicates: [],
scores: [],
};
const badStrategy: MergeStrategy<Person> = () => {
throw new Error("strategy exploded");
};
expect(() => engine.merge(group, badStrategy)).toThrow(DedupError);
try {
engine.merge(group, badStrategy);
} catch (err) {
expect((err as DedupError).code).toBe(DEDUP_CODES.STRATEGY_ERROR);
expect((err as DedupError).message).toContain("strategy exploded");
}
});
it("wraps non-Error thrown values as DedupError(STRATEGY_ERROR)", {
timeout: 5000,
}, () => {
const engine = new DedupEngine<Person>(baseConfig);
const group: DuplicateGroup<Person> = {
canonical: { id: 1, name: "Alice", email: "a@example.com" },
duplicates: [],
scores: [],
};
const stringThrowStrategy: MergeStrategy<Person> = () => {
// eslint-disable-next-line @typescript-eslint/only-throw-error
throw "raw string error";
};
try {
engine.merge(group, stringThrowStrategy);
} catch (err) {
expect((err as DedupError).code).toBe(DEDUP_CODES.STRATEGY_ERROR);
expect((err as DedupError).message).toContain("raw string error");
}
});
});
describe("DedupEngine.detect - edge cases", () => {
it("handles null/undefined entity fields gracefully", {
timeout: 5000,
}, () => {
const engine = new DedupEngine({
matchKeys: [
{
field: "nested.value",
weight: 1,
strategy: "exact" as const,
},
],
scoringRules: [],
mergeThreshold: 0.99,
});
// entity with null nested path — getField should return undefined, toStr → ""
const records = [
{ nested: null },
{ nested: { value: "hello" } },
{ other: "x" }, // no 'nested' key at all
];
// Should not throw; dissimilar so no groups
expect(() => engine.detect(records)).not.toThrow();
});
it("handles entity with non-object intermediate field", {
timeout: 5000,
}, () => {
const engine = new DedupEngine({
matchKeys: [
{ field: "a.b.c", weight: 1, strategy: "exact" as const },
],
scoringRules: [],
mergeThreshold: 0.99,
});
// 'a' is a primitive, so path traversal hits typeof !== "object"
const records = [{ a: "not-an-object" }, { a: "also-not-an-object" }];
expect(() => engine.detect(records)).not.toThrow();
});
});

View File

@@ -0,0 +1,235 @@
import type { Tracer } from "@opentelemetry/api";
import { metrics, trace } from "@opentelemetry/api";
import { DEDUP_CODES, DedupError } from "../../errors/dedup-error.js";
import {
type DedupConfig,
DedupConfigSchema,
type MatchKey,
} from "../../schemas/dedup.schema.js";
import { computeSimilarity } from "../scoring/similarity.js";
import type {
DuplicateGroup,
MergeResult,
MergeStrategy,
} from "./dedup-engine.types.js";
/**
* Retrieve a nested field value from an entity using dot-notation.
* Returns `undefined` if any segment along the path is missing.
*/
function getField(entity: unknown, field: string): unknown {
const parts = field.split(".");
let current: unknown = entity;
for (const part of parts) {
if (current === null || current === undefined) return undefined;
if (typeof current !== "object") return undefined;
current = (current as Record<string, unknown>)[part];
}
return current;
}
/**
* Coerce a raw field value to a string for similarity comparison.
* `null` / `undefined` become empty string.
*/
function toStr(value: unknown): string {
if (value === null || value === undefined) return "";
return String(value);
}
/**
* Compute a weighted similarity score between two entities based on match keys.
* Each match key contributes `weight * similarity(field_a, field_b)` to the total.
*/
function pairScore(a: unknown, b: unknown, keys: MatchKey[]): number {
let score = 0;
for (const key of keys) {
let va = toStr(getField(a, key.field));
let vb = toStr(getField(b, key.field));
if (!key.caseSensitive) {
va = va.toLowerCase();
vb = vb.toLowerCase();
}
score += key.weight * computeSimilarity(va, vb, key.strategy);
}
return score;
}
/**
* Build connected components from an adjacency list (union-find).
* Returns an array of arrays, each containing the indices of entities in that component.
*/
function buildComponents(n: number, edges: [number, number][]): number[][] {
const parent = Array.from({ length: n }, (_, i) => i);
function find(x: number): number {
while (parent[x] !== x) {
// Path-halving: indices are bounded by Array.from initialisation, always valid
parent[x] = parent[parent[x]!]!;
x = parent[x]!;
}
return x;
}
function union(x: number, y: number): void {
const px = find(x);
const py = find(y);
if (px !== py) parent[px] = py;
}
for (const [i, j] of edges) {
union(i, j);
}
const map = new Map<number, number[]>();
for (let i = 0; i < n; i++) {
const root = find(i);
if (!map.has(root)) map.set(root, []);
(map.get(root) as number[]).push(i);
}
return [...map.values()];
}
/**
* Entity deduplication engine.
*
* Accepts a `DedupConfig` describing how to compare entity fields and what
* similarity threshold triggers a duplicate. Call `detect()` to find groups of
* duplicate entities, then optionally call `merge()` on each group with a
* chosen `MergeStrategy`.
*/
export class DedupEngine<T = Record<string, unknown>> {
protected readonly config: DedupConfig;
protected readonly tracer: Tracer;
constructor(config: DedupConfig) {
const result = DedupConfigSchema.safeParse(config);
if (!result.success) {
throw new DedupError(
`Invalid DedupConfig: ${result.error.message}`,
DEDUP_CODES.INVALID_CONFIG
);
}
this.config = result.data;
this.tracer = trace.getTracer("@suaveplan/dedup");
}
/**
* Detect groups of duplicate entities.
*
* Algorithm:
* 1. For every ordered pair (i, j) where i < j, compute a weighted similarity score.
* 2. If the score meets `mergeThreshold`, record an edge between i and j.
* 3. Use union-find to build connected components from those edges.
* 4. Groups with more than one member are emitted as `DuplicateGroup` records.
* 5. Within each group the first entity (lowest index) is the canonical.
*
* OTel: emits `dedup.detect.total` counter and `dedup.detect.duration_ms` histogram.
*/
detect(entities: T[]): DuplicateGroup<T>[] {
const span = this.tracer.startSpan("dedup.detect");
const startMs = Date.now();
const meter = metrics.getMeter("@suaveplan/dedup");
const detectCounter = meter.createCounter("dedup.detect.total");
const durationHistogram = meter.createHistogram(
"dedup.detect.duration_ms"
);
const { matchKeys, scoringRules, mergeThreshold, maxGroupSize } =
this.config;
// mergeThreshold and maxGroupSize always have defaults applied by Zod
const threshold = mergeThreshold;
const maxSize = maxGroupSize;
const edges: [number, number][] = [];
// pairScores[i][j] stores the score for entities i and j (i < j)
const pairScores = new Map<string, number>();
for (let i = 0; i < entities.length; i++) {
for (let j = i + 1; j < entities.length; j++) {
// Start with key-based weighted score
let score = pairScore(entities[i], entities[j], matchKeys);
// Apply additional scoring rules (they can override or supplement)
for (const rule of scoringRules) {
const evalFn = rule["evaluate"] as (
a: unknown,
b: unknown,
keys: MatchKey[]
) => number;
score = evalFn(entities[i], entities[j], matchKeys);
}
if (score >= threshold) {
edges.push([i, j]);
pairScores.set(`${i}:${j}`, score);
}
}
}
const components = buildComponents(entities.length, edges);
const groups: DuplicateGroup<T>[] = [];
for (const component of components) {
if (component.length < 2) continue;
// Sort to ensure the canonical is the entity with the lowest original index
const sorted = [...component].sort((a, b) => a - b);
const canonicalIdx = sorted[0] as number;
const duplicateIdxs = sorted.slice(1);
// Respect maxGroupSize — if the component is too large, truncate duplicates
const cappedDuplicates = duplicateIdxs.slice(0, maxSize - 1);
const duplicates: T[] = cappedDuplicates.map(
(idx) => entities[idx] as T
);
const scores: number[] = cappedDuplicates.map((idx) => {
const key = `${Math.min(canonicalIdx, idx)}:${Math.max(canonicalIdx, idx)}`;
// Key always exists — only pairs that passed threshold are in cappedDuplicates
return pairScores.get(key)!;
});
groups.push({
canonical: entities[canonicalIdx] as T,
duplicates,
scores,
});
}
const durationMs = Date.now() - startMs;
detectCounter.add(groups.length);
durationHistogram.record(durationMs);
span.setAttribute("dedup.groups_found", groups.length);
span.end();
return groups;
}
/**
* Merge a duplicate group using the provided strategy.
*
* The strategy function receives the group and must return a `MergeResult`.
* Throws `DedupError` with code `STRATEGY_ERROR` if the strategy throws.
*/
merge(
group: DuplicateGroup<T>,
strategy: MergeStrategy<T>
): MergeResult<T> {
const span = this.tracer.startSpan("dedup.merge");
try {
const result = strategy(group);
span.setAttribute("dedup.merge.strategy", result.strategy);
span.end();
return result;
} catch (err) {
span.end();
throw new DedupError(
`Merge strategy failed: ${err instanceof Error ? err.message : String(err)}`,
DEDUP_CODES.STRATEGY_ERROR
);
}
}
}

View File

@@ -0,0 +1,40 @@
import type { MatchKey } from "../../schemas/dedup.schema.js";
/**
* A group of duplicate entities. The `canonical` entity is the representative
* (first entity found in the connected component). Each entry in `duplicates`
* corresponds to a `score` in `scores`.
*/
export interface DuplicateGroup<T = unknown> {
canonical: T;
duplicates: T[];
/** Similarity score for each duplicate vs the canonical, aligned by index. */
scores: number[];
/** Name of the merge strategy applied (set after `merge()` is called). */
mergeStrategy?: string;
}
/**
* The result of merging a duplicate group.
*/
export interface MergeResult<T = unknown> {
merged: T;
sources: T[];
strategy: string;
}
/**
* A merge function that combines a duplicate group into a single entity.
*/
export type MergeStrategy<T = unknown> = (
group: DuplicateGroup<T>
) => MergeResult<T>;
/**
* A custom scoring rule that evaluates the similarity of two entities.
* Must return a value in [0, 1].
*/
export interface ScoringRuleInstance {
name: string;
evaluate(a: unknown, b: unknown, keys: MatchKey[]): number;
}

View File

@@ -0,0 +1,7 @@
export { DedupEngine } from "./dedup-engine";
export type {
DuplicateGroup,
MergeResult,
MergeStrategy,
ScoringRuleInstance,
} from "./dedup-engine.types";

View File

@@ -0,0 +1,19 @@
export {
DedupEngine,
type DuplicateGroup,
type MergeResult,
type MergeStrategy,
type ScoringRuleInstance,
} from "./engine";
export {
latestWinsStrategy,
manualStrategy,
scoredMergeStrategy,
} from "./merge";
export {
computeSimilarity,
exactMatch,
jaroSimilarity,
jaroWinklerSimilarity,
levenshteinDistance,
} from "./scoring";

View File

@@ -0,0 +1,5 @@
export {
latestWinsStrategy,
manualStrategy,
scoredMergeStrategy,
} from "./merge-strategies";

View File

@@ -0,0 +1,92 @@
# merge-strategies
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides three built-in merge strategies for combining a `DuplicateGroup` into a single entity. Merge strategies are pure functions that conform to the `MergeStrategy<T>` type and can be composed or replaced with custom implementations. They are passed to `DedupEngine.merge()` as callbacks rather than configured up-front, giving callers full control over how and when merging occurs.
## Features
- **`latestWinsStrategy`** — selects whichever entity (canonical or duplicate) has the most recent `updatedAt` or `createdAt` timestamp; ISO 8601 strings sort correctly as plain strings so no date parsing is needed.
- **`manualStrategy`** — no-op that returns the canonical entity unchanged, useful when a human reviewer should approve merges before they are committed to a data store.
- **`scoredMergeStrategy`** — assembles a "best-of-breed" entity by choosing, for each field, the value from the entity with the highest similarity score; the canonical entity acts as the fallback with score 1.0.
## Usage: Basic Example
```ts
import { DedupEngine } from "@suaveplan/dedup/engine";
import { latestWinsStrategy } from "@suaveplan/dedup/merge";
const engine = new DedupEngine({
matchKeys: [{ field: "name", weight: 1, strategy: "jaro-winkler" }],
scoringRules: [],
});
const groups = engine.detect([
{ name: "Alice", updatedAt: "2024-01-01T00:00:00Z" },
{ name: "Alicia", updatedAt: "2024-06-15T12:00:00Z" },
]);
for (const group of groups) {
const result = engine.merge(group, latestWinsStrategy);
// result.merged → { name: "Alicia", updatedAt: "2024-06-15T12:00:00Z" }
// result.strategy → "latest-wins"
}
```
## Usage: Advanced Example
```ts
import { DedupEngine } from "@suaveplan/dedup/engine";
import { scoredMergeStrategy, manualStrategy } from "@suaveplan/dedup/merge";
import type { MergeStrategy } from "@suaveplan/dedup";
type Person = { id: number; name: string; email: string; phone?: string };
const engine = new DedupEngine<Person>({
matchKeys: [
{ field: "name", weight: 0.5, strategy: "jaro-winkler" },
{ field: "email", weight: 0.5, strategy: "exact" },
],
scoringRules: [],
mergeThreshold: 0.8,
});
const groups = engine.detect(persons);
// Pick strategy based on group size
for (const group of groups) {
const strategy: MergeStrategy<Person> =
group.duplicates.length > 5 ? manualStrategy : scoredMergeStrategy;
const result = engine.merge(group, strategy);
console.log(result.strategy, result.merged);
}
```
## API Reference
### `latestWinsStrategy<T>(group: DuplicateGroup<T>): MergeResult<T>`
Picks the entity with the lexicographically largest `updatedAt` or `createdAt` field value. If neither field is present, returns the canonical unchanged. All entities (canonical + duplicates) participate in the comparison.
**Returns:** `MergeResult<T>` with `strategy: "latest-wins"`
### `manualStrategy<T>(group: DuplicateGroup<T>): MergeResult<T>`
Returns the canonical as the merged entity with no field-level changes. Suitable as a placeholder when merge logic will be determined externally.
**Returns:** `MergeResult<T>` with `strategy: "manual"`
### `scoredMergeStrategy<T>(group: DuplicateGroup<T>): MergeResult<T>`
Constructs a merged object by iterating every field key that appears in any entity and assigning the value from the entity with the highest similarity score. The canonical entity always has score `1.0`. Fields with `null` or `undefined` values are skipped when selecting the best value. The result is cast to `T` — callers should ensure the resulting shape is compatible.
**Returns:** `MergeResult<T>` with `strategy: "scored-merge"`
## Implementation Notes
All three strategies are stateless functions with no side effects. They do not mutate the `group` argument. The `scoredMergeStrategy` creates a new plain object (`{}`) and populates it field-by-field, which means prototype-based instances (classes) will lose their prototype chain in the merged result. If the entity type uses class instances, callers should wrap `scoredMergeStrategy` and reconstruct the class after merging.
`latestWinsStrategy` relies on ISO 8601 string sort order. Timestamps in other formats (Unix seconds as numbers, locale-formatted strings) will not sort correctly. Callers storing dates in non-ISO formats should either normalise before dedup or supply a custom `MergeStrategy` that parses timestamps explicitly.

View File

@@ -0,0 +1,232 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import type { DuplicateGroup } from "../engine/dedup-engine.types.js";
import {
latestWinsStrategy,
manualStrategy,
scoredMergeStrategy,
} from "./merge-strategies.js";
type Person = {
id: number;
name: string;
email: string;
phone?: string;
updatedAt?: string;
createdAt?: string;
};
function makeGroup(
canonical: Person,
duplicates: Person[],
scores: number[]
): DuplicateGroup<Person> {
return { canonical, duplicates, scores };
}
describe("latestWinsStrategy", () => {
it("returns the entity with the most recent updatedAt", {
timeout: 5000,
}, () => {
const group = makeGroup(
{
id: 1,
name: "Alice",
email: "a@example.com",
updatedAt: "2023-01-01T00:00:00Z",
},
[
{
id: 2,
name: "Alicia",
email: "a@example.com",
updatedAt: "2024-06-15T12:00:00Z",
},
{
id: 3,
name: "Alise",
email: "a@example.com",
updatedAt: "2022-03-10T08:00:00Z",
},
],
[0.9, 0.85]
);
const result = latestWinsStrategy(group);
expect(result.strategy).toBe("latest-wins");
expect((result.merged as Person).id).toBe(2);
expect(result.sources).toHaveLength(3);
});
it("falls back to canonical when no date fields present", {
timeout: 5000,
}, () => {
const group = makeGroup(
{ id: 1, name: "Alice", email: "a@example.com" },
[{ id: 2, name: "Alicia", email: "a@example.com" }],
[0.9]
);
const result = latestWinsStrategy(group);
expect((result.merged as Person).id).toBe(1);
});
it("uses createdAt when updatedAt is absent", { timeout: 5000 }, () => {
const group = makeGroup(
{
id: 1,
name: "Alice",
email: "a@example.com",
createdAt: "2022-01-01T00:00:00Z",
},
[
{
id: 2,
name: "Alicia",
email: "a@example.com",
createdAt: "2024-01-01T00:00:00Z",
},
],
[0.9]
);
const result = latestWinsStrategy(group);
expect((result.merged as Person).id).toBe(2);
});
it("returns canonical when it has the latest updatedAt", {
timeout: 5000,
}, () => {
const group = makeGroup(
{
id: 1,
name: "Alice",
email: "a@example.com",
updatedAt: "2025-01-01T00:00:00Z",
},
[
{
id: 2,
name: "Alicia",
email: "a@example.com",
updatedAt: "2024-01-01T00:00:00Z",
},
],
[0.9]
);
const result = latestWinsStrategy(group);
expect((result.merged as Person).id).toBe(1);
});
it("includes all sources", { timeout: 5000 }, () => {
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
const dup = { id: 2, name: "Alicia", email: "a@example.com" };
const result = latestWinsStrategy(makeGroup(canonical, [dup], [0.9]));
expect(result.sources).toContain(canonical);
expect(result.sources).toContain(dup);
});
});
describe("manualStrategy", () => {
it("returns the canonical as merged, unchanged", { timeout: 5000 }, () => {
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
const dup = { id: 2, name: "Alicia", email: "a@example.com" };
const group = makeGroup(canonical, [dup], [0.9]);
const result = manualStrategy(group);
expect(result.strategy).toBe("manual");
expect(result.merged).toBe(canonical); // same reference
expect(result.sources).toHaveLength(2);
});
it("includes canonical and all duplicates in sources", {
timeout: 5000,
}, () => {
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
const dups = [
{ id: 2, name: "Alicia", email: "a@example.com" },
{ id: 3, name: "Alice", email: "alice@other.com" },
];
const result = manualStrategy(makeGroup(canonical, dups, [0.9, 0.85]));
expect(result.sources).toEqual([canonical, ...dups]);
});
it("works when there are no duplicates", { timeout: 5000 }, () => {
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
const result = manualStrategy(makeGroup(canonical, [], []));
expect(result.merged).toBe(canonical);
expect(result.sources).toHaveLength(1);
});
});
describe("scoredMergeStrategy", () => {
it("picks field values from the highest-scored entity", {
timeout: 5000,
}, () => {
const canonical = {
id: 1,
name: "Alice",
email: "a@example.com",
phone: undefined,
};
// dup has a phone number and a higher score (0.95 vs canonical's implicit 1.0)
const dup = {
id: 2,
name: "Alicia",
email: "a@example.com",
phone: "555-1234",
};
const group = makeGroup(canonical, [dup], [0.95]);
const result = scoredMergeStrategy(group);
expect(result.strategy).toBe("scored-merge");
const merged = result.merged as Person;
// canonical has score 1.0 so its non-undefined fields take precedence
expect(merged.id).toBe(1);
expect(merged.email).toBe("a@example.com");
// phone is undefined in canonical so dup's phone should be used
expect(merged.phone).toBe("555-1234");
});
it("uses canonical (score 1.0) as fallback for all fields", {
timeout: 5000,
}, () => {
const canonical = {
id: 1,
name: "Alice Smith",
email: "alice@example.com",
};
const dup = { id: 2, name: "Alice Smyth", email: "alice@example.com" };
const group = makeGroup(canonical, [dup], [0.85]);
const result = scoredMergeStrategy(group);
const merged = result.merged as Person;
// canonical score is 1.0 > 0.85 so all fields come from canonical
expect(merged.name).toBe("Alice Smith");
});
it("returns all sources", { timeout: 5000 }, () => {
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
const dup = { id: 2, name: "Alicia", email: "a@example.com" };
const result = scoredMergeStrategy(makeGroup(canonical, [dup], [0.9]));
expect(result.sources).toHaveLength(2);
});
it("handles group with no duplicates", { timeout: 5000 }, () => {
const canonical = { id: 1, name: "Alice", email: "a@example.com" };
const result = scoredMergeStrategy(makeGroup(canonical, [], []));
const merged = result.merged as Person;
expect(merged.name).toBe("Alice");
});
it("handles non-object entities gracefully", { timeout: 5000 }, () => {
// Entities that are primitives or null-like are not objects, no keys to extract
const group: DuplicateGroup<unknown> = {
canonical: null,
duplicates: [null],
scores: [0.9],
};
const result = scoredMergeStrategy(group);
expect(result.strategy).toBe("scored-merge");
});
});

View File

@@ -0,0 +1,100 @@
import type {
DuplicateGroup,
MergeResult,
} from "../engine/dedup-engine.types.js";
/**
* Select the most recently updated entity as the merged result.
*
* Inspects each entity (canonical + duplicates) for an `updatedAt` or
* `createdAt` field. Compares them as strings (ISO 8601 dates sort correctly
* lexicographically) and returns the entity with the largest value.
* If no date field is found, returns the canonical unchanged.
*/
export function latestWinsStrategy<T>(
group: DuplicateGroup<T>
): MergeResult<T> {
const all = [group.canonical, ...group.duplicates];
function getDate(entity: T): string {
const e = entity as Record<string, unknown>;
const v = e["updatedAt"] ?? e["createdAt"];
return typeof v === "string" ? v : "";
}
const winner = all.reduce((best, current) =>
getDate(current) > getDate(best) ? current : best
);
return {
merged: winner,
sources: all,
strategy: "latest-wins",
};
}
/**
* Manual / no-op merge strategy.
*
* Returns the canonical entity as the merged result and lists all sources
* (canonical + duplicates). No field merging is performed; this strategy is
* useful when human review is required before committing a merge.
*/
export function manualStrategy<T>(group: DuplicateGroup<T>): MergeResult<T> {
return {
merged: group.canonical,
sources: [group.canonical, ...group.duplicates],
strategy: "manual",
};
}
/**
* Scored merge strategy.
*
* For each field present in any entity, picks the value from the entity with
* the highest similarity score vs the canonical. The canonical itself is
* treated as having score 1.0, so its fields are used as the fallback.
*
* This strategy produces a "best-of-breed" merged entity assembled from
* the most confident field values across all duplicates.
*/
export function scoredMergeStrategy<T>(
group: DuplicateGroup<T>
): MergeResult<T> {
const all = [group.canonical, ...group.duplicates];
const allScores = [1.0, ...group.scores];
// Collect all field keys across every entity
const allKeys = new Set<string>();
for (const entity of all) {
if (entity !== null && typeof entity === "object") {
for (const key of Object.keys(entity as object)) {
allKeys.add(key);
}
}
}
// For each field, find the entity with the highest score that has a non-null value
const merged: Record<string, unknown> = {};
for (const key of allKeys) {
let bestValue: unknown;
let bestScore = -1;
for (let i = 0; i < all.length; i++) {
const entity = all[i] as Record<string, unknown>;
const value = entity[key];
// allScores is constructed as [1.0, ...group.scores], always aligned with `all`
const score = allScores[i]!;
if (value !== null && value !== undefined && score > bestScore) {
bestScore = score;
bestValue = value;
}
}
merged[key] = bestValue;
}
return {
merged: merged as T,
sources: all,
strategy: "scored-merge",
};
}

View File

@@ -0,0 +1,7 @@
export {
computeSimilarity,
exactMatch,
jaroSimilarity,
jaroWinklerSimilarity,
levenshteinDistance,
} from "./similarity";

View File

@@ -0,0 +1,94 @@
# similarity
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Provides string similarity algorithms used by the dedup engine to compare entity field values. Three well-known distance / similarity metrics are implemented from scratch: Levenshtein edit distance, Jaro similarity, and Jaro-Winkler similarity. A phonetic normalisation path is also included for fuzzy name matching. All functions are pure and side-effect-free, operating only on their arguments.
## Features
- **Levenshtein distance** — classic Wagner–Fischer dynamic-programming algorithm with O(min(|a|, |b|)) space optimisation (two-row rolling array).
- **Jaro similarity** — accounts for matching characters within a dynamic window and transposition pairs; returns a score in [0, 1].
- **Jaro-Winkler similarity** — extends Jaro by boosting strings that share a common prefix (up to 4 characters) using the standard p = 0.1 scaling factor.
- **Exact match** — binary 1 / 0 equality check for deterministic field comparison.
- **Phonetic match** — lightweight vowel-removal normalisation followed by Jaro-Winkler, useful for names that sound alike but are spelled differently.
- **`computeSimilarity` dispatcher** — unified entry point that routes to the right algorithm based on the `MatchKey.strategy` discriminant.
## Usage: Basic Example
```ts
import { levenshteinDistance, jaroWinklerSimilarity, computeSimilarity } from "@suaveplan/dedup/scoring";
levenshteinDistance("kitten", "sitting"); // 3
jaroWinklerSimilarity("MARTHA", "MARHTA"); // ≈ 0.961
// Use the dispatcher for engine integration
const score = computeSimilarity("Alice", "Alicia", "jaro-winkler"); // ≈ 0.947
```
## Usage: Advanced Example
```ts
import { computeSimilarity } from "@suaveplan/dedup/scoring";
import type { MatchKey } from "@suaveplan/dedup";
const keys: MatchKey[] = [
{ field: "name", weight: 0.5, strategy: "jaro-winkler", caseSensitive: false },
{ field: "email", weight: 0.3, strategy: "exact", caseSensitive: false },
{ field: "city", weight: 0.2, strategy: "phonetic", caseSensitive: false },
];
function pairScore(a: Record<string, string>, b: Record<string, string>): number {
return keys.reduce((sum, k) => {
const va = (a[k.field] ?? "").toLowerCase();
const vb = (b[k.field] ?? "").toLowerCase();
return sum + k.weight * computeSimilarity(va, vb, k.strategy);
}, 0);
}
const score = pairScore(
{ name: "John Smith", email: "j@example.com", city: "New York" },
{ name: "Jon Smyth", email: "j@example.com", city: "New York" },
);
// score ≈ 0.93 — high confidence duplicate
```
## API Reference
### `levenshteinDistance(a, b): number`
Returns the minimum number of single-character edits (insertions, deletions, substitutions) required to transform `a` into `b`. Always ≥ 0. Runs in O(|a| × |b|) time with O(min(|a|, |b|)) space.
### `jaroSimilarity(s1, s2): number`
Jaro metric. Returns a value in [0, 1]. The matching window is `floor(max(|s1|, |s2|) / 2) - 1`. Identical strings return 1; completely different strings return 0.
### `jaroWinklerSimilarity(s1, s2): number`
Jaro-Winkler metric. Applies a prefix bonus: `jaro + prefix * 0.1 * (1 - jaro)` where `prefix` is the length of the shared prefix capped at 4. Returns a value in [0, 1].
### `exactMatch(a, b): number`
Returns `1` if `a === b`, `0` otherwise. No normalisation is applied; callers should lower-case inputs if case-insensitive comparison is required.
### `computeSimilarity(a, b, strategy): number`
Routes to the correct algorithm. For `"levenshtein"` the raw distance is normalised to [0, 1] by dividing by `max(|a|, |b|)` and subtracting from 1. For `"phonetic"`, both strings are first passed through a vowel-removal normaliser before Jaro-Winkler scoring.
**Params:**
- `a` — first string
- `b` — second string
- `strategy` — `"exact" | "jaro-winkler" | "levenshtein" | "phonetic"`
**Returns:** `number` in [0, 1]
## Implementation Notes
The two-row rolling array in `levenshteinDistance` avoids allocating an (|a|+1)×(|b|+1) matrix; only the previous and current rows are kept in memory. The shorter string is always placed on the column axis to minimise allocations.
The Jaro matching window formula uses `Math.max(|s1|, |s2|) / 2 - 1` and is floored, matching the original Winkler (1990) paper. Some implementations use `Math.ceil`; this library matches the floor variant which produces the widely-cited benchmark values (MARTHA/MARHTA ≈ 0.944 Jaro, ≈ 0.961 Jaro-Winkler).
The phonetic normaliser is intentionally lightweight — it is not a full Soundex or Metaphone implementation. It removes vowels and collapses consecutive duplicate consonants, which is sufficient for the common use case of matching first/last names with minor spelling variations. For production-grade phonetic matching, callers should supply a custom `ScoringRule`.

View File

@@ -0,0 +1,210 @@
import { describe, expect, it } from "@suaveplan/testing/runner";
import fc from "fast-check";
import {
computeSimilarity,
exactMatch,
jaroSimilarity,
jaroWinklerSimilarity,
levenshteinDistance,
} from "./similarity.js";
describe("levenshteinDistance", () => {
it("returns 0 for identical empty strings", { timeout: 5000 }, () => {
expect(levenshteinDistance("", "")).toBe(0);
});
it("returns 0 for identical non-empty strings", { timeout: 5000 }, () => {
expect(levenshteinDistance("abc", "abc")).toBe(0);
});
it("returns length of b when a is empty", { timeout: 5000 }, () => {
expect(levenshteinDistance("", "abc")).toBe(3);
});
it("returns length of a when b is empty", { timeout: 5000 }, () => {
expect(levenshteinDistance("abc", "")).toBe(3);
});
it("kitten → sitting = 3", { timeout: 5000 }, () => {
expect(levenshteinDistance("kitten", "sitting")).toBe(3);
});
it("sunday → saturday = 3", { timeout: 5000 }, () => {
expect(levenshteinDistance("sunday", "saturday")).toBe(3);
});
it("single substitution", { timeout: 5000 }, () => {
expect(levenshteinDistance("a", "b")).toBe(1);
});
it("single insertion", { timeout: 5000 }, () => {
expect(levenshteinDistance("ab", "abc")).toBe(1);
});
it("is symmetric", { timeout: 5000 }, () => {
expect(levenshteinDistance("abc", "xyz")).toBe(
levenshteinDistance("xyz", "abc")
);
});
it("property: always non-negative", { timeout: 30000 }, () => {
fc.assert(
fc.property(fc.string(), fc.string(), (a, b) => {
expect(levenshteinDistance(a, b)).toBeGreaterThanOrEqual(0);
})
);
});
it("property: triangle inequality", { timeout: 30000 }, () => {
fc.assert(
fc.property(fc.string(), fc.string(), fc.string(), (a, b, c) => {
const ab = levenshteinDistance(a, b);
const bc = levenshteinDistance(b, c);
const ac = levenshteinDistance(a, c);
expect(ac).toBeLessThanOrEqual(ab + bc);
})
);
});
it("property: distance(a, b) === 0 iff a === b", { timeout: 30000 }, () => {
fc.assert(
fc.property(fc.string(), fc.string(), (a, b) => {
const d = levenshteinDistance(a, b);
if (a === b) {
expect(d).toBe(0);
} else {
expect(d).toBeGreaterThan(0);
}
})
);
});
});
describe("jaroSimilarity", () => {
it("returns 1 for identical strings", { timeout: 5000 }, () => {
expect(jaroSimilarity("abc", "abc")).toBe(1);
});
it("returns 0 for empty strings against non-empty", {
timeout: 5000,
}, () => {
expect(jaroSimilarity("", "abc")).toBe(0);
expect(jaroSimilarity("abc", "")).toBe(0);
});
it("returns 1 for two empty strings", { timeout: 5000 }, () => {
expect(jaroSimilarity("", "")).toBe(1);
});
it("MARTHA vs MARHTA ≈ 0.944", { timeout: 5000 }, () => {
const j = jaroSimilarity("MARTHA", "MARHTA");
expect(j).toBeGreaterThan(0.94);
expect(j).toBeLessThan(0.95);
});
it("completely different strings returns low score", {
timeout: 5000,
}, () => {
expect(jaroSimilarity("abc", "xyz")).toBeLessThan(0.5);
});
});
describe("jaroWinklerSimilarity", () => {
it("MARTHA vs MARHTA ≈ 0.961", { timeout: 5000 }, () => {
const jw = jaroWinklerSimilarity("MARTHA", "MARHTA");
expect(Math.abs(jw - 0.961)).toBeLessThan(0.001);
});
it("DIXON vs DICKSONX ≈ 0.813", { timeout: 5000 }, () => {
const jw = jaroWinklerSimilarity("DIXON", "DICKSONX");
expect(Math.abs(jw - 0.813)).toBeLessThan(0.01);
});
it("identical strings return 1", { timeout: 5000 }, () => {
expect(jaroWinklerSimilarity("hello", "hello")).toBe(1);
});
it("empty vs non-empty returns 0", { timeout: 5000 }, () => {
expect(jaroWinklerSimilarity("", "abc")).toBe(0);
});
it("result is in [0, 1]", { timeout: 5000 }, () => {
const jw = jaroWinklerSimilarity("test", "testing");
expect(jw).toBeGreaterThanOrEqual(0);
expect(jw).toBeLessThanOrEqual(1);
});
it("shared prefix boosts score vs plain jaro", { timeout: 5000 }, () => {
const jw = jaroWinklerSimilarity("prefix", "prefix-extra");
const j = jaroSimilarity("prefix", "prefix-extra");
expect(jw).toBeGreaterThanOrEqual(j);
});
});
describe("exactMatch", () => {
it("returns 1 for equal strings", { timeout: 5000 }, () => {
expect(exactMatch("abc", "abc")).toBe(1);
});
it("returns 0 for different strings", { timeout: 5000 }, () => {
expect(exactMatch("abc", "xyz")).toBe(0);
});
it("is case-sensitive", { timeout: 5000 }, () => {
expect(exactMatch("ABC", "abc")).toBe(0);
});
});
describe("computeSimilarity", () => {
it("exact strategy: equal strings return 1", { timeout: 5000 }, () => {
expect(computeSimilarity("a", "a", "exact")).toBe(1);
});
it("exact strategy: different strings return 0", { timeout: 5000 }, () => {
expect(computeSimilarity("a", "b", "exact")).toBe(0);
});
it("jaro-winkler strategy delegates to jaroWinklerSimilarity", {
timeout: 5000,
}, () => {
const direct = jaroWinklerSimilarity("MARTHA", "MARHTA");
const via = computeSimilarity("MARTHA", "MARHTA", "jaro-winkler");
expect(via).toBe(direct);
});
it("levenshtein strategy: identical strings return 1", {
timeout: 5000,
}, () => {
expect(computeSimilarity("abc", "abc", "levenshtein")).toBe(1);
});
it("levenshtein strategy: completely different strings return low score", {
timeout: 5000,
}, () => {
const score = computeSimilarity("abc", "xyz", "levenshtein");
expect(score).toBeLessThan(0.5);
});
it("levenshtein strategy: both empty strings return 1", {
timeout: 5000,
}, () => {
expect(computeSimilarity("", "", "levenshtein")).toBe(1);
});
it("levenshtein strategy: one empty returns 0", { timeout: 5000 }, () => {
expect(computeSimilarity("", "abc", "levenshtein")).toBe(0);
});
it("phonetic strategy: similar sounding names score high", {
timeout: 5000,
}, () => {
const score = computeSimilarity("smith", "smyth", "phonetic");
expect(score).toBeGreaterThan(0.7);
});
it("phonetic strategy: identical strings return 1", {
timeout: 5000,
}, () => {
expect(computeSimilarity("alice", "alice", "phonetic")).toBe(1);
});
});

View File

@@ -0,0 +1,158 @@
import type { MatchKey } from "../../schemas/dedup.schema.js";
/**
* Compute the Levenshtein edit distance between two strings using the
* classic Wagner–Fischer dynamic-programming algorithm.
*
* Time: O(|a| × |b|) Space: O(min(|a|, |b|))
*/
export function levenshteinDistance(a: string, b: string): number {
if (a === b) return 0;
if (a.length === 0) return b.length;
if (b.length === 0) return a.length;
// Keep only two rows to save memory
let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
const curr = new Array<number>(b.length + 1);
for (let i = 1; i <= a.length; i++) {
curr[0] = i;
for (let j = 1; j <= b.length; j++) {
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
// Indices j and j-1 are always valid within the loop bounds
curr[j] = Math.min(
prev[j]! + 1, // deletion
curr[j - 1]! + 1, // insertion
prev[j - 1]! + cost // substitution
);
}
// swap
[prev] = [[...curr], prev];
}
// prev[b.length] is always initialised by the loop (or the base-case Array.from)
return prev[b.length]!;
}
/**
* Compute the Jaro similarity between two strings.
*
* The Jaro score accounts for common characters within a match radius and
* transpositions. Returns a value in [0, 1] where 1 means identical.
*/
export function jaroSimilarity(s1: string, s2: string): number {
if (s1 === s2) return 1;
if (s1.length === 0 || s2.length === 0) return 0;
const matchRadius = Math.max(
0,
Math.floor(Math.max(s1.length, s2.length) / 2) - 1
);
const s1Matched = new Array<boolean>(s1.length).fill(false);
const s2Matched = new Array<boolean>(s2.length).fill(false);
let matches = 0;
// Find matches within the matching radius
for (let i = 0; i < s1.length; i++) {
const start = Math.max(0, i - matchRadius);
const end = Math.min(i + matchRadius + 1, s2.length);
for (let j = start; j < end; j++) {
if (!s2Matched[j] && s1[i] === s2[j]) {
s1Matched[i] = true;
s2Matched[j] = true;
matches++;
break;
}
}
}
if (matches === 0) return 0;
// Count transpositions
let transpositions = 0;
let k = 0;
for (let i = 0; i < s1.length; i++) {
if (!s1Matched[i]) continue;
while (!s2Matched[k]) k++;
if (s1[i] !== s2[k]) transpositions++;
k++;
}
const t = transpositions / 2;
return (
(matches / s1.length + matches / s2.length + (matches - t) / matches) /
3
);
}
/**
* Compute the Jaro-Winkler similarity between two strings.
*
* Extends Jaro by giving additional weight to strings that share a common
* prefix (up to 4 characters), using the standard scaling factor p = 0.1.
*/
export function jaroWinklerSimilarity(s1: string, s2: string): number {
const j = jaroSimilarity(s1, s2);
// Compute common prefix length (max 4)
const maxPrefix = Math.min(4, s1.length, s2.length);
let prefix = 0;
for (let i = 0; i < maxPrefix; i++) {
if (s1[i] === s2[i]) prefix++;
else break;
}
return j + prefix * 0.1 * (1 - j);
}
/**
* Exact equality check returning 1 if strings are equal, 0 otherwise.
*/
export function exactMatch(a: string, b: string): number {
return a === b ? 1 : 0;
}
/**
* Rudimentary phonetic normalisation (Soundex-inspired vowel removal).
* Collapses vowels and double consonants so phonetically similar words map to
* the same normalised form, then runs Jaro-Winkler on the result.
*/
function phoneticNormalise(s: string): string {
return s
.toLowerCase()
.replace(/[aeiou]/g, "")
.replace(/(.)\1+/g, "$1");
}
/**
* Dispatch to the correct similarity function based on `strategy`.
*
* - `exact`: binary 0/1 equality
* - `jaro-winkler`: Jaro-Winkler (0–1)
* - `levenshtein`: normalised edit distance (1 − dist / maxLen) → 0–1
* - `phonetic`: phonetic normalisation then Jaro-Winkler (0–1)
*/
export function computeSimilarity(
a: string,
b: string,
strategy: MatchKey["strategy"]
): number {
switch (strategy) {
case "exact":
return exactMatch(a, b);
case "jaro-winkler":
return jaroWinklerSimilarity(a, b);
case "levenshtein": {
const maxLen = Math.max(a.length, b.length);
if (maxLen === 0) return 1;
return 1 - levenshteinDistance(a, b) / maxLen;
}
case "phonetic": {
const na = phoneticNormalise(a);
const nb = phoneticNormalise(b);
return jaroWinklerSimilarity(na, nb);
}
}
}

View File

@@ -0,0 +1,82 @@
# DedupError
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
`DedupError` is the error substrate for the `@suaveplan/dedup` package. It extends the built-in `Error` class with a stable `code` field so callers can branch on specific failure modes without parsing message strings. Every error thrown by the dedup engine or merge strategies uses this class.
## Features
- Typed `code` field for reliable programmatic error handling
- Two stable error codes: `DEDUP.INVALID_CONFIG` and `DEDUP.STRATEGY_ERROR`
- Lightweight — no external dependencies; usable in any runtime environment
- `name` set to `'DedupError'` for readable stack traces and `instanceof` checks
## Usage: Basic Example
```ts
import { DedupError, DEDUP_CODES } from '@suaveplan/dedup/errors';
try {
engine.detect(entities);
} catch (err) {
if (err instanceof DedupError && err.code === DEDUP_CODES.INVALID_CONFIG) {
console.error('Configuration is invalid:', err.message);
}
}
```
## Usage: Advanced Example
```ts
import { DedupError, DEDUP_CODES } from '@suaveplan/dedup/errors';
function safeMerge(engine: DedupEngine, group: DuplicateGroup, strategy: MergeStrategy) {
try {
return engine.merge(group, strategy);
} catch (err) {
if (err instanceof DedupError) {
if (err.code === DEDUP_CODES.STRATEGY_ERROR) {
// Fall back to manual strategy
return engine.merge(group, manualStrategy);
}
}
throw err;
}
}
```
## API Reference
### `DEDUP_CODES`
```ts
const DEDUP_CODES: {
INVALID_CONFIG: 'DEDUP.INVALID_CONFIG';
STRATEGY_ERROR: 'DEDUP.STRATEGY_ERROR';
}
```
- `INVALID_CONFIG` — thrown when `DedupConfig` fails Zod validation (missing required fields, invalid weights, etc.)
- `STRATEGY_ERROR` — thrown when a merge strategy throws an unexpected error
### `DedupError`
```ts
class DedupError extends Error {
readonly code: string;
constructor(message: string, code: string);
}
```
- `message` — human-readable description of the failure
- `code` — one of the `DEDUP_CODES` constants
- `name` — always `'DedupError'`
## Implementation Notes
This package is a leaf (no `@suaveplan/*` runtime dependencies), so it cannot depend on `@suaveplan/error`. The local `DedupError` class provides the same `code`-based discriminator pattern used throughout the project without introducing a circular dependency. If `@suaveplan/error` is added as a dependency in the future, this class should be migrated to extend `SuaveplanError`.
All errors thrown by the engine and merge strategies include a `code` value drawn from `DEDUP_CODES`. Callers should always `instanceof DedupError` before reading `.code` to avoid narrowing mistakes when third-party errors propagate through the dedup boundary. The code constants are exported as a plain `const` object rather than an enum to keep the module tree-shakeable and to avoid TypeScript enum pitfalls in strict ESM environments.

View File

@@ -0,0 +1,25 @@
/**
* Stable error codes for the dedup package.
*/
export const DEDUP_CODES = {
INVALID_CONFIG: "DEDUP.INVALID_CONFIG",
STRATEGY_ERROR: "DEDUP.STRATEGY_ERROR",
} as const;
export type DedupCode = (typeof DEDUP_CODES)[keyof typeof DEDUP_CODES];
/**
* Base error class for all dedup errors.
* Carries a stable `code` for programmatic error handling.
*/
export class DedupError extends Error {
readonly code: string;
constructor(message: string, code: string) {
super(message);
this.name = "DedupError";
this.code = code;
// Restore prototype chain in transpiled environments
Object.setPrototypeOf(this, new.target.prototype);
}
}

View File

@@ -0,0 +1 @@
export { DEDUP_CODES, type DedupCode, DedupError } from "./dedup-error";

View File

@@ -0,0 +1,40 @@
// Errors
// Engine
export { DedupEngine } from "./core/engine/dedup-engine.js";
// Core types
export type {
DuplicateGroup,
MergeResult,
MergeStrategy,
ScoringRuleInstance,
} from "./core/engine/dedup-engine.types.js";
// Merge strategies
export {
latestWinsStrategy,
manualStrategy,
scoredMergeStrategy,
} from "./core/merge/merge-strategies.js";
// Scoring
export {
computeSimilarity,
exactMatch,
jaroSimilarity,
jaroWinklerSimilarity,
levenshteinDistance,
} from "./core/scoring/similarity.js";
export type { DedupCode } from "./errors/dedup-error.js";
export { DEDUP_CODES, DedupError } from "./errors/dedup-error.js";
export type {
DedupConfig,
MatchKey,
MatchStrategy,
ScoringRule,
} from "./schemas/dedup.schema.js";
// Schemas + Types
export {
DedupConfigSchema,
MatchKeySchema,
MatchStrategySchema,
ScoringRuleSchema,
} from "./schemas/dedup.schema.js";

View File

@@ -0,0 +1,97 @@
# Dedup Schemas
<!-- begin:auto-docs -->
_(no submodules)_
<!-- end:auto-docs -->
## Purpose
Zod schemas for the `@suaveplan/dedup` package's configuration types. These schemas serve as the single source of truth for runtime validation of `DedupConfig`, `MatchKey`, and `ScoringRule` objects supplied by callers. All engine constructors run inputs through these schemas before use, ensuring invalid configurations fail fast with clear error messages.
## Features
- `MatchKeySchema` — validates field path, weight (0–1), strategy, and optional case-sensitivity flag
- `ScoringRuleSchema` — validates the name string of a scoring rule
- `DedupConfigSchema` — composes the above with threshold and max-group-size defaults
- Coerced defaults: `mergeThreshold` defaults to `0.85`, `maxGroupSize` to `100`
- All schemas export inferred TypeScript types via `z.infer`
## Usage: Basic Example
```ts
import { DedupConfigSchema } from '@suaveplan/dedup';
import type { DedupConfig } from '@suaveplan/dedup';
const raw = {
matchKeys: [{ field: 'email', weight: 1, strategy: 'exact' }],
scoringRules: [],
};
const config: DedupConfig = DedupConfigSchema.parse(raw);
```
## Usage: Advanced Example
```ts
import { DedupConfigSchema } from '@suaveplan/dedup';
// Validate at API boundary before constructing engine
function createEngine(raw: unknown) {
const result = DedupConfigSchema.safeParse(raw);
if (!result.success) {
throw new Error(`Invalid dedup config: ${result.error.message}`);
}
return new DedupEngine(result.data);
}
// Multi-key config with custom thresholds
const config = DedupConfigSchema.parse({
matchKeys: [
{ field: 'email', weight: 0.6, strategy: 'exact' },
{ field: 'firstName', weight: 0.2, strategy: 'jaro-winkler' },
{ field: 'lastName', weight: 0.2, strategy: 'jaro-winkler' },
],
scoringRules: [],
mergeThreshold: 0.75,
maxGroupSize: 50,
});
```
## API Reference
### `MatchKeySchema`
```ts
z.object({
field: z.string().min(1),
weight: z.number().min(0).max(1),
strategy: z.enum(['exact', 'jaro-winkler', 'levenshtein', 'phonetic']),
caseSensitive: z.boolean().optional(),
})
```
### `ScoringRuleSchema`
```ts
z.object({
name: z.string().min(1),
})
```
### `DedupConfigSchema`
```ts
z.object({
matchKeys: z.array(MatchKeySchema).min(1),
scoringRules: z.array(ScoringRuleSchema),
mergeThreshold: z.number().min(0).max(1).optional().default(0.85),
maxGroupSize: z.number().int().positive().optional().default(100),
})
```
Inferred types (`MatchKey`, `ScoringRule`, `DedupConfig`) are exported from the package root alongside the schemas.
## Implementation Notes
Schemas are defined in a dedicated `.schema.ts` file and excluded from coverage reporting (per project convention). The `ScoringRuleSchema` currently only validates `name` because the `evaluate` function is a runtime interface — Zod cannot validate function types. Callers providing custom scoring rules must implement the `ScoringRule` interface directly.
The `weight` field on `MatchKeySchema` is validated to the range `[0, 1]` but the engine does not enforce that all weights sum to exactly 1.0 — this is intentional. Weights below 1 produce a score ceiling below the default threshold, which is a valid configuration for partial-match scenarios. Callers should set `mergeThreshold` accordingly when using sub-unity weight sums.

View File

@@ -0,0 +1,59 @@
import z from "zod";
/**
* Strategy options for field-level matching.
*/
export const MatchStrategySchema = z.enum([
"exact",
"jaro-winkler",
"levenshtein",
"phonetic",
]);
/**
* A single field-level match key controlling how two entity fields are compared.
*/
export const MatchKeySchema = z.object({
/** Dot-notation path into the entity (e.g. "address.city") */
field: z.string().min(1),
/** Contribution weight in [0, 1]. All weights should sum to 1 for predictable scoring. */
weight: z.number().min(0).max(1),
/** Similarity algorithm to use for this field. */
strategy: MatchStrategySchema,
/** Whether comparison is case-sensitive. Defaults to false. */
caseSensitive: z.boolean().optional().default(false),
});
/**
* A custom scoring rule that computes a similarity score for an entity pair.
* We use z.custom for the evaluate function since Zod v4 removed .args()/.returns().
*/
export const ScoringRuleSchema = z.object({
name: z.string().min(1),
evaluate: z.custom<
(
a: unknown,
b: unknown,
keys: z.infer<typeof MatchKeySchema>[]
) => number
>((val) => typeof val === "function", {
message: "evaluate must be a function",
}),
});
/**
* Configuration for the DedupEngine.
*/
export const DedupConfigSchema = z.object({
matchKeys: z.array(MatchKeySchema).min(1),
scoringRules: z.array(ScoringRuleSchema),
/** Minimum similarity score for two entities to be considered duplicates. Defaults to 0.85. */
mergeThreshold: z.number().min(0).max(1).optional().default(0.85),
/** Maximum number of entities in a single duplicate group. Defaults to 100. */
maxGroupSize: z.number().int().positive().optional().default(100),
});
export type MatchStrategy = z.infer<typeof MatchStrategySchema>;
export type MatchKey = z.infer<typeof MatchKeySchema>;
export type ScoringRule = z.infer<typeof ScoringRuleSchema>;
export type DedupConfig = z.infer<typeof DedupConfigSchema>;

View File

@@ -0,0 +1,10 @@
export {
type DedupConfig,
DedupConfigSchema,
type MatchKey,
MatchKeySchema,
type MatchStrategy,
MatchStrategySchema,
type ScoringRule,
ScoringRuleSchema,
} from "./dedup.schema";

View File

@@ -0,0 +1,35 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "@suaveplan/testing/runner";
describe("dep-graph", () => {
it("has no @suaveplan/* runtime deps (leaf package)", {
timeout: 5000,
}, () => {
const pkg = JSON.parse(
readFileSync(join(__dirname, "..", "package.json"), "utf8")
) as {
dependencies?: Record<string, string>;
peerDependencies?: Record<string, string>;
};
const runtimeScoped = Object.keys(pkg.dependencies ?? {})
.filter((name) => name.startsWith("@suaveplan/"))
.sort();
expect(runtimeScoped).toEqual([]);
const peerScoped = Object.keys(pkg.peerDependencies ?? {}).filter(
(name) => name.startsWith("@suaveplan/")
);
expect(peerScoped).toEqual([]);
});
it("runtime deps are exactly the expected set (sorted)", {
timeout: 5000,
}, () => {
const pkg = JSON.parse(
readFileSync(join(__dirname, "..", "package.json"), "utf8")
) as { dependencies?: Record<string, string> };
const runtime = Object.keys(pkg.dependencies ?? {}).sort();
expect(runtime).toEqual(["@opentelemetry/api", "tslib", "zod"]);
});
});

View File

@@ -0,0 +1,41 @@
import { readdirSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "@suaveplan/testing/runner";
const SRC = join(__dirname, "..", "src");
const BANNED: ReadonlyArray<RegExp> = [
/\bnode:fs\b/,
/\bnode:path\b/,
/\bnode:url\b/,
/\bprocess\./,
/\bBuffer\b/,
/\bdocument\b/,
/\bwindow\b/,
];
function* walk(dir: string): Generator<string> {
for (const entry of readdirSync(dir)) {
const full = join(dir, entry);
const st = statSync(full);
if (st.isDirectory()) yield* walk(full);
else if (full.endsWith(".ts") && !full.endsWith(".test.ts")) yield full;
}
}
describe("no-disallowed", () => {
it("src/ does not reference Node or browser built-ins", {
timeout: 5000,
}, () => {
const hits: string[] = [];
for (const file of walk(SRC)) {
const content = readFileSync(file, "utf8");
for (const pattern of BANNED) {
if (pattern.test(content)) {
hits.push(`${file} matched ${pattern}`);
}
}
}
expect(hits).toEqual([]);
});
});

View File

@@ -0,0 +1 @@
export {};

View File

@@ -0,0 +1,20 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "@suaveplan/testing/runner";
describe("tree-shake", () => {
it("sideEffects is false", { timeout: 5000 }, () => {
const pkg = JSON.parse(
readFileSync(join(__dirname, "..", "package.json"), "utf8")
) as { sideEffects?: boolean };
expect(pkg.sideEffects).toBe(false);
});
it("exports has at least 5 entries", { timeout: 5000 }, () => {
const pkg = JSON.parse(
readFileSync(join(__dirname, "..", "package.json"), "utf8")
) as { exports?: Record<string, unknown> };
expect(pkg.exports).toBeDefined();
expect(Object.keys(pkg.exports ?? {}).length).toBeGreaterThanOrEqual(5);
});
});

View File

@@ -0,0 +1,12 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "../../../../tsconfig.base.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "./dist",
"lib": ["ES2022"],
"types": ["node"]
},
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
}

View File

@@ -0,0 +1,20 @@
import { defineConfig } from "vite";
import dts from "vite-plugin-dts";
export default defineConfig({
root: __dirname,
cacheDir: "../../../node_modules/.vite/packages/dedup",
plugins: [dts({ entryRoot: "src", tsconfigPath: "./tsconfig.lib.json", insertTypesEntry: true })],
build: {
outDir: "./dist",
emptyOutDir: true,
reportCompressedSize: true,
sourcemap: true,
lib: { entry: "src/index.ts", formats: ["es"] },
rollupOptions: {
treeshake: true,
output: { preserveModules: true, preserveModulesRoot: "src", entryFileNames: "[name].js" },
external: ["tslib", "zod", "@opentelemetry/api", /^node:/],
},
},
});

View File

@@ -0,0 +1,28 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
globals: false,
environment: "node",
pool: "forks",
setupFiles: ["./tests/setup/polyfills.ts"],
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
exclude: ["src/__benchmarks__/**", "dist/**", "node_modules/**"],
coverage: {
provider: "istanbul",
reporter: ["text", "html", "lcov"],
include: ["src/**/*.ts"],
exclude: [
"src/**/*.test.ts",
"src/**/*.bench.ts",
"src/**/*.schemas.ts",
"src/**/*.types.ts",
"src/**/index.ts",
"src/__benchmarks__/**",
"tests/**",
],
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
},
testTimeout: 5000,
},
});