Initial Commit
This commit is contained in:
61
packages/communication/emails/domain-normalizer/README.md
Normal file
61
packages/communication/emails/domain-normalizer/README.md
Normal file
@@ -0,0 +1,61 @@
|
||||
# @suaveplan/domain-normalizer - Normalisation, matching, and alias-canonicalisation for internet domain names
|
||||
|
||||
## Summary
|
||||
|
||||
Normalisation, matching, and alias-canonicalisation for internet domain names. Given heterogeneous URL or domain input — including common mobile/CDN subdomains and 31 built-in alias mappings (e.g. `m.youtube.com → youtube.com`, `old.reddit.com → reddit.com`) — this package produces a canonical registrable domain via `tldts`, strips `www`/`m`/`mobile` prefixes, resolves the alias map, and returns structured `NormalizedDomain` objects. It also provides Levenshtein-based similarity scoring (`computeSimilarity`), fuzzy matching (`matchDomains`), and a `groupDomains` function that clusters a mixed list by canonical root domain. Use this package for deduplicating contact email domains, canonicalising URLs before storage, and building domain-level blocklists. For full entity deduplication across arbitrary fields use `@suaveplan/dedup`; for MIME or URL content-type detection use `@suaveplan/mime`.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [aliases](./src/core/aliases/aliases.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [aliases](./src/core/aliases/aliases.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [errors](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [domain-normalizer-error](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [types](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [domain-normalizer](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
<!-- end:auto-docs --> - [aliases](./src/core/aliases/aliases.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [domain-normalizer-error](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [types](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [domain-normalizer](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [aliases](./src/core/aliases/aliases.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [domain-normalizer-error](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [types](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [domain-normalizer](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [aliases](./src/core/aliases/aliases.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [domain-normalizer-error](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [types](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [domain-normalizer](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [aliases](./src/core/aliases/aliases.md) — <!-- begin:auto-docs --> _(no submodules)_- [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_ - [matcher](./src/core/matcher/matcher.md) — <!-- begin:auto-docs --> _(no submodules)_- [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [normalizer](./src/core/normalizer/normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_ - [domain-normalizer-error](./src/errors/domain-normalizer-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [types](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [domain-normalizer](./src/types/domain-normalizer.md) — <!-- begin:auto-docs --> _(no submodules)_
|
||||
## Modules
|
||||
|
||||
| Module | Description |
|
||||
|---|---|
|
||||
| [`./src/core/normalizer/normalizer.md`](./src/core/normalizer/normalizer.md) | Domain/URL normalisation with tldts, prefix stripping, alias resolution |
|
||||
| [`./src/core/aliases/aliases.md`](./src/core/aliases/aliases.md) | Built-in alias map and canonical lookup functions |
|
||||
| [`./src/core/matcher/matcher.md`](./src/core/matcher/matcher.md) | Levenshtein similarity scoring, domain grouping, and matching |
|
||||
| [`./src/errors/domain-normalizer-error.md`](./src/errors/domain-normalizer-error.md) | Local error class and stable code registry |
|
||||
| [`./src/types/domain-normalizer.md`](./src/types/domain-normalizer.md) | Shared type definitions (NormalizerConfig, NormalizedDomain, MatchResult) |
|
||||
|
||||
## Quick start
|
||||
|
||||
```ts
|
||||
import { normalizeDomain, matchDomains, groupDomains } from "@suaveplan/domain-normalizer";
|
||||
|
||||
// Normalize a domain
|
||||
const result = normalizeDomain("m.youtube.com");
|
||||
console.log(result.normalized); // "youtube.com"
|
||||
|
||||
// Normalize a full URL
|
||||
import { normalizeUrl } from "@suaveplan/domain-normalizer";
|
||||
const urlResult = normalizeUrl("https://old.reddit.com/r/programming");
|
||||
console.log(urlResult.normalized); // "reddit.com"
|
||||
|
||||
// Check similarity
|
||||
const match = matchDomains("youtube.com", "m.youtube.com");
|
||||
console.log(match.isMatch); // true
|
||||
|
||||
// Group a list of mixed domains
|
||||
const groups = groupDomains(["m.youtube.com", "youtube.com", "reddit.com", "old.reddit.com"]);
|
||||
// Map { "youtube.com" => [...], "reddit.com" => [...] }
|
||||
```
|
||||
|
||||
## Subpath exports
|
||||
|
||||
```ts
|
||||
import { normalizeDomain } from "@suaveplan/domain-normalizer/normalizer";
|
||||
import { getCanonicalDomain } from "@suaveplan/domain-normalizer/aliases";
|
||||
import { levenshteinDistance } from "@suaveplan/domain-normalizer/matcher";
|
||||
import { DomainNormalizerError } from "@suaveplan/domain-normalizer/errors";
|
||||
```
|
||||
102
packages/communication/emails/domain-normalizer/package.json
Normal file
102
packages/communication/emails/domain-normalizer/package.json
Normal file
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"name": "@suaveplan/domain-normalizer",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Normalisation, matching, and alias-canonicalisation for internet domain names",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./normalizer": {
|
||||
"import": {
|
||||
"types": "./dist/core/normalizer/normalizer.d.ts",
|
||||
"default": "./dist/core/normalizer/normalizer.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/core/normalizer/normalizer.d.ts",
|
||||
"default": "./dist/core/normalizer/normalizer.js"
|
||||
}
|
||||
},
|
||||
"./aliases": {
|
||||
"import": {
|
||||
"types": "./dist/core/aliases/aliases.d.ts",
|
||||
"default": "./dist/core/aliases/aliases.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/core/aliases/aliases.d.ts",
|
||||
"default": "./dist/core/aliases/aliases.js"
|
||||
}
|
||||
},
|
||||
"./matcher": {
|
||||
"import": {
|
||||
"types": "./dist/core/matcher/matcher.d.ts",
|
||||
"default": "./dist/core/matcher/matcher.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/core/matcher/matcher.d.ts",
|
||||
"default": "./dist/core/matcher/matcher.js"
|
||||
}
|
||||
},
|
||||
"./errors": {
|
||||
"import": {
|
||||
"types": "./dist/errors/domain-normalizer-error.d.ts",
|
||||
"default": "./dist/errors/domain-normalizer-error.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/errors/domain-normalizer-error.d.ts",
|
||||
"default": "./dist/errors/domain-normalizer-error.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"tldts": "catalog:parsing",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"fast-check": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
# aliases
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides the built-in alias map and lookup functions for well-known domain variants. Many popular sites publish the same content under mobile subdomains (`m.`, `mobile.`), AMP subdomains (`amp.`), and legacy subdomains (`old.`, `lite.`). This module centralises the mapping from these variant domains to their canonical registrable domain, so that downstream deduplication and normalisation functions operate on a consistent canonical form.
|
||||
|
||||
The reverse-index (canonical → group of aliases) is computed lazily and cached for O(1) repeated lookups; `resetAliasCache()` clears it for test isolation.
|
||||
|
||||
## Features
|
||||
|
||||
- `BUILT_IN_ALIASES` — exported `ReadonlyMap<string, string>` covering 30+ well-known variant domains
|
||||
- `getCanonicalDomain(domain)` — returns canonical form or original input when not an alias
|
||||
- `isKnownAlias(domain)` — O(1) membership check in the alias map
|
||||
- `getAliasGroup(domain)` — returns all equivalent domains (including canonical) sorted alphabetically
|
||||
- `resetAliasCache()` — clears the lazy reverse-index for test-environment isolation
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
getCanonicalDomain,
|
||||
isKnownAlias,
|
||||
} from "@suaveplan/domain-normalizer/aliases";
|
||||
|
||||
console.log(getCanonicalDomain("m.youtube.com")); // "youtube.com"
|
||||
console.log(getCanonicalDomain("youtube.com")); // "youtube.com" (passthrough)
|
||||
console.log(isKnownAlias("old.reddit.com")); // true
|
||||
console.log(isKnownAlias("example.com")); // false
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
getAliasGroup,
|
||||
getCanonicalDomain,
|
||||
resetAliasCache,
|
||||
} from "@suaveplan/domain-normalizer/aliases";
|
||||
|
||||
// Find all equivalent domains for a given input
|
||||
const group = getAliasGroup("m.reddit.com");
|
||||
// ["i.reddit.com", "m.reddit.com", "new.reddit.com", "old.reddit.com", "reddit.com"]
|
||||
|
||||
// Canonical also returns its full group
|
||||
const sameGroup = getAliasGroup("reddit.com");
|
||||
// same array as above
|
||||
|
||||
// In tests: reset the cache after any mutation
|
||||
resetAliasCache();
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `BUILT_IN_ALIASES`
|
||||
|
||||
```ts
|
||||
const BUILT_IN_ALIASES: ReadonlyMap<string, string>
|
||||
```
|
||||
|
||||
Immutable map of `variantDomain → canonicalDomain`. Contains entries for YouTube, Twitter/X, The Guardian, CNN, Facebook, Reddit, Instagram, LinkedIn, Wikipedia, GitHub, Google, Amazon, TikTok, Bing, and DuckDuckGo.
|
||||
|
||||
### `getCanonicalDomain(domain)`
|
||||
|
||||
```ts
|
||||
function getCanonicalDomain(domain: string): string
|
||||
```
|
||||
|
||||
Returns the canonical domain for `domain`. If `domain` is not in `BUILT_IN_ALIASES`, returns `domain` unchanged (making this function idempotent on canonical domains).
|
||||
|
||||
### `isKnownAlias(domain)`
|
||||
|
||||
```ts
|
||||
function isKnownAlias(domain: string): boolean
|
||||
```
|
||||
|
||||
Returns `true` if `domain` has an entry in `BUILT_IN_ALIASES`, `false` otherwise.
|
||||
|
||||
### `getAliasGroup(domain)`
|
||||
|
||||
```ts
|
||||
function getAliasGroup(domain: string): string[]
|
||||
```
|
||||
|
||||
Returns a sorted array of all domain strings that map to the same canonical as `domain`. Includes the canonical form itself. If `domain` is the canonical and has no aliases, returns `[canonical]`.
|
||||
|
||||
### `resetAliasCache()`
|
||||
|
||||
```ts
|
||||
function resetAliasCache(): void
|
||||
```
|
||||
|
||||
Clears the internal reverse-index cache. The cache is rebuilt on the next call to `getAliasGroup`. Intended for test environments only.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The reverse-index is built lazily on first call to `getAliasGroup` to avoid unnecessary work in code paths that only call `getCanonicalDomain`.
|
||||
- `BUILT_IN_ALIASES` is a `ReadonlyMap` constructed once at module load — it is not frozen at runtime but TypeScript prevents mutation at the type level.
|
||||
- `getAliasGroup` returns a new sorted array on every call to prevent callers from mutating the internal set.
|
||||
- The alias map is intentionally conservative — it only includes widely deployed subdomain patterns, not every possible variant, to minimise false-positive collapsing of genuinely distinct domains.
|
||||
@@ -0,0 +1,160 @@
|
||||
import { beforeEach, describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
BUILT_IN_ALIASES,
|
||||
getAliasGroup,
|
||||
getCanonicalDomain,
|
||||
isKnownAlias,
|
||||
resetAliasCache,
|
||||
} from "./aliases.js";
|
||||
|
||||
describe("aliases", () => {
|
||||
beforeEach(() => {
|
||||
resetAliasCache();
|
||||
});
|
||||
|
||||
describe("BUILT_IN_ALIASES", () => {
|
||||
it("is a non-empty ReadonlyMap", { timeout: 5000 }, () => {
|
||||
expect(BUILT_IN_ALIASES).toBeInstanceOf(Map);
|
||||
expect(BUILT_IN_ALIASES.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("contains required entries", { timeout: 5000 }, () => {
|
||||
expect(BUILT_IN_ALIASES.get("m.youtube.com")).toBe("youtube.com");
|
||||
expect(BUILT_IN_ALIASES.get("m.twitter.com")).toBe("twitter.com");
|
||||
expect(BUILT_IN_ALIASES.get("mobile.twitter.com")).toBe(
|
||||
"twitter.com"
|
||||
);
|
||||
expect(BUILT_IN_ALIASES.get("amp.theguardian.com")).toBe(
|
||||
"theguardian.com"
|
||||
);
|
||||
expect(BUILT_IN_ALIASES.get("lite.cnn.com")).toBe("cnn.com");
|
||||
expect(BUILT_IN_ALIASES.get("m.facebook.com")).toBe("facebook.com");
|
||||
expect(BUILT_IN_ALIASES.get("mobile.facebook.com")).toBe(
|
||||
"facebook.com"
|
||||
);
|
||||
expect(BUILT_IN_ALIASES.get("www.facebook.com")).toBe(
|
||||
"facebook.com"
|
||||
);
|
||||
expect(BUILT_IN_ALIASES.get("m.reddit.com")).toBe("reddit.com");
|
||||
expect(BUILT_IN_ALIASES.get("old.reddit.com")).toBe("reddit.com");
|
||||
expect(BUILT_IN_ALIASES.get("new.reddit.com")).toBe("reddit.com");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getCanonicalDomain", () => {
|
||||
it("returns canonical for known alias", { timeout: 5000 }, () => {
|
||||
expect(getCanonicalDomain("m.youtube.com")).toBe("youtube.com");
|
||||
expect(getCanonicalDomain("old.reddit.com")).toBe("reddit.com");
|
||||
expect(getCanonicalDomain("amp.theguardian.com")).toBe(
|
||||
"theguardian.com"
|
||||
);
|
||||
});
|
||||
|
||||
it("returns input unchanged for unknown domains", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(getCanonicalDomain("example.com")).toBe("example.com");
|
||||
expect(getCanonicalDomain("unknown.subdomain.net")).toBe(
|
||||
"unknown.subdomain.net"
|
||||
);
|
||||
});
|
||||
|
||||
it("is idempotent — canonical returns itself", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(getCanonicalDomain("youtube.com")).toBe("youtube.com");
|
||||
expect(getCanonicalDomain("reddit.com")).toBe("reddit.com");
|
||||
expect(getCanonicalDomain("facebook.com")).toBe("facebook.com");
|
||||
});
|
||||
|
||||
it("handles www.facebook.com alias", { timeout: 5000 }, () => {
|
||||
expect(getCanonicalDomain("www.facebook.com")).toBe("facebook.com");
|
||||
});
|
||||
|
||||
it("handles lite.cnn.com alias", { timeout: 5000 }, () => {
|
||||
expect(getCanonicalDomain("lite.cnn.com")).toBe("cnn.com");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isKnownAlias", () => {
|
||||
it("returns true for registered aliases", { timeout: 5000 }, () => {
|
||||
expect(isKnownAlias("m.youtube.com")).toBe(true);
|
||||
expect(isKnownAlias("old.reddit.com")).toBe(true);
|
||||
expect(isKnownAlias("mobile.facebook.com")).toBe(true);
|
||||
expect(isKnownAlias("m.twitter.com")).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for canonical domains", { timeout: 5000 }, () => {
|
||||
expect(isKnownAlias("youtube.com")).toBe(false);
|
||||
expect(isKnownAlias("reddit.com")).toBe(false);
|
||||
expect(isKnownAlias("facebook.com")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for unknown domains", { timeout: 5000 }, () => {
|
||||
expect(isKnownAlias("example.com")).toBe(false);
|
||||
expect(isKnownAlias("notaliased.io")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getAliasGroup", () => {
|
||||
it("returns all reddit aliases from a variant", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const group = getAliasGroup("m.reddit.com");
|
||||
expect(group).toContain("reddit.com");
|
||||
expect(group).toContain("m.reddit.com");
|
||||
expect(group).toContain("old.reddit.com");
|
||||
expect(group).toContain("new.reddit.com");
|
||||
});
|
||||
|
||||
it("returns same group from canonical", { timeout: 5000 }, () => {
|
||||
const fromAlias = getAliasGroup("m.reddit.com");
|
||||
const fromCanonical = getAliasGroup("reddit.com");
|
||||
expect(fromAlias).toEqual(fromCanonical);
|
||||
});
|
||||
|
||||
it("returns sorted array", { timeout: 5000 }, () => {
|
||||
const group = getAliasGroup("m.reddit.com");
|
||||
const sorted = [...group].sort();
|
||||
expect(group).toEqual(sorted);
|
||||
});
|
||||
|
||||
it("returns [canonical] for unknown domain", { timeout: 5000 }, () => {
|
||||
const group = getAliasGroup("example.com");
|
||||
expect(group).toEqual(["example.com"]);
|
||||
});
|
||||
|
||||
it("includes the canonical in the group", { timeout: 5000 }, () => {
|
||||
const group = getAliasGroup("m.facebook.com");
|
||||
expect(group).toContain("facebook.com");
|
||||
});
|
||||
|
||||
it("facebook group includes all variants", { timeout: 5000 }, () => {
|
||||
const group = getAliasGroup("facebook.com");
|
||||
expect(group).toContain("m.facebook.com");
|
||||
expect(group).toContain("mobile.facebook.com");
|
||||
expect(group).toContain("www.facebook.com");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resetAliasCache", () => {
|
||||
it("clears the reverse index cache without error", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// Prime the cache
|
||||
getAliasGroup("reddit.com");
|
||||
// Reset
|
||||
expect(() => resetAliasCache()).not.toThrow();
|
||||
// Still works after reset
|
||||
const group = getAliasGroup("reddit.com");
|
||||
expect(group).toContain("reddit.com");
|
||||
});
|
||||
|
||||
it("cache is rebuilt after reset", { timeout: 5000 }, () => {
|
||||
const g1 = getAliasGroup("m.reddit.com");
|
||||
resetAliasCache();
|
||||
const g2 = getAliasGroup("m.reddit.com");
|
||||
expect(g1).toEqual(g2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Built-in alias map and lookup functions for well-known domain aliases.
|
||||
* Maps subdomain variants (e.g. m.youtube.com) to their canonical registrable domain.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Built-in alias map: variant domain → canonical domain.
|
||||
* Covers major mobile/AMP/legacy subdomain patterns for popular domains.
|
||||
*/
|
||||
export const BUILT_IN_ALIASES: ReadonlyMap<string, string> = new Map([
|
||||
// YouTube
|
||||
["m.youtube.com", "youtube.com"],
|
||||
// Twitter / X
|
||||
["mobile.twitter.com", "twitter.com"],
|
||||
["m.twitter.com", "twitter.com"],
|
||||
["mobile.x.com", "x.com"],
|
||||
["m.x.com", "x.com"],
|
||||
// The Guardian
|
||||
["amp.theguardian.com", "theguardian.com"],
|
||||
// CNN
|
||||
["lite.cnn.com", "cnn.com"],
|
||||
["amp.cnn.com", "cnn.com"],
|
||||
["m.cnn.com", "cnn.com"],
|
||||
// Facebook
|
||||
["m.facebook.com", "facebook.com"],
|
||||
["mobile.facebook.com", "facebook.com"],
|
||||
["www.facebook.com", "facebook.com"],
|
||||
// Reddit
|
||||
["m.reddit.com", "reddit.com"],
|
||||
["old.reddit.com", "reddit.com"],
|
||||
["new.reddit.com", "reddit.com"],
|
||||
["i.reddit.com", "reddit.com"],
|
||||
// Instagram
|
||||
["m.instagram.com", "instagram.com"],
|
||||
// LinkedIn
|
||||
["m.linkedin.com", "linkedin.com"],
|
||||
// Wikipedia
|
||||
["m.wikipedia.org", "wikipedia.org"],
|
||||
// GitHub
|
||||
["m.github.com", "github.com"],
|
||||
// Google
|
||||
["m.google.com", "google.com"],
|
||||
// Amazon
|
||||
["m.amazon.com", "amazon.com"],
|
||||
["smile.amazon.com", "amazon.com"],
|
||||
// TikTok
|
||||
["m.tiktok.com", "tiktok.com"],
|
||||
// Bing
|
||||
["m.bing.com", "bing.com"],
|
||||
// DuckDuckGo
|
||||
["m.duckduckgo.com", "duckduckgo.com"],
|
||||
]);
|
||||
|
||||
/**
|
||||
* Reverse-index: canonical domain → Set of all aliases for that canonical.
|
||||
* Built once at module load for O(1) group lookups.
|
||||
*/
|
||||
let _reverseIndex: Map<string, Set<string>> | null = null;
|
||||
|
||||
function getReverseIndex(): Map<string, Set<string>> {
|
||||
if (_reverseIndex !== null) return _reverseIndex;
|
||||
|
||||
const idx = new Map<string, Set<string>>();
|
||||
for (const [alias, canonical] of BUILT_IN_ALIASES) {
|
||||
let group = idx.get(canonical);
|
||||
if (!group) {
|
||||
group = new Set([canonical]);
|
||||
idx.set(canonical, group);
|
||||
}
|
||||
group.add(alias);
|
||||
}
|
||||
_reverseIndex = idx;
|
||||
return idx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the canonical domain for a given alias, or the input itself
|
||||
* if it is not a known alias.
|
||||
*
|
||||
* @param domain - The domain string to look up (e.g. "m.youtube.com")
|
||||
* @returns Canonical domain (e.g. "youtube.com") or original domain
|
||||
*/
|
||||
export function getCanonicalDomain(domain: string): string {
|
||||
return BUILT_IN_ALIASES.get(domain) ?? domain;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether the given domain is a known alias in the built-in map.
|
||||
*
|
||||
* @param domain - The domain string to check
|
||||
* @returns true if domain is a registered alias
|
||||
*/
|
||||
export function isKnownAlias(domain: string): boolean {
|
||||
return BUILT_IN_ALIASES.has(domain);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all domain strings that share the same canonical form as the given domain.
|
||||
* Includes the canonical domain itself.
|
||||
*
|
||||
* @param domain - A domain or alias to look up (e.g. "m.reddit.com" or "reddit.com")
|
||||
* @returns Array of all equivalent domains including the canonical
|
||||
*/
|
||||
export function getAliasGroup(domain: string): string[] {
|
||||
const canonical = getCanonicalDomain(domain);
|
||||
const idx = getReverseIndex();
|
||||
const group = idx.get(canonical);
|
||||
if (!group) {
|
||||
// domain is canonical but has no aliases in our map — return self
|
||||
return [canonical];
|
||||
}
|
||||
return Array.from(group).sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the internal reverse-index cache.
|
||||
* Useful in test environments when the alias map is modified.
|
||||
*/
|
||||
export function resetAliasCache(): void {
|
||||
_reverseIndex = null;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
BUILT_IN_ALIASES,
|
||||
getAliasGroup,
|
||||
getCanonicalDomain,
|
||||
isKnownAlias,
|
||||
resetAliasCache,
|
||||
} from "./aliases";
|
||||
@@ -0,0 +1,22 @@
|
||||
export {
|
||||
BUILT_IN_ALIASES,
|
||||
getAliasGroup,
|
||||
getCanonicalDomain,
|
||||
isKnownAlias,
|
||||
resetAliasCache,
|
||||
} from "./aliases";
|
||||
export {
|
||||
computeSimilarity,
|
||||
DEFAULT_MATCH_THRESHOLD,
|
||||
findMatchingDomains,
|
||||
groupDomains,
|
||||
levenshteinDistance,
|
||||
matchDomains,
|
||||
} from "./matcher";
|
||||
export {
|
||||
getDomainGroup,
|
||||
getRegistrableDomain,
|
||||
getSubdomain,
|
||||
normalizeDomain,
|
||||
normalizeUrl,
|
||||
} from "./normalizer";
|
||||
@@ -0,0 +1,8 @@
|
||||
export {
|
||||
computeSimilarity,
|
||||
DEFAULT_MATCH_THRESHOLD,
|
||||
findMatchingDomains,
|
||||
groupDomains,
|
||||
levenshteinDistance,
|
||||
matchDomains,
|
||||
} from "./matcher";
|
||||
@@ -0,0 +1,127 @@
|
||||
# matcher
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides domain similarity scoring and grouping functions for deduplication workflows. Uses Levenshtein edit distance on registrable domain parts (eTLD+1) to compute a normalized similarity score in `[0, 1]`, enabling fuzzy matching of typos, regional TLD variants, and subdomain variants that survive normalization.
|
||||
|
||||
The grouping function builds on the normalizer's canonical domain resolution, collecting all aliases of the same domain into a single group for deduplication pipelines.
|
||||
|
||||
## Features
|
||||
|
||||
- `levenshteinDistance(a, b)` — classic two-row DP edit distance, O(m×n) time, O(n) space
|
||||
- `computeSimilarity(a, b)` — normalized distance score: 1.0 = exact, 0.0 = fully different
|
||||
- `matchDomains(domain1, domain2)` — full `MatchResult` with similarity and boolean threshold flag
|
||||
- `findMatchingDomains(needle, haystack, threshold?)` — sorted list of haystack entries that match a needle
|
||||
- `groupDomains(domains)` — groups a list of domain strings by their canonical registrable domain
|
||||
- `DEFAULT_MATCH_THRESHOLD` — exported constant (0.8) for custom threshold decisions
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { matchDomains, computeSimilarity } from "@suaveplan/domain-normalizer/matcher";
|
||||
|
||||
const result = matchDomains("youtube.com", "m.youtube.com");
|
||||
console.log(result.similarity); // 1.0 (both normalize to youtube.com)
|
||||
console.log(result.isMatch); // true
|
||||
|
||||
const score = computeSimilarity("google.com", "googel.com");
|
||||
console.log(score); // ~0.857 (one typo in 7-char string)
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
findMatchingDomains,
|
||||
groupDomains,
|
||||
levenshteinDistance,
|
||||
} from "@suaveplan/domain-normalizer/matcher";
|
||||
|
||||
// Find duplicates of a known domain in a list
|
||||
const matches = findMatchingDomains("reddit.com", [
|
||||
"m.reddit.com",
|
||||
"old.reddit.com",
|
||||
"example.com",
|
||||
"reddlt.com",
|
||||
], 0.8);
|
||||
// ["m.reddit.com", "old.reddit.com", "reddlt.com"]
|
||||
|
||||
// Group a crawled link list by canonical domain
|
||||
const grouped = groupDomains([
|
||||
"m.youtube.com",
|
||||
"youtube.com",
|
||||
"old.reddit.com",
|
||||
"reddit.com",
|
||||
"example.com",
|
||||
]);
|
||||
// Map {
|
||||
// "youtube.com" => ["m.youtube.com", "youtube.com"],
|
||||
// "reddit.com" => ["old.reddit.com", "reddit.com"],
|
||||
// "example.com" => ["example.com"]
|
||||
// }
|
||||
|
||||
// Raw edit distance
|
||||
console.log(levenshteinDistance("kitten", "sitting")); // 3
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `DEFAULT_MATCH_THRESHOLD`
|
||||
|
||||
```ts
|
||||
const DEFAULT_MATCH_THRESHOLD: number // 0.8
|
||||
```
|
||||
|
||||
### `levenshteinDistance(a, b)`
|
||||
|
||||
```ts
|
||||
function levenshteinDistance(a: string, b: string): number
|
||||
```
|
||||
|
||||
Returns the minimum number of single-character edits (insertions, deletions, substitutions) to transform `a` into `b`. Returns 0 for identical strings. Property: `distance(a, a) === 0`; triangle inequality holds.
|
||||
|
||||
### `computeSimilarity(a, b)`
|
||||
|
||||
```ts
|
||||
function computeSimilarity(a: string, b: string): number
|
||||
```
|
||||
|
||||
Returns a score in `[0, 1]`. Computes `1 - levenshtein(regA, regB) / max(len(regA), len(regB))` where `regA`/`regB` are the registrable domain parts. Returns `1.0` for two empty strings.
|
||||
|
||||
### `matchDomains(domain1, domain2)`
|
||||
|
||||
```ts
|
||||
function matchDomains(domain1: string, domain2: string): MatchResult
|
||||
```
|
||||
|
||||
Returns `{ domain1, domain2, similarity, isMatch }` where `isMatch = similarity >= DEFAULT_MATCH_THRESHOLD`.
|
||||
|
||||
### `findMatchingDomains(needle, haystack, threshold?)`
|
||||
|
||||
```ts
|
||||
function findMatchingDomains(
|
||||
needle: string,
|
||||
haystack: string[],
|
||||
threshold?: number,
|
||||
): string[]
|
||||
```
|
||||
|
||||
Returns haystack entries with `computeSimilarity(needle, entry) >= threshold`, sorted by similarity descending.
|
||||
|
||||
### `groupDomains(domains)`
|
||||
|
||||
```ts
|
||||
function groupDomains(domains: string[]): Map<string, string[]>
|
||||
```
|
||||
|
||||
Returns a `Map<canonicalDomain, string[]>` grouping all input domains by their canonical registrable domain. Preserves insertion order within each group.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- `levenshteinDistance` uses the standard two-row optimization: only two rows of the DP table are kept in memory at a time, keeping space complexity at O(n).
|
||||
- `computeSimilarity` calls `getRegistrableDomain` first — two aliases that normalize to the same eTLD+1 always score 1.0.
|
||||
- `findMatchingDomains` always computes similarity for every haystack entry; it does not short-circuit for large haystacks. For very large lists (thousands of entries), consider pre-grouping first.
|
||||
- `groupDomains` relies on `getDomainGroup` which never throws, so this function is also safe to use in non-throwing map/reduce pipelines.
|
||||
@@ -0,0 +1,297 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
computeSimilarity,
|
||||
DEFAULT_MATCH_THRESHOLD,
|
||||
findMatchingDomains,
|
||||
groupDomains,
|
||||
levenshteinDistance,
|
||||
matchDomains,
|
||||
} from "./matcher.js";
|
||||
|
||||
describe("levenshteinDistance", () => {
|
||||
describe("basic cases", () => {
|
||||
it("returns 0 for identical strings", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("", "")).toBe(0);
|
||||
expect(levenshteinDistance("abc", "abc")).toBe(0);
|
||||
expect(levenshteinDistance("example.com", "example.com")).toBe(0);
|
||||
});
|
||||
|
||||
it("returns length of other string when one is empty", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(levenshteinDistance("", "abc")).toBe(3);
|
||||
expect(levenshteinDistance("abc", "")).toBe(3);
|
||||
});
|
||||
|
||||
it("returns 1 for single insertion", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("ab", "abc")).toBe(1);
|
||||
expect(levenshteinDistance("abc", "ab")).toBe(1);
|
||||
});
|
||||
|
||||
it("returns 1 for single substitution", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("abc", "axc")).toBe(1);
|
||||
});
|
||||
|
||||
it("returns 1 for single deletion", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("abc", "bc")).toBe(1);
|
||||
});
|
||||
|
||||
it("computes kitten → sitting = 3", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("kitten", "sitting")).toBe(3);
|
||||
});
|
||||
|
||||
it("computes saturday → sunday = 3", { timeout: 5000 }, () => {
|
||||
expect(levenshteinDistance("saturday", "sunday")).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("triangle inequality", () => {
|
||||
it("dist(a,c) <= dist(a,b) + dist(b,c)", { timeout: 5000 }, () => {
|
||||
const triples: [string, string, string][] = [
|
||||
["abc", "abd", "acd"],
|
||||
["example.com", "exampl.com", "exmpl.com"],
|
||||
["google.com", "gogle.com", "goggle.com"],
|
||||
["", "a", "ab"],
|
||||
["foo", "foo", "bar"],
|
||||
];
|
||||
for (const [a, b, c] of triples) {
|
||||
const dac = levenshteinDistance(a, c);
|
||||
const dab = levenshteinDistance(a, b);
|
||||
const dbc = levenshteinDistance(b, c);
|
||||
expect(dac).toBeLessThanOrEqual(dab + dbc);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("property: distance(a,a) === 0", { timeout: 5000 }, () => {
|
||||
it("holds for various strings", { timeout: 5000 }, () => {
|
||||
const strings = [
|
||||
"",
|
||||
"a",
|
||||
"abc",
|
||||
"example.com",
|
||||
"youtube.com",
|
||||
"foo.bar.baz",
|
||||
];
|
||||
for (const s of strings) {
|
||||
expect(levenshteinDistance(s, s)).toBe(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("property: symmetry", () => {
|
||||
it("dist(a,b) === dist(b,a)", { timeout: 5000 }, () => {
|
||||
const pairs: [string, string][] = [
|
||||
["abc", "xyz"],
|
||||
["example.com", "exampl.com"],
|
||||
["", "abc"],
|
||||
["kitten", "sitting"],
|
||||
];
|
||||
for (const [a, b] of pairs) {
|
||||
expect(levenshteinDistance(a, b)).toBe(
|
||||
levenshteinDistance(b, a)
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeSimilarity", () => {
|
||||
it("returns 1.0 for identical domains", { timeout: 5000 }, () => {
|
||||
expect(computeSimilarity("example.com", "example.com")).toBe(1.0);
|
||||
});
|
||||
|
||||
it("returns 1.0 for alias and canonical (same registrable domain)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(computeSimilarity("m.youtube.com", "youtube.com")).toBe(1.0);
|
||||
expect(computeSimilarity("old.reddit.com", "reddit.com")).toBe(1.0);
|
||||
});
|
||||
|
||||
it("returns 1.0 for two aliases of same canonical", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(computeSimilarity("m.youtube.com", "m.youtube.com")).toBe(1.0);
|
||||
});
|
||||
|
||||
it("returns value in [0, 1]", { timeout: 5000 }, () => {
|
||||
const score = computeSimilarity("google.com", "facebook.com");
|
||||
expect(score).toBeGreaterThanOrEqual(0);
|
||||
expect(score).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("returns high score for near-duplicate domains (typo)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const score = computeSimilarity("google.com", "googel.com");
|
||||
expect(score).toBeGreaterThan(0.7);
|
||||
});
|
||||
|
||||
it("returns lower score for different domains", { timeout: 5000 }, () => {
|
||||
const score = computeSimilarity("google.com", "facebook.com");
|
||||
expect(score).toBeLessThan(0.8);
|
||||
});
|
||||
|
||||
it("is symmetric", { timeout: 5000 }, () => {
|
||||
const s1 = computeSimilarity("google.com", "gogle.com");
|
||||
const s2 = computeSimilarity("gogle.com", "google.com");
|
||||
expect(s1).toBe(s2);
|
||||
});
|
||||
|
||||
it("uses raw input when getRegistrableDomain returns null (bare TLD)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// "com" has no registrable domain — getRegistrableDomain returns null,
|
||||
// fallback to lowercase trim of raw input
|
||||
const score = computeSimilarity("com", "com");
|
||||
expect(score).toBe(1.0);
|
||||
});
|
||||
|
||||
it("handles two inputs where both fall back to raw (covers ?? branch)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const score = computeSimilarity("com", "org");
|
||||
expect(score).toBeGreaterThanOrEqual(0);
|
||||
expect(score).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("returns 1.0 when both inputs are empty strings", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// Both getRegistrableDomain("") return null, fallback to "" — regA === regB → 1.0
|
||||
const score = computeSimilarity("", "");
|
||||
expect(score).toBe(1.0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("matchDomains", () => {
|
||||
it("returns isMatch true for identical domains", { timeout: 5000 }, () => {
|
||||
const result = matchDomains("example.com", "example.com");
|
||||
expect(result.similarity).toBe(1.0);
|
||||
expect(result.isMatch).toBe(true);
|
||||
});
|
||||
|
||||
it("returns isMatch true for aliases of same canonical", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = matchDomains("m.youtube.com", "youtube.com");
|
||||
expect(result.isMatch).toBe(true);
|
||||
expect(result.similarity).toBe(1.0);
|
||||
});
|
||||
|
||||
it("returns isMatch false for unrelated domains", { timeout: 5000 }, () => {
|
||||
const result = matchDomains("google.com", "amazon.com");
|
||||
expect(result.isMatch).toBe(false);
|
||||
});
|
||||
|
||||
it("includes domain1 and domain2 in result", { timeout: 5000 }, () => {
|
||||
const result = matchDomains("a.com", "b.com");
|
||||
expect(result.domain1).toBe("a.com");
|
||||
expect(result.domain2).toBe("b.com");
|
||||
});
|
||||
|
||||
it("similarity is in [0, 1]", { timeout: 5000 }, () => {
|
||||
const result = matchDomains("foo.com", "bar.com");
|
||||
expect(result.similarity).toBeGreaterThanOrEqual(0);
|
||||
expect(result.similarity).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findMatchingDomains", () => {
|
||||
it("returns matches above default threshold", { timeout: 5000 }, () => {
|
||||
const matches = findMatchingDomains("youtube.com", [
|
||||
"m.youtube.com",
|
||||
"youtube.com",
|
||||
"example.com",
|
||||
]);
|
||||
expect(matches).toContain("m.youtube.com");
|
||||
expect(matches).toContain("youtube.com");
|
||||
expect(matches).not.toContain("example.com");
|
||||
});
|
||||
|
||||
it("returns sorted by similarity descending", { timeout: 5000 }, () => {
|
||||
const matches = findMatchingDomains("reddit.com", [
|
||||
"m.reddit.com",
|
||||
"old.reddit.com",
|
||||
"new.reddit.com",
|
||||
]);
|
||||
expect(matches.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("returns empty array when nothing matches", { timeout: 5000 }, () => {
|
||||
const matches = findMatchingDomains("example.com", [
|
||||
"totally-different.org",
|
||||
]);
|
||||
expect(matches).toEqual([]);
|
||||
});
|
||||
|
||||
it("respects custom threshold", { timeout: 5000 }, () => {
|
||||
const matchesHigh = findMatchingDomains(
|
||||
"google.com",
|
||||
["gogle.com"],
|
||||
1.0
|
||||
);
|
||||
expect(matchesHigh).toEqual([]);
|
||||
|
||||
const matchesLow = findMatchingDomains(
|
||||
"google.com",
|
||||
["gogle.com"],
|
||||
0.5
|
||||
);
|
||||
expect(matchesLow).toContain("gogle.com");
|
||||
});
|
||||
|
||||
it("handles empty haystack", { timeout: 5000 }, () => {
|
||||
expect(findMatchingDomains("example.com", [])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupDomains", () => {
|
||||
it("groups aliases under the same canonical key", { timeout: 5000 }, () => {
|
||||
const groups = groupDomains([
|
||||
"m.youtube.com",
|
||||
"youtube.com",
|
||||
"old.reddit.com",
|
||||
"reddit.com",
|
||||
]);
|
||||
expect(groups.get("youtube.com")).toContain("m.youtube.com");
|
||||
expect(groups.get("youtube.com")).toContain("youtube.com");
|
||||
expect(groups.get("reddit.com")).toContain("old.reddit.com");
|
||||
expect(groups.get("reddit.com")).toContain("reddit.com");
|
||||
});
|
||||
|
||||
it("places unique domains in their own group", { timeout: 5000 }, () => {
|
||||
const groups = groupDomains(["example.com", "other.org"]);
|
||||
expect(groups.get("example.com")).toEqual(["example.com"]);
|
||||
expect(groups.get("other.org")).toEqual(["other.org"]);
|
||||
});
|
||||
|
||||
it("handles empty input", { timeout: 5000 }, () => {
|
||||
const groups = groupDomains([]);
|
||||
expect(groups.size).toBe(0);
|
||||
});
|
||||
|
||||
it("handles duplicate entries", { timeout: 5000 }, () => {
|
||||
const groups = groupDomains(["example.com", "example.com"]);
|
||||
const group = groups.get("example.com");
|
||||
expect(group).toBeDefined();
|
||||
expect(group?.length).toBe(2);
|
||||
});
|
||||
|
||||
it("returns Map instance", { timeout: 5000 }, () => {
|
||||
expect(groupDomains(["a.com"])).toBeInstanceOf(Map);
|
||||
});
|
||||
|
||||
it("preserves all input domains across groups", { timeout: 5000 }, () => {
|
||||
const domains = ["m.youtube.com", "youtube.com", "example.com"];
|
||||
const groups = groupDomains(domains);
|
||||
const allGrouped = Array.from(groups.values()).flat();
|
||||
expect(allGrouped.sort()).toEqual(domains.sort());
|
||||
});
|
||||
});
|
||||
|
||||
describe("DEFAULT_MATCH_THRESHOLD", () => {
|
||||
it("is 0.8", { timeout: 5000 }, () => {
|
||||
expect(DEFAULT_MATCH_THRESHOLD).toBe(0.8);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Domain matching and similarity functions for deduplication and grouping.
|
||||
* Uses Levenshtein distance normalized by max length for similarity scoring.
|
||||
*/
|
||||
|
||||
import type { MatchResult } from "../../types/domain-normalizer.types.js";
|
||||
import {
|
||||
getDomainGroup,
|
||||
getRegistrableDomain,
|
||||
} from "../normalizer/normalizer.js";
|
||||
|
||||
/** Default similarity threshold for isMatch. */
|
||||
export const DEFAULT_MATCH_THRESHOLD = 0.8;
|
||||
|
||||
/**
|
||||
* Computes the Levenshtein edit distance between two strings.
|
||||
* Returns 0 for identical strings. Uses a standard two-row DP approach.
|
||||
*
|
||||
* @param a - First string
|
||||
* @param b - Second string
|
||||
* @returns Non-negative integer edit distance
|
||||
*/
|
||||
export function levenshteinDistance(a: string, b: string): number {
|
||||
if (a === b) return 0;
|
||||
if (a.length === 0) return b.length;
|
||||
if (b.length === 0) return a.length;
|
||||
|
||||
const bLen = b.length;
|
||||
// Use typed arrays for guaranteed numeric values (avoids noUncheckedIndexedAccess issues)
|
||||
let prev = new Int32Array(bLen + 1);
|
||||
let curr = new Int32Array(bLen + 1);
|
||||
|
||||
for (let j = 0; j <= bLen; j++) prev[j] = j;
|
||||
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= bLen; j++) {
|
||||
if (a[i - 1] === b[j - 1]) {
|
||||
curr[j] = prev[j - 1] as number;
|
||||
} else {
|
||||
curr[j] =
|
||||
1 +
|
||||
Math.min(
|
||||
prev[j - 1] as number,
|
||||
prev[j] as number,
|
||||
curr[j - 1] as number
|
||||
);
|
||||
}
|
||||
}
|
||||
const tmp = prev;
|
||||
prev = curr;
|
||||
curr = tmp;
|
||||
}
|
||||
|
||||
return prev[bLen] as number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes a similarity score between two domain strings in the range [0, 1].
|
||||
* Compares the registrable domain parts using Levenshtein distance normalized
|
||||
* by the length of the longer string.
|
||||
*
|
||||
* @param a - First domain string
|
||||
* @param b - Second domain string
|
||||
* @returns Similarity score: 1.0 = exact match, 0.0 = completely different
|
||||
*/
|
||||
export function computeSimilarity(a: string, b: string): number {
|
||||
const regA = getRegistrableDomain(a) ?? a.toLowerCase().trim();
|
||||
const regB = getRegistrableDomain(b) ?? b.toLowerCase().trim();
|
||||
|
||||
if (regA === regB) return 1.0;
|
||||
|
||||
const maxLen = Math.max(regA.length, regB.length);
|
||||
const dist = levenshteinDistance(regA, regB);
|
||||
return 1 - dist / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares two domain strings and returns a MatchResult with similarity score.
|
||||
* Two domains are considered a match when their similarity meets the default threshold (0.8).
|
||||
*
|
||||
* @param domain1 - First domain string
|
||||
* @param domain2 - Second domain string
|
||||
* @returns MatchResult with similarity score and isMatch flag
|
||||
*/
|
||||
export function matchDomains(domain1: string, domain2: string): MatchResult {
|
||||
const similarity = computeSimilarity(domain1, domain2);
|
||||
return {
|
||||
domain1,
|
||||
domain2,
|
||||
similarity,
|
||||
isMatch: similarity >= DEFAULT_MATCH_THRESHOLD,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds all domains in a haystack that match a needle domain above a similarity threshold.
|
||||
*
|
||||
* @param needle - The domain to search for
|
||||
* @param haystack - Array of domain strings to search in
|
||||
* @param threshold - Minimum similarity score (default: DEFAULT_MATCH_THRESHOLD)
|
||||
* @returns Array of matching domain strings from haystack, sorted by similarity descending
|
||||
*/
|
||||
export function findMatchingDomains(
|
||||
needle: string,
|
||||
haystack: string[],
|
||||
threshold: number = DEFAULT_MATCH_THRESHOLD
|
||||
): string[] {
|
||||
return haystack
|
||||
.map((candidate) => ({
|
||||
candidate,
|
||||
similarity: computeSimilarity(needle, candidate),
|
||||
}))
|
||||
.filter(({ similarity }) => similarity >= threshold)
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.map(({ candidate }) => candidate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups an array of domain strings by their canonical registrable domain.
|
||||
* All aliases and variants that share the same canonical are placed in the same group.
|
||||
*
|
||||
* @param domains - Array of domain strings
|
||||
* @returns Map from canonical registrable domain to array of input domains in that group
|
||||
*/
|
||||
export function groupDomains(domains: string[]): Map<string, string[]> {
|
||||
const groups = new Map<string, string[]>();
|
||||
|
||||
for (const domain of domains) {
|
||||
const key = getDomainGroup(domain);
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = [];
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.push(domain);
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
export {
|
||||
getDomainGroup,
|
||||
getRegistrableDomain,
|
||||
getSubdomain,
|
||||
normalizeDomain,
|
||||
normalizeUrl,
|
||||
} from "./normalizer";
|
||||
@@ -0,0 +1,110 @@
|
||||
# normalizer
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides the core domain and URL normalization functions for `@suaveplan/domain-normalizer`. Uses `tldts` for accurate public suffix list (PSL) lookups to extract registrable domains (eTLD+1), and integrates with the alias module to resolve known mobile/AMP subdomain variants to their canonical form.
|
||||
|
||||
The normalizer is designed to be idempotent: `normalizeDomain(normalizeDomain(x))` produces the same result as `normalizeDomain(x)` for any valid input.
|
||||
|
||||
## Features
|
||||
|
||||
- `normalizeDomain(input, config?)` — normalizes a bare domain string with configurable stripping and alias resolution
|
||||
- `normalizeUrl(url, config?)` — extracts hostname from a full URL, then normalizes it
|
||||
- `getRegistrableDomain(input)` — returns the eTLD+1 or null for invalid inputs
|
||||
- `getSubdomain(input)` — returns the subdomain portion or null
|
||||
- `getDomainGroup(input)` — returns the canonical registrable domain as a grouping key
|
||||
- Configurable via `NormalizerConfig`: strip www, strip mobile prefixes, lowercase, resolve aliases
|
||||
- Returns rich `NormalizedDomain` objects with subdomain, publicSuffix, and isPrivateDomain metadata
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { normalizeDomain } from "@suaveplan/domain-normalizer/normalizer";
|
||||
|
||||
const result = normalizeDomain("m.youtube.com");
|
||||
console.log(result.normalized); // "youtube.com"
|
||||
console.log(result.registrableDomain); // "youtube.com"
|
||||
console.log(result.publicSuffix); // "com"
|
||||
console.log(result.original); // "m.youtube.com"
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { normalizeUrl, getDomainGroup } from "@suaveplan/domain-normalizer/normalizer";
|
||||
|
||||
// Normalize a full URL
|
||||
const result = normalizeUrl("https://old.reddit.com/r/programming");
|
||||
console.log(result.normalized); // "reddit.com"
|
||||
console.log(result.original); // "https://old.reddit.com/r/programming"
|
||||
|
||||
// Group a mixed list of domains
|
||||
const domains = ["m.youtube.com", "youtube.com", "old.reddit.com", "reddit.com"];
|
||||
const groups = new Map<string, string[]>();
|
||||
for (const d of domains) {
|
||||
const key = getDomainGroup(d);
|
||||
const arr = groups.get(key) ?? [];
|
||||
arr.push(d);
|
||||
groups.set(key, arr);
|
||||
}
|
||||
// Map { "youtube.com" => ["m.youtube.com", "youtube.com"], "reddit.com" => [...] }
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `normalizeDomain(input, config?)`
|
||||
|
||||
```ts
|
||||
function normalizeDomain(input: string, config?: NormalizerConfig): NormalizedDomain
|
||||
```
|
||||
|
||||
Normalizes a bare domain string. Applies lowercase, alias resolution, www/mobile stripping, and PSL parsing in that order.
|
||||
|
||||
- **Throws** `DomainNormalizerError(INVALID_INPUT)` for empty/whitespace-only inputs
|
||||
- **Throws** `DomainNormalizerError(PARSE_ERROR)` when tldts cannot extract a registrable domain
|
||||
|
||||
### `normalizeUrl(url, config?)`
|
||||
|
||||
```ts
|
||||
function normalizeUrl(url: string, config?: NormalizerConfig): NormalizedDomain
|
||||
```
|
||||
|
||||
Extracts the hostname from `url` (using `URL` constructor with protocol injection if needed), then calls `normalizeDomain` on it. The `original` field in the result contains the full URL.
|
||||
|
||||
- **Throws** `DomainNormalizerError(INVALID_INPUT)` for empty inputs
|
||||
- **Throws** `DomainNormalizerError(PARSE_ERROR)` when hostname cannot be extracted
|
||||
|
||||
### `getRegistrableDomain(input)`
|
||||
|
||||
```ts
|
||||
function getRegistrableDomain(input: string): string | null
|
||||
```
|
||||
|
||||
Returns the eTLD+1 for `input`, or `null` if parsing fails or input is empty. Does not throw.
|
||||
|
||||
### `getSubdomain(input)`
|
||||
|
||||
```ts
|
||||
function getSubdomain(input: string): string | null
|
||||
```
|
||||
|
||||
Returns the subdomain portion of `input`, or `null` if there is none. Does not throw.
|
||||
|
||||
### `getDomainGroup(input)`
|
||||
|
||||
```ts
|
||||
function getDomainGroup(input: string): string
|
||||
```
|
||||
|
||||
Returns the canonical registrable domain for use as a deduplication/grouping key. Falls back to lowercased trimmed input if normalization fails. Does not throw.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Alias resolution is applied before prefix stripping so that an alias like `www.facebook.com` maps to `facebook.com` (rather than stripping `www.` first and getting `facebook.com` without alias resolution running).
|
||||
- `tldts` handles private domains (e.g. `github.io`) correctly when `allowPrivateDomains: true` is set.
|
||||
- The `URL` constructor is used for URL hostname extraction to handle percent-encoding, authentication components, and port numbers transparently.
|
||||
- `stripPrefixes` loops until no more changes occur to handle edge cases like hypothetical `m.m.domain.com` inputs.
|
||||
- `getDomainGroup` catches all errors and falls back to the lowercased input to ensure it never throws — making it safe for use in array mapping operations.
|
||||
@@ -0,0 +1,335 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
DOMAIN_NORMALIZER_CODES,
|
||||
DomainNormalizerError,
|
||||
} from "../../errors/domain-normalizer-error.js";
|
||||
import {
|
||||
getDomainGroup,
|
||||
getRegistrableDomain,
|
||||
getSubdomain,
|
||||
normalizeDomain,
|
||||
normalizeUrl,
|
||||
} from "./normalizer.js";
|
||||
|
||||
describe("normalizeDomain", () => {
|
||||
describe("basic normalization", () => {
|
||||
it("returns registrable domain for simple input", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = normalizeDomain("example.com");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
expect(result.registrableDomain).toBe("example.com");
|
||||
expect(result.original).toBe("example.com");
|
||||
});
|
||||
|
||||
it("lowercases by default", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("EXAMPLE.COM");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("strips www by default", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("www.example.com");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("strips m. mobile prefix by default", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("m.example.com");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("strips mobile. prefix by default", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("mobile.example.com");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("strips amp. prefix by default", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("amp.example.com");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("strips lite. prefix by default", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("lite.example.com");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("resolves m.youtube.com alias", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("m.youtube.com");
|
||||
expect(result.normalized).toBe("youtube.com");
|
||||
expect(result.registrableDomain).toBe("youtube.com");
|
||||
});
|
||||
|
||||
it("resolves old.reddit.com alias", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("old.reddit.com");
|
||||
expect(result.normalized).toBe("reddit.com");
|
||||
});
|
||||
|
||||
it("resolves www.facebook.com alias", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("www.facebook.com");
|
||||
expect(result.normalized).toBe("facebook.com");
|
||||
});
|
||||
|
||||
it("preserves public suffix metadata", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("example.com");
|
||||
expect(result.publicSuffix).toBe("com");
|
||||
});
|
||||
|
||||
it("preserves original input in result", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("M.YouTube.COM");
|
||||
expect(result.original).toBe("M.YouTube.COM");
|
||||
});
|
||||
|
||||
it("sets isPrivateDomain for private-suffix domains (covers isPrivate branch)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// github.io is in the private section of the public suffix list
|
||||
const result = normalizeDomain("user.github.io");
|
||||
expect(result.isPrivateDomain).toBe(true);
|
||||
expect(result.registrableDomain).toBe("user.github.io");
|
||||
});
|
||||
});
|
||||
|
||||
describe("subdomains", () => {
|
||||
it("returns no subdomain for root domain", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("example.com");
|
||||
expect(result.subdomain).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns remaining subdomain after stripping www", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// e.g. sub.www.example.com — after stripping www, sub remains
|
||||
// tldts would parse "sub.example.com" subdomain as "sub"
|
||||
const result = normalizeDomain("blog.example.com");
|
||||
expect(result.subdomain).toBe("blog");
|
||||
});
|
||||
});
|
||||
|
||||
describe("config options", () => {
|
||||
it("respects stripWww: false", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("www.example.com", {
|
||||
stripWww: false,
|
||||
});
|
||||
// www.example.com → registrable domain is still example.com
|
||||
// but normalized should reflect no stripping of alias
|
||||
// tldts: domain is example.com, subdomain is www
|
||||
expect(result.registrableDomain).toBe("example.com");
|
||||
expect(result.subdomain).toBe("www");
|
||||
});
|
||||
|
||||
it("respects stripMobile: false", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("m.example.com", {
|
||||
stripMobile: false,
|
||||
});
|
||||
expect(result.registrableDomain).toBe("example.com");
|
||||
expect(result.subdomain).toBe("m");
|
||||
});
|
||||
|
||||
it("respects toLowercase: false", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("Example.COM", {
|
||||
toLowercase: false,
|
||||
});
|
||||
// tldts should still parse this
|
||||
expect(result.registrableDomain).toBeDefined();
|
||||
});
|
||||
|
||||
it("respects resolveAliases: false", { timeout: 5000 }, () => {
|
||||
const result = normalizeDomain("m.youtube.com", {
|
||||
resolveAliases: false,
|
||||
});
|
||||
// Without alias resolution, the subdomain m. should remain (if not stripped by stripMobile)
|
||||
// With stripMobile: true (default), m. is stripped → youtube.com
|
||||
expect(result.registrableDomain).toBe("youtube.com");
|
||||
});
|
||||
|
||||
it("resolveAliases false + stripMobile false leaves subdomain intact", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const result = normalizeDomain("m.youtube.com", {
|
||||
resolveAliases: false,
|
||||
stripMobile: false,
|
||||
});
|
||||
expect(result.registrableDomain).toBe("youtube.com");
|
||||
expect(result.subdomain).toBe("m");
|
||||
});
|
||||
});
|
||||
|
||||
describe("idempotency", () => {
|
||||
it("normalizeDomain is idempotent for simple domains", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const once = normalizeDomain("example.com");
|
||||
const twice = normalizeDomain(once.normalized);
|
||||
expect(twice.normalized).toBe(once.normalized);
|
||||
});
|
||||
|
||||
it("normalizeDomain is idempotent for aliases", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const once = normalizeDomain("m.youtube.com");
|
||||
const twice = normalizeDomain(once.normalized);
|
||||
expect(twice.normalized).toBe(once.normalized);
|
||||
});
|
||||
|
||||
it("normalizeDomain is idempotent for www domains", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const once = normalizeDomain("www.google.com");
|
||||
const twice = normalizeDomain(once.normalized);
|
||||
expect(twice.normalized).toBe(once.normalized);
|
||||
});
|
||||
});
|
||||
|
||||
describe("error cases", () => {
|
||||
it("throws INVALID_INPUT for empty string", { timeout: 5000 }, () => {
|
||||
expect(() => normalizeDomain("")).toThrow(DomainNormalizerError);
|
||||
expect(() => normalizeDomain("")).toThrow(
|
||||
expect.objectContaining({
|
||||
code: DOMAIN_NORMALIZER_CODES.INVALID_INPUT,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("throws INVALID_INPUT for whitespace-only string", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(() => normalizeDomain(" ")).toThrow(DomainNormalizerError);
|
||||
});
|
||||
|
||||
it("throws PARSE_ERROR for unparseable domain", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// A string that tldts cannot extract a registrable domain from
|
||||
expect(() => normalizeDomain("com")).toThrow(DomainNormalizerError);
|
||||
expect(() => normalizeDomain("com")).toThrow(
|
||||
expect.objectContaining({
|
||||
code: DOMAIN_NORMALIZER_CODES.PARSE_ERROR,
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeUrl", () => {
|
||||
it("handles full https URL", { timeout: 5000 }, () => {
|
||||
const result = normalizeUrl(
|
||||
"https://m.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
);
|
||||
expect(result.normalized).toBe("youtube.com");
|
||||
expect(result.original).toBe(
|
||||
"https://m.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
);
|
||||
});
|
||||
|
||||
it("handles http URL", { timeout: 5000 }, () => {
|
||||
const result = normalizeUrl("http://old.reddit.com/r/programming");
|
||||
expect(result.normalized).toBe("reddit.com");
|
||||
});
|
||||
|
||||
it("handles URL without protocol (bare domain)", { timeout: 5000 }, () => {
|
||||
const result = normalizeUrl("example.com/path");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("preserves full URL as original", { timeout: 5000 }, () => {
|
||||
const url = "https://amp.theguardian.com/news/article";
|
||||
const result = normalizeUrl(url);
|
||||
expect(result.original).toBe(url);
|
||||
expect(result.normalized).toBe("theguardian.com");
|
||||
});
|
||||
|
||||
it("throws INVALID_INPUT for empty string", { timeout: 5000 }, () => {
|
||||
expect(() => normalizeUrl("")).toThrow(DomainNormalizerError);
|
||||
expect(() => normalizeUrl("")).toThrow(
|
||||
expect.objectContaining({
|
||||
code: DOMAIN_NORMALIZER_CODES.INVALID_INPUT,
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it("handles URL with port", { timeout: 5000 }, () => {
|
||||
const result = normalizeUrl("https://example.com:8080/path");
|
||||
expect(result.normalized).toBe("example.com");
|
||||
});
|
||||
|
||||
it("normalizeUrl is idempotent via its normalized output", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const once = normalizeUrl("https://m.youtube.com/watch");
|
||||
const twice = normalizeDomain(once.normalized);
|
||||
expect(twice.normalized).toBe(once.normalized);
|
||||
});
|
||||
|
||||
it("falls back to raw input when URL constructor throws (covers catch branch)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
// "not a url://example.com" contains "://" so no protocol is prepended,
|
||||
// and "new URL('not a url://example.com')" throws — the catch block then
|
||||
// falls back to treating the entire string as a bare domain
|
||||
// normalizeDomain("not a url://example.com") will also throw PARSE_ERROR,
|
||||
// which is expected behaviour for genuinely malformed input
|
||||
expect(() => normalizeUrl("not a url://example.com")).toThrow(
|
||||
DomainNormalizerError
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getRegistrableDomain", () => {
|
||||
it("returns eTLD+1 for simple domain", { timeout: 5000 }, () => {
|
||||
expect(getRegistrableDomain("example.com")).toBe("example.com");
|
||||
});
|
||||
|
||||
it("returns eTLD+1 for subdomain", { timeout: 5000 }, () => {
|
||||
expect(getRegistrableDomain("m.youtube.com")).toBe("youtube.com");
|
||||
});
|
||||
|
||||
it("returns null for empty string", { timeout: 5000 }, () => {
|
||||
expect(getRegistrableDomain("")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for bare TLD", { timeout: 5000 }, () => {
|
||||
expect(getRegistrableDomain("com")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not throw on invalid input", { timeout: 5000 }, () => {
|
||||
expect(() => getRegistrableDomain("not a domain")).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getSubdomain", () => {
|
||||
it("returns subdomain for full domain", { timeout: 5000 }, () => {
|
||||
expect(getSubdomain("m.youtube.com")).toBe("m");
|
||||
});
|
||||
|
||||
it("returns null for root domain", { timeout: 5000 }, () => {
|
||||
expect(getSubdomain("youtube.com")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for empty string", { timeout: 5000 }, () => {
|
||||
expect(getSubdomain("")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns multi-level subdomain", { timeout: 5000 }, () => {
|
||||
expect(getSubdomain("blog.news.example.com")).toBe("blog.news");
|
||||
});
|
||||
});
|
||||
|
||||
describe("getDomainGroup", () => {
|
||||
it("returns canonical registrable domain", { timeout: 5000 }, () => {
|
||||
expect(getDomainGroup("m.youtube.com")).toBe("youtube.com");
|
||||
expect(getDomainGroup("old.reddit.com")).toBe("reddit.com");
|
||||
});
|
||||
|
||||
it("handles already canonical domain", { timeout: 5000 }, () => {
|
||||
expect(getDomainGroup("example.com")).toBe("example.com");
|
||||
});
|
||||
|
||||
it("does not throw on invalid input", { timeout: 5000 }, () => {
|
||||
expect(() => getDomainGroup("")).not.toThrow();
|
||||
expect(() => getDomainGroup(" ")).not.toThrow();
|
||||
});
|
||||
|
||||
it("returns fallback for unparseable input", { timeout: 5000 }, () => {
|
||||
const result = getDomainGroup("not-a-valid-tld");
|
||||
expect(typeof result).toBe("string");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,215 @@
|
||||
/**
|
||||
* Domain normalization functions using tldts for public suffix resolution.
|
||||
* Provides URL and domain normalization, subdomain stripping, and alias resolution.
|
||||
*/
|
||||
import { parse } from "tldts";
|
||||
import {
|
||||
DOMAIN_NORMALIZER_CODES,
|
||||
DomainNormalizerError,
|
||||
} from "../../errors/domain-normalizer-error.js";
|
||||
import type {
|
||||
NormalizedDomain,
|
||||
NormalizerConfig,
|
||||
} from "../../types/domain-normalizer.types.js";
|
||||
import { getCanonicalDomain } from "../aliases/aliases.js";
|
||||
|
||||
const MOBILE_PREFIXES = ["amp.", "lite.", "mobile.", "m."];
|
||||
|
||||
const DEFAULT_CONFIG: Required<NormalizerConfig> = {
|
||||
stripWww: true,
|
||||
stripMobile: true,
|
||||
toLowercase: true,
|
||||
resolveAliases: true,
|
||||
};
|
||||
|
||||
function mergeConfig(config?: NormalizerConfig): Required<NormalizerConfig> {
|
||||
if (!config) return DEFAULT_CONFIG;
|
||||
return {
|
||||
stripWww: config.stripWww ?? DEFAULT_CONFIG.stripWww,
|
||||
stripMobile: config.stripMobile ?? DEFAULT_CONFIG.stripMobile,
|
||||
toLowercase: config.toLowercase ?? DEFAULT_CONFIG.toLowercase,
|
||||
resolveAliases: config.resolveAliases ?? DEFAULT_CONFIG.resolveAliases,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips www and/or mobile prefixes from a full domain string.
|
||||
* Returns the stripped domain and any remaining subdomain.
|
||||
*/
|
||||
function stripPrefixes(
|
||||
fullDomain: string,
|
||||
cfg: Required<NormalizerConfig>
|
||||
): { stripped: string; remainingSubdomain: string | undefined } {
|
||||
let working = fullDomain;
|
||||
let changed = true;
|
||||
|
||||
while (changed) {
|
||||
changed = false;
|
||||
if (cfg.stripWww && working.startsWith("www.")) {
|
||||
working = working.slice(4);
|
||||
changed = true;
|
||||
}
|
||||
if (cfg.stripMobile) {
|
||||
for (const prefix of MOBILE_PREFIXES) {
|
||||
if (working.startsWith(prefix)) {
|
||||
working = working.slice(prefix.length);
|
||||
changed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse the stripped domain to find remaining subdomain
|
||||
const parsed = parse(working, { allowPrivateDomains: true });
|
||||
const remainingSubdomain =
|
||||
parsed.subdomain && parsed.subdomain.length > 0
|
||||
? parsed.subdomain
|
||||
: undefined;
|
||||
|
||||
return { stripped: working, remainingSubdomain };
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a domain name string to its canonical registrable domain.
|
||||
*
|
||||
* @param input - A domain name (e.g. "m.youtube.com", "YOUTUBE.COM")
|
||||
* @param config - Optional configuration overrides
|
||||
* @returns A NormalizedDomain result object
|
||||
* @throws DomainNormalizerError with INVALID_INPUT if input is empty or whitespace-only
|
||||
* @throws DomainNormalizerError with PARSE_ERROR if tldts cannot parse a registrable domain
|
||||
*/
|
||||
export function normalizeDomain(
|
||||
input: string,
|
||||
config?: NormalizerConfig
|
||||
): NormalizedDomain {
|
||||
if (!input || input.trim().length === 0) {
|
||||
throw new DomainNormalizerError(
|
||||
"Domain input must not be empty",
|
||||
DOMAIN_NORMALIZER_CODES.INVALID_INPUT
|
||||
);
|
||||
}
|
||||
|
||||
const cfg = mergeConfig(config);
|
||||
const original = input;
|
||||
|
||||
// Lowercase before parsing so tldts lookup is reliable
|
||||
const working = cfg.toLowercase ? input.trim().toLowerCase() : input.trim();
|
||||
|
||||
// Apply alias resolution first (before stripping prefixes)
|
||||
const aliasResolved = cfg.resolveAliases
|
||||
? getCanonicalDomain(working)
|
||||
: working;
|
||||
|
||||
// Parse with tldts
|
||||
const parsed = parse(aliasResolved, { allowPrivateDomains: true });
|
||||
|
||||
if (!parsed.domain) {
|
||||
throw new DomainNormalizerError(
|
||||
`Could not determine registrable domain for: ${input}`,
|
||||
DOMAIN_NORMALIZER_CODES.PARSE_ERROR
|
||||
);
|
||||
}
|
||||
|
||||
const registrableDomain = parsed.domain;
|
||||
|
||||
// Now strip prefixes from the alias-resolved domain for the normalized output
|
||||
const { remainingSubdomain } = stripPrefixes(aliasResolved, cfg);
|
||||
|
||||
const normalized = registrableDomain;
|
||||
|
||||
// Build result — tldts always provides publicSuffix and isPrivate when domain is found
|
||||
const result: NormalizedDomain = {
|
||||
original,
|
||||
normalized,
|
||||
registrableDomain,
|
||||
// publicSuffix is always present when domain is non-null
|
||||
publicSuffix: parsed.publicSuffix as string,
|
||||
// isPrivate is always a boolean when domain is non-null
|
||||
isPrivateDomain: parsed.isPrivate as boolean,
|
||||
};
|
||||
if (remainingSubdomain !== undefined) result.subdomain = remainingSubdomain;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a URL by first extracting the hostname, then normalizing the domain.
|
||||
*
|
||||
* @param url - A full URL string (e.g. "https://m.youtube.com/watch?v=...")
|
||||
* @param config - Optional configuration overrides
|
||||
* @returns A NormalizedDomain result with the original URL preserved
|
||||
* @throws DomainNormalizerError with INVALID_INPUT if URL is empty
|
||||
* @throws DomainNormalizerError with PARSE_ERROR if URL hostname cannot be parsed
|
||||
*/
|
||||
export function normalizeUrl(
|
||||
url: string,
|
||||
config?: NormalizerConfig
|
||||
): NormalizedDomain {
|
||||
if (!url || url.trim().length === 0) {
|
||||
throw new DomainNormalizerError(
|
||||
"URL input must not be empty",
|
||||
DOMAIN_NORMALIZER_CODES.INVALID_INPUT
|
||||
);
|
||||
}
|
||||
|
||||
let hostname: string;
|
||||
try {
|
||||
// Add protocol if missing so URL constructor can parse it
|
||||
const withProtocol = url.includes("://") ? url : `https://${url}`;
|
||||
const parsed = new URL(withProtocol);
|
||||
hostname = parsed.hostname;
|
||||
} catch {
|
||||
// URL constructor failed; treat the whole input as a bare domain
|
||||
hostname = url.trim();
|
||||
}
|
||||
|
||||
const result = normalizeDomain(hostname, config);
|
||||
// Keep original URL as the original field
|
||||
return { ...result, original: url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the registrable domain (eTLD+1) for a given input, or null if parsing fails.
|
||||
*
|
||||
* @param input - A domain or URL string
|
||||
* @returns Registrable domain string (e.g. "youtube.com") or null
|
||||
*/
|
||||
export function getRegistrableDomain(input: string): string | null {
|
||||
if (!input || input.trim().length === 0) return null;
|
||||
const parsed = parse(input.toLowerCase().trim(), {
|
||||
allowPrivateDomains: true,
|
||||
});
|
||||
return parsed.domain ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the subdomain for a given input, or null if there is none.
|
||||
*
|
||||
* @param input - A domain or URL string
|
||||
* @returns Subdomain string (e.g. "m") or null
|
||||
*/
|
||||
export function getSubdomain(input: string): string | null {
|
||||
if (!input || input.trim().length === 0) return null;
|
||||
const parsed = parse(input.toLowerCase().trim(), {
|
||||
allowPrivateDomains: true,
|
||||
});
|
||||
return parsed.subdomain && parsed.subdomain.length > 0
|
||||
? parsed.subdomain
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the canonical registrable domain for grouping purposes.
|
||||
* Applies alias resolution and normalization, then returns the registrable domain.
|
||||
*
|
||||
* @param input - A domain string
|
||||
* @returns Canonical registrable domain for use as a grouping key
|
||||
*/
|
||||
export function getDomainGroup(input: string): string {
|
||||
try {
|
||||
const result = normalizeDomain(input);
|
||||
return result.registrableDomain;
|
||||
} catch {
|
||||
return input.toLowerCase().trim();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
# domain-normalizer-error
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides the stable error class and error-code registry for the `@suaveplan/domain-normalizer` package. All errors thrown by normalizer, matcher, and alias functions are instances of `DomainNormalizerError` with a stable machine-readable code string, enabling programmatic error handling without string matching on messages.
|
||||
|
||||
This module deliberately avoids depending on `@suaveplan/error` so that `domain-normalizer` stays a leaf package with no workspace runtime dependencies.
|
||||
|
||||
## Features
|
||||
|
||||
- `DOMAIN_NORMALIZER_CODES` — frozen const object with every stable code string
|
||||
- `DomainNormalizerError` — extends `Error`, adds `readonly code: string` and sets `name = 'DomainNormalizerError'`
|
||||
- `DomainNormalizerCode` — union type of all defined code strings for exhaustive switch guards
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { DomainNormalizerError, DOMAIN_NORMALIZER_CODES } from "@suaveplan/domain-normalizer/errors";
|
||||
|
||||
try {
|
||||
// any domain-normalizer call
|
||||
} catch (err) {
|
||||
if (err instanceof DomainNormalizerError) {
|
||||
if (err.code === DOMAIN_NORMALIZER_CODES.INVALID_INPUT) {
|
||||
console.error("Bad input:", err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { DomainNormalizerError, DOMAIN_NORMALIZER_CODES, type DomainNormalizerCode } from "@suaveplan/domain-normalizer/errors";
|
||||
|
||||
function handleError(err: unknown): string {
|
||||
if (!(err instanceof DomainNormalizerError)) return "unknown error";
|
||||
|
||||
const code: DomainNormalizerCode = err.code as DomainNormalizerCode;
|
||||
switch (code) {
|
||||
case DOMAIN_NORMALIZER_CODES.INVALID_INPUT:
|
||||
return `Invalid domain input: ${err.message}`;
|
||||
case DOMAIN_NORMALIZER_CODES.PARSE_ERROR:
|
||||
return `Failed to parse domain: ${err.message}`;
|
||||
default:
|
||||
return `Domain normalizer error [${code}]: ${err.message}`;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `DOMAIN_NORMALIZER_CODES`
|
||||
|
||||
```ts
|
||||
const DOMAIN_NORMALIZER_CODES: {
|
||||
readonly INVALID_INPUT: "DOMAIN_NORMALIZER.INVALID_INPUT";
|
||||
readonly PARSE_ERROR: "DOMAIN_NORMALIZER.PARSE_ERROR";
|
||||
}
|
||||
```
|
||||
|
||||
Stable code strings for every error produced by this package. Use these constants for programmatic error matching rather than message strings.
|
||||
|
||||
### `DomainNormalizerError`
|
||||
|
||||
```ts
|
||||
class DomainNormalizerError extends Error {
|
||||
readonly code: string;
|
||||
constructor(message: string, code: string): DomainNormalizerError;
|
||||
}
|
||||
```
|
||||
|
||||
- `message` — human-readable description of the error condition
|
||||
- `code` — machine-readable code from `DOMAIN_NORMALIZER_CODES`
|
||||
- `name` — always `"DomainNormalizerError"` for `instanceof`-free detection
|
||||
|
||||
### `DomainNormalizerCode`
|
||||
|
||||
```ts
|
||||
type DomainNormalizerCode = "DOMAIN_NORMALIZER.INVALID_INPUT" | "DOMAIN_NORMALIZER.PARSE_ERROR";
|
||||
```
|
||||
|
||||
Union type derived from `DOMAIN_NORMALIZER_CODES` values, suitable for exhaustive switch statements.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The class intentionally calls `super(message)` before setting `this.name` to work correctly with all JS runtimes.
|
||||
- No `Error.captureStackTrace` call is made — this keeps the module universal (works in browsers without V8).
|
||||
- The `code` field is `readonly` to prevent accidental mutation after construction.
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Error codes and error class for the domain-normalizer package.
|
||||
* Uses a local error class to avoid runtime dependencies on @suaveplan/error.
|
||||
*/
|
||||
|
||||
export const DOMAIN_NORMALIZER_CODES = {
|
||||
INVALID_INPUT: "DOMAIN_NORMALIZER.INVALID_INPUT",
|
||||
PARSE_ERROR: "DOMAIN_NORMALIZER.PARSE_ERROR",
|
||||
} as const;
|
||||
|
||||
export type DomainNormalizerCode =
|
||||
(typeof DOMAIN_NORMALIZER_CODES)[keyof typeof DOMAIN_NORMALIZER_CODES];
|
||||
|
||||
/**
|
||||
* Error thrown by domain-normalizer operations.
|
||||
*/
|
||||
export class DomainNormalizerError extends Error {
|
||||
readonly code: string;
|
||||
|
||||
constructor(message: string, code: string) {
|
||||
super(message);
|
||||
this.name = "DomainNormalizerError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
DOMAIN_NORMALIZER_CODES,
|
||||
type DomainNormalizerCode,
|
||||
DomainNormalizerError,
|
||||
} from "./domain-normalizer-error";
|
||||
38
packages/communication/emails/domain-normalizer/src/index.ts
Normal file
38
packages/communication/emails/domain-normalizer/src/index.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
// Errors
|
||||
|
||||
// Aliases
|
||||
export {
|
||||
BUILT_IN_ALIASES,
|
||||
getAliasGroup,
|
||||
getCanonicalDomain,
|
||||
isKnownAlias,
|
||||
resetAliasCache,
|
||||
} from "./core/aliases/aliases.js";
|
||||
// Matcher
|
||||
export {
|
||||
computeSimilarity,
|
||||
DEFAULT_MATCH_THRESHOLD,
|
||||
findMatchingDomains,
|
||||
groupDomains,
|
||||
levenshteinDistance,
|
||||
matchDomains,
|
||||
} from "./core/matcher/matcher.js";
|
||||
// Normalizer
|
||||
export {
|
||||
getDomainGroup,
|
||||
getRegistrableDomain,
|
||||
getSubdomain,
|
||||
normalizeDomain,
|
||||
normalizeUrl,
|
||||
} from "./core/normalizer/normalizer.js";
|
||||
export {
|
||||
DOMAIN_NORMALIZER_CODES,
|
||||
type DomainNormalizerCode,
|
||||
DomainNormalizerError,
|
||||
} from "./errors/domain-normalizer-error.js";
|
||||
// Types
|
||||
export type {
|
||||
MatchResult,
|
||||
NormalizedDomain,
|
||||
NormalizerConfig,
|
||||
} from "./types/domain-normalizer.types.js";
|
||||
@@ -0,0 +1,86 @@
|
||||
# domain-normalizer types
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Defines the shared TypeScript interfaces consumed by every module in `@suaveplan/domain-normalizer`. Centralising type declarations here prevents circular imports and makes the public API shape visible in one place.
|
||||
|
||||
These types are shipped as part of the package's type declarations and re-exported from the root barrel so consumers can import them without importing any runtime code.
|
||||
|
||||
## Features
|
||||
|
||||
- `NormalizerConfig` — configuration object controlling stripping and alias-resolution behaviour
|
||||
- `NormalizedDomain` — rich result object returned by `normalizeDomain` and `normalizeUrl`
|
||||
- `MatchResult` — comparison result from `matchDomains` with a numeric similarity score and boolean threshold flag
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type { NormalizerConfig, NormalizedDomain } from "@suaveplan/domain-normalizer";
|
||||
|
||||
const config: NormalizerConfig = { stripWww: true, stripMobile: true };
|
||||
|
||||
function process(result: NormalizedDomain): void {
|
||||
console.log(result.registrableDomain, result.publicSuffix);
|
||||
}
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { MatchResult, NormalizerConfig } from "@suaveplan/domain-normalizer";
|
||||
import { matchDomains } from "@suaveplan/domain-normalizer/matcher";
|
||||
|
||||
function deduplicate(domains: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
for (const d of domains) {
|
||||
let isDup = false;
|
||||
for (const u of unique) {
|
||||
const result: MatchResult = matchDomains(d, u);
|
||||
if (result.isMatch) { isDup = true; break; }
|
||||
}
|
||||
if (!isDup) { unique.push(d); seen.add(d); }
|
||||
}
|
||||
return unique;
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `NormalizerConfig`
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `stripWww` | `boolean?` | `true` | Remove `www.` prefix |
|
||||
| `stripMobile` | `boolean?` | `true` | Remove `m.`, `mobile.`, `lite.`, `amp.` prefixes |
|
||||
| `toLowercase` | `boolean?` | `true` | Lowercase the result |
|
||||
| `resolveAliases` | `boolean?` | `true` | Apply built-in alias map |
|
||||
|
||||
### `NormalizedDomain`
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `original` | `string` | Unmodified input |
|
||||
| `normalized` | `string` | Canonical registrable domain |
|
||||
| `subdomain` | `string?` | Remaining subdomain after stripping |
|
||||
| `registrableDomain` | `string` | eTLD+1 (e.g. `"youtube.com"`) |
|
||||
| `publicSuffix` | `string?` | eTLD (e.g. `"com"`) |
|
||||
| `isPrivateDomain` | `boolean?` | From public suffix list private section |
|
||||
|
||||
### `MatchResult`
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `domain1` | `string` | First input |
|
||||
| `domain2` | `string` | Second input |
|
||||
| `similarity` | `number` | Score in `[0, 1]` |
|
||||
| `isMatch` | `boolean` | `true` when similarity ≥ default threshold (0.8) |
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- All interfaces use optional (`?`) fields rather than union with `undefined` for better JSON round-trip compatibility.
|
||||
- `NormalizerConfig` fields are all optional so callers can pass a partial config; the normalizer merges with defaults internally.
|
||||
- `MatchResult.similarity` is always in the range `[0, 1]` inclusive — callers can apply their own threshold by comparing to `similarity` directly.
|
||||
@@ -0,0 +1,88 @@
|
||||
# Domain Normalizer Types
|
||||
|
||||
## Purpose
|
||||
|
||||
Shared TypeScript type definitions for the `@suaveplan/domain-normalizer` package. These interfaces describe the configuration surface and output shapes used by every normalizer, alias, and matcher function. Centralising them here prevents circular imports between modules and provides a single place to extend the public API contract.
|
||||
|
||||
## Features
|
||||
|
||||
- `NormalizerConfig` — four boolean flags that control how raw input is transformed before canonicalization
|
||||
- `NormalizedDomain` — rich output record containing the original input, canonical form, subdomain, registrable domain (eTLD+1), public suffix, and private-domain flag
|
||||
- `MatchResult` — pair-wise comparison result with a `[0, 1]` similarity score and a boolean threshold indicator
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type { NormalizerConfig, NormalizedDomain } from '@suaveplan/domain-normalizer';
|
||||
|
||||
const config: NormalizerConfig = {
|
||||
stripWww: true,
|
||||
stripMobile: true,
|
||||
toLowercase: true,
|
||||
resolveAliases: true,
|
||||
};
|
||||
|
||||
// NormalizedDomain returned by normalizeDomain()
|
||||
const result: NormalizedDomain = {
|
||||
original: 'https://m.YouTube.com/watch?v=dQw4w9WgXcQ',
|
||||
normalized: 'youtube.com',
|
||||
registrableDomain: 'youtube.com',
|
||||
publicSuffix: 'com',
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { MatchResult, NormalizerConfig } from '@suaveplan/domain-normalizer';
|
||||
import { matchDomains, normalizeDomain } from '@suaveplan/domain-normalizer';
|
||||
|
||||
// Use a stricter config that keeps subdomains intact
|
||||
const strictConfig: NormalizerConfig = {
|
||||
stripWww: false,
|
||||
stripMobile: false,
|
||||
toLowercase: true,
|
||||
resolveAliases: false,
|
||||
};
|
||||
|
||||
const a = normalizeDomain('blog.example.com', strictConfig);
|
||||
const b = normalizeDomain('BLOG.EXAMPLE.COM', strictConfig);
|
||||
|
||||
const result: MatchResult = matchDomains(a.normalized, b.normalized);
|
||||
// result.similarity === 1, result.isMatch === true
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `NormalizerConfig`
|
||||
|
||||
| Field | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `stripWww` | `boolean?` | `true` | Remove `www.` prefix |
|
||||
| `stripMobile` | `boolean?` | `true` | Remove `m.`, `mobile.`, `lite.`, `amp.` prefixes |
|
||||
| `toLowercase` | `boolean?` | `true` | Lowercase the entire domain |
|
||||
| `resolveAliases` | `boolean?` | `true` | Map known alias subdomains to their canonical registrable domain |
|
||||
|
||||
### `NormalizedDomain`
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `original` | `string` | The raw input passed to the normalizer |
|
||||
| `normalized` | `string` | The canonical domain after all transformations |
|
||||
| `subdomain` | `string?` | Remaining subdomain after stripping www/mobile |
|
||||
| `registrableDomain` | `string` | The eTLD+1 (e.g. `youtube.com`) |
|
||||
| `publicSuffix` | `string?` | The effective TLD (e.g. `com`, `co.uk`) |
|
||||
| `isPrivateDomain` | `boolean?` | `true` for private registries per public suffix list |
|
||||
|
||||
### `MatchResult`
|
||||
|
||||
| Field | Type | Description |
|
||||
|---|---|---|
|
||||
| `domain1` | `string` | First domain in the comparison |
|
||||
| `domain2` | `string` | Second domain in the comparison |
|
||||
| `similarity` | `number` | Levenshtein-normalized score in `[0, 1]` |
|
||||
| `isMatch` | `boolean` | `true` when `similarity >= 0.8` (default threshold) |
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
All interfaces are exported from `src/index.ts` and re-exported under the `./types` subpath export. The types file contains no runtime code and therefore has no test file, but it is included in `docs:check` coverage because it forms part of the package's documented public API surface.
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Shared type definitions for @suaveplan/domain-normalizer.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Configuration options for domain normalization operations.
|
||||
*/
|
||||
export interface NormalizerConfig {
|
||||
/**
|
||||
* Strip "www." prefix from domains. Default: true.
|
||||
*/
|
||||
stripWww?: boolean;
|
||||
/**
|
||||
* Strip mobile subdomains (m., mobile., lite., amp.). Default: true.
|
||||
*/
|
||||
stripMobile?: boolean;
|
||||
/**
|
||||
* Convert domain to lowercase. Default: true.
|
||||
*/
|
||||
toLowercase?: boolean;
|
||||
/**
|
||||
* Resolve known aliases (e.g. m.youtube.com → youtube.com). Default: true.
|
||||
*/
|
||||
resolveAliases?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of normalizing a domain name.
|
||||
*/
|
||||
export interface NormalizedDomain {
|
||||
/** The original input string. */
|
||||
original: string;
|
||||
/** The canonical normalized domain. */
|
||||
normalized: string;
|
||||
/** Subdomain portion, if any (after stripping mobile/www prefixes). */
|
||||
subdomain?: string;
|
||||
/** The registrable domain (eTLD+1), e.g. "youtube.com". */
|
||||
registrableDomain: string;
|
||||
/** The public suffix / eTLD, e.g. "com". */
|
||||
publicSuffix?: string;
|
||||
/** Whether this is a private domain (per public suffix list). */
|
||||
isPrivateDomain?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result of a domain match comparison.
|
||||
*/
|
||||
export interface MatchResult {
|
||||
/** First domain compared. */
|
||||
domain1: string;
|
||||
/** Second domain compared. */
|
||||
domain2: string;
|
||||
/** Similarity score between 0 (no match) and 1 (exact match). */
|
||||
similarity: number;
|
||||
/** Whether the similarity meets the default threshold. */
|
||||
isMatch: boolean;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export type {
|
||||
MatchResult,
|
||||
NormalizedDomain,
|
||||
NormalizerConfig,
|
||||
} from "./domain-normalizer.types";
|
||||
@@ -0,0 +1,38 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("dep-graph", () => {
|
||||
it("has no @suaveplan/* runtime deps", { timeout: 5000 }, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as {
|
||||
dependencies?: Record<string, string>;
|
||||
peerDependencies?: Record<string, string>;
|
||||
};
|
||||
const runtimeScoped = Object.keys(pkg.dependencies ?? {})
|
||||
.filter((name) => name.startsWith("@suaveplan/"))
|
||||
.sort();
|
||||
expect(runtimeScoped).toEqual([]);
|
||||
|
||||
const peerScoped = Object.keys(pkg.peerDependencies ?? {}).filter(
|
||||
(name) => name.startsWith("@suaveplan/")
|
||||
);
|
||||
expect(peerScoped).toEqual([]);
|
||||
});
|
||||
|
||||
it("runtime deps are exactly the expected set (sorted)", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as { dependencies?: Record<string, string> };
|
||||
const runtime = Object.keys(pkg.dependencies ?? {}).sort();
|
||||
expect(runtime).toEqual([
|
||||
"@opentelemetry/api",
|
||||
"tldts",
|
||||
"tslib",
|
||||
"zod",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
import { readdirSync, readFileSync, statSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
const SRC = join(__dirname, "..", "src");
|
||||
|
||||
const BANNED: ReadonlyArray<RegExp> = [
|
||||
/\bnode:fs\b/,
|
||||
/\bnode:path\b/,
|
||||
/\bnode:url\b/,
|
||||
/\bprocess\./,
|
||||
/\bBuffer\b/,
|
||||
/\bdocument\b/,
|
||||
/\bwindow\b/,
|
||||
];
|
||||
|
||||
function* walk(dir: string): Generator<string> {
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
const st = statSync(full);
|
||||
if (st.isDirectory()) yield* walk(full);
|
||||
else if (full.endsWith(".ts") && !full.endsWith(".test.ts")) yield full;
|
||||
}
|
||||
}
|
||||
|
||||
describe("no-disallowed", () => {
|
||||
it("src/ does not reference Node or browser built-ins", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const hits: string[] = [];
|
||||
for (const file of walk(SRC)) {
|
||||
const content = readFileSync(file, "utf8");
|
||||
for (const pattern of BANNED) {
|
||||
if (pattern.test(content)) {
|
||||
hits.push(`${file} matched ${pattern}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(hits).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
export {};
|
||||
@@ -0,0 +1,20 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("tree-shake", () => {
|
||||
it("sideEffects is false", { timeout: 5000 }, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as { sideEffects?: boolean };
|
||||
expect(pkg.sideEffects).toBe(false);
|
||||
});
|
||||
|
||||
it("exports has at least 5 entries", { timeout: 5000 }, () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(__dirname, "..", "package.json"), "utf8")
|
||||
) as { exports?: Record<string, unknown> };
|
||||
expect(pkg.exports).toBeDefined();
|
||||
expect(Object.keys(pkg.exports ?? {}).length).toBeGreaterThanOrEqual(5);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/domain-normalizer",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: ["tldts", "tslib", "zod", "@opentelemetry/api", /^node:/],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: ["src/__benchmarks__/**", "dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.bench.ts",
|
||||
"src/**/*.schemas.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
},
|
||||
testTimeout: 5000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user