Initial Commit
This commit is contained in:
13
packages/persistence/search/README.md
Normal file
13
packages/persistence/search/README.md
Normal file
@@ -0,0 +1,13 @@
|
||||
# `search`
|
||||
|
||||
Full-text search, BM25, fuzzy matching, and faceted search.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Packages in `search`
|
||||
|
||||
1 package.
|
||||
|
||||
| Package | Description |
|
||||
|---|---|
|
||||
| [`@suaveplan/search`](./search/README.md) | Full-text and fuzzy search primitives with in-memory BM25 engine, trigram fuzzy matching, faceted search, and pagination support. |
|
||||
<!-- end:auto-docs -->
|
||||
1
packages/persistence/search/_intro.md
Normal file
1
packages/persistence/search/_intro.md
Normal file
@@ -0,0 +1 @@
|
||||
Full-text search, BM25, fuzzy matching, and faceted search.
|
||||
54
packages/persistence/search/search/README.md
Normal file
54
packages/persistence/search/search/README.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# @suaveplan/search - Full-text and fuzzy search primitives with in-memory BM25 engine, trigram fuzzy matching, faceted search, and pagination support
|
||||
|
||||
## Summary
|
||||
|
||||
Full-text and fuzzy search primitives with in-memory BM25 engine, trigram fuzzy matching, faceted search, and pagination support.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [memory-engine](./src/engine/memory-engine/memory-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [memory-engine](./src/engine/memory-engine/memory-engine.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [errors](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [search-error](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [search.schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_ <!-- end:auto-docs -->
|
||||
<!-- end:auto-docs --> - [memory-engine](./src/engine/memory-engine/memory-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [search-error](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [search.schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_- [memory-engine](./src/engine/memory-engine/memory-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [search-error](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [search.schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_- [memory-engine](./src/engine/memory-engine/memory-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [search-error](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [search.schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_- [memory-engine](./src/engine/memory-engine/memory-engine.md) — <!-- begin:auto-docs --> _(no submodules)_- [errors](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_ - [search-error](./src/errors/search-error.md) — <!-- begin:auto-docs --> _(no submodules)_- [schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_ - [search.schemas](./src/schemas/search.schemas.md) — <!-- begin:auto-docs --> _(no submodules)_- [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [scorer](./src/text/scorer/scorer.md) — <!-- begin:auto-docs --> _(no submodules)_- [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_ - [tokenizer](./src/text/tokenizer/tokenizer.md) — <!-- begin:auto-docs --> _(no submodules)_- [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_ - [trigram](./src/text/trigram/trigram.md) — <!-- begin:auto-docs --> _(no submodules)_
|
||||
## Modules
|
||||
|
||||
- [MemorySearchEngine](src/engine/memory-engine/memory-engine.md) — in-memory BM25 search engine
|
||||
- [Tokenizer](src/text/tokenizer/tokenizer.md) — text tokenization and stemming
|
||||
- [Trigram](src/text/trigram/trigram.md) — trigram-based fuzzy similarity
|
||||
- [Scorer](src/text/scorer/scorer.md) — BM25 and TF-IDF scoring functions
|
||||
- [SearchError](src/errors/search-error.md) — structured errors with stable codes
|
||||
- [Schemas](src/schemas/search.schemas.md) — Zod schemas for all query types
|
||||
- [Types](src/types/search.types.md) — shared TypeScript interfaces
|
||||
|
||||
## Quick Start
|
||||
|
||||
```ts
|
||||
import { createMemorySearchEngine } from "@suaveplan/search";
|
||||
|
||||
interface Product { id: string; name: string; price: number }
|
||||
|
||||
const engine = createMemorySearchEngine<Product>({
|
||||
fields: [
|
||||
{ name: "name", type: "text", searchable: true },
|
||||
{ name: "price", type: "number", sortable: true },
|
||||
],
|
||||
});
|
||||
|
||||
await engine.index([
|
||||
{ id: "1", name: "Laptop Pro", price: 999 },
|
||||
{ id: "2", name: "Laptop Air", price: 799 },
|
||||
]);
|
||||
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
console.log(result.hits.length); // 2
|
||||
```
|
||||
76
packages/persistence/search/search/package.json
Normal file
76
packages/persistence/search/search/package.json
Normal file
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"name": "@suaveplan/search",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Full-text and fuzzy search primitives with in-memory BM25 engine, trigram fuzzy matching, faceted search, and pagination support",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
|
||||
"require": { "types": "./dist/index.d.ts", "default": "./dist/index.js" }
|
||||
},
|
||||
"./errors/search-error": {
|
||||
"import": { "types": "./dist/errors/search-error.d.ts", "default": "./dist/errors/search-error.js" },
|
||||
"require": { "types": "./dist/errors/search-error.d.ts", "default": "./dist/errors/search-error.js" }
|
||||
},
|
||||
"./engine/memory-engine/memory-engine": {
|
||||
"import": { "types": "./dist/engine/memory-engine/memory-engine.d.ts", "default": "./dist/engine/memory-engine/memory-engine.js" },
|
||||
"require": { "types": "./dist/engine/memory-engine/memory-engine.d.ts", "default": "./dist/engine/memory-engine/memory-engine.js" }
|
||||
},
|
||||
"./text/tokenizer/tokenizer": {
|
||||
"import": { "types": "./dist/text/tokenizer/tokenizer.d.ts", "default": "./dist/text/tokenizer/tokenizer.js" },
|
||||
"require": { "types": "./dist/text/tokenizer/tokenizer.d.ts", "default": "./dist/text/tokenizer/tokenizer.js" }
|
||||
},
|
||||
"./text/trigram/trigram": {
|
||||
"import": { "types": "./dist/text/trigram/trigram.d.ts", "default": "./dist/text/trigram/trigram.js" },
|
||||
"require": { "types": "./dist/text/trigram/trigram.d.ts", "default": "./dist/text/trigram/trigram.js" }
|
||||
},
|
||||
"./text/scorer/scorer": {
|
||||
"import": { "types": "./dist/text/scorer/scorer.d.ts", "default": "./dist/text/scorer/scorer.js" },
|
||||
"require": { "types": "./dist/text/scorer/scorer.d.ts", "default": "./dist/text/scorer/scorer.js" }
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"bench": "bun run src/__benchmarks__/search.bench.ts",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"@suaveplan/pagination": "workspace:^",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms",
|
||||
"@opentelemetry/api": "catalog:observability"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"fast-check": "catalog:testing",
|
||||
"mitata": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": ["dist", "src", "README.md"]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
/** Default number of results per page. */
|
||||
export const DEFAULT_PAGE_SIZE = 10;
|
||||
/** Default BM25 k1 parameter. */
|
||||
export const BM25_K1 = 1.2;
|
||||
/** Default BM25 b parameter. */
|
||||
export const BM25_B = 0.75;
|
||||
/** Default fuzzy similarity threshold. */
|
||||
export const DEFAULT_FUZZY_THRESHOLD = 0.3;
|
||||
@@ -0,0 +1,4 @@
|
||||
export {
|
||||
createMemorySearchEngine,
|
||||
MemorySearchEngine,
|
||||
} from "./memory-engine.js";
|
||||
@@ -0,0 +1,85 @@
|
||||
# memory-engine
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides `createMemorySearchEngine`, an in-memory implementation of `SearchEngine` backed by a term-frequency inverted index and BM25 scoring. Designed for single-process use, testing, and small-to-medium datasets (up to ~100k documents). Supports full-text search, structured filters, fuzzy trigram matching, faceted aggregations, and keyword highlighting.
|
||||
|
||||
## Features
|
||||
|
||||
- Inverted index built from tokenized searchable fields
|
||||
- BM25 scoring (k1=1.2, b=0.75) for relevance ranking
|
||||
- Trigram fuzzy matching with configurable threshold
|
||||
- Faceted search: bucket counts for `facetable` fields
|
||||
- Highlight extraction with configurable tag (default `<mark>`)
|
||||
- Structured filters: `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `in`, `contains`
|
||||
- Multi-field sorting with direction
|
||||
- Cursor and offset pagination
|
||||
- `suggest(prefix)` for autocomplete from indexed terms
|
||||
- OTel spans on `index`, `search`, and `suggest`
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { createMemorySearchEngine } from "@suaveplan/search";
|
||||
|
||||
interface Product { id: string; name: string; price: number }
|
||||
|
||||
const engine = createMemorySearchEngine<Product>({
|
||||
fields: [
|
||||
{ name: "name", type: "text", searchable: true },
|
||||
{ name: "price", type: "number", sortable: true },
|
||||
],
|
||||
});
|
||||
|
||||
await engine.index([{ id: "1", name: "Laptop Pro", price: 999 }]);
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
const engine = createMemorySearchEngine<Product>({
|
||||
fields: [
|
||||
{ name: "name", type: "text", searchable: true, facetable: false },
|
||||
{ name: "category", type: "keyword", facetable: true },
|
||||
],
|
||||
fuzzyThreshold: 0.4,
|
||||
stemming: true,
|
||||
});
|
||||
|
||||
await engine.index(products);
|
||||
|
||||
const result = await engine.search({
|
||||
text: "laptops",
|
||||
filters: [{ field: "price", op: "lte", value: 1000 }],
|
||||
sort: [{ field: "price", direction: "asc" }],
|
||||
pagination: { type: "offset", offset: 0, limit: 20 },
|
||||
fuzzy: true,
|
||||
});
|
||||
|
||||
console.log(result.facets?.["category"]);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `createMemorySearchEngine<T>(options)`
|
||||
|
||||
- `options.fields: FieldConfig<T>[]` — field schema with type and searchability flags
|
||||
- `options.stemming?: boolean` — enable simple English stemming
|
||||
- `options.stopWords?: string[]` — custom stop-word list
|
||||
- `options.fuzzyThreshold?: number` — trigram similarity threshold for fuzzy (default 0.3)
|
||||
|
||||
### `SearchEngine<T>`
|
||||
|
||||
- `index(documents, options?)` — adds documents; replaces if `replace !== false`
|
||||
- `search(query, options?)` — returns `SearchResult<T>` with hits, total, facets
|
||||
- `suggest(prefix, options?)` — returns terms from inverted index starting with prefix
|
||||
- `remove(ids)` — removes documents by ID
|
||||
- `clear()` — empties the entire index
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The inverted index maps `term → Map<docId, tf>`. Field lengths per document are tracked for BM25 length normalization. Fuzzy matching runs after BM25 scoring and assigns a 50% score weight to fuzzy hits so they rank below exact matches.
|
||||
@@ -0,0 +1,569 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
createMemorySearchEngine,
|
||||
MemorySearchEngine,
|
||||
} from "./memory-engine.js";
|
||||
|
||||
interface Doc {
|
||||
id: string;
|
||||
title: string;
|
||||
category: string;
|
||||
price: number;
|
||||
active: boolean;
|
||||
}
|
||||
|
||||
function makeEngine() {
|
||||
return createMemorySearchEngine<Doc>({
|
||||
fields: [
|
||||
{ name: "title", type: "text", searchable: true, facetable: false },
|
||||
{ name: "category", type: "keyword", facetable: true },
|
||||
{ name: "price", type: "number", sortable: true },
|
||||
{ name: "active", type: "boolean" },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const docs: Doc[] = [
|
||||
{
|
||||
id: "1",
|
||||
title: "Laptop Pro",
|
||||
category: "electronics",
|
||||
price: 999,
|
||||
active: true,
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
title: "Laptop Air",
|
||||
category: "electronics",
|
||||
price: 799,
|
||||
active: true,
|
||||
},
|
||||
{
|
||||
id: "3",
|
||||
title: "Running Shoes",
|
||||
category: "sports",
|
||||
price: 120,
|
||||
active: false,
|
||||
},
|
||||
{
|
||||
id: "4",
|
||||
title: "Coffee Mug",
|
||||
category: "kitchen",
|
||||
price: 15,
|
||||
active: true,
|
||||
},
|
||||
];
|
||||
|
||||
describe("MemorySearchEngine", () => {
|
||||
it("indexes and searches documents", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
expect(
|
||||
result.hits.every((h) =>
|
||||
h.document.title.toLowerCase().includes("laptop")
|
||||
)
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("returns total count", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({});
|
||||
expect(result.total).toBe(4);
|
||||
});
|
||||
|
||||
it("filters with eq operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "category", op: "eq", value: "electronics" }],
|
||||
});
|
||||
expect(result.total).toBe(2);
|
||||
});
|
||||
|
||||
it("filters with lte operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "price", op: "lte", value: 200 }],
|
||||
});
|
||||
expect(result.hits.every((h) => h.document.price <= 200)).toBe(true);
|
||||
});
|
||||
|
||||
it("filters with neq operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "category", op: "neq", value: "electronics" }],
|
||||
});
|
||||
expect(
|
||||
result.hits.every((h) => h.document.category !== "electronics")
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("filters with gt operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "price", op: "gt", value: 500 }],
|
||||
});
|
||||
expect(result.hits.every((h) => h.document.price > 500)).toBe(true);
|
||||
});
|
||||
|
||||
it("filters with gte operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "price", op: "gte", value: 799 }],
|
||||
});
|
||||
expect(result.hits.every((h) => h.document.price >= 799)).toBe(true);
|
||||
});
|
||||
|
||||
it("filters with lt operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "price", op: "lt", value: 200 }],
|
||||
});
|
||||
expect(result.hits.every((h) => h.document.price < 200)).toBe(true);
|
||||
});
|
||||
|
||||
it("filters with in operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [
|
||||
{
|
||||
field: "category",
|
||||
op: "in",
|
||||
value: ["electronics", "kitchen"],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.total).toBe(3);
|
||||
});
|
||||
|
||||
it("filters with contains operator", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "title", op: "contains", value: "Laptop" }],
|
||||
});
|
||||
expect(result.total).toBe(2);
|
||||
});
|
||||
|
||||
it("sorts by field ascending", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
sort: [{ field: "price", direction: "asc" }],
|
||||
});
|
||||
const prices = result.hits.map((h) => h.document.price);
|
||||
expect(prices).toEqual([...prices].sort((a, b) => a - b));
|
||||
});
|
||||
|
||||
it("sorts by field descending", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
sort: [{ field: "price", direction: "desc" }],
|
||||
});
|
||||
const prices = result.hits.map((h) => h.document.price);
|
||||
expect(prices).toEqual([...prices].sort((a, b) => b - a));
|
||||
});
|
||||
|
||||
it("paginates with offset", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
pagination: { type: "offset", offset: 2, limit: 2 },
|
||||
});
|
||||
expect(result.hits).toHaveLength(2);
|
||||
expect(result.total).toBe(4);
|
||||
});
|
||||
|
||||
it("faceted search returns counts", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({});
|
||||
expect(result.facets).toBeDefined();
|
||||
expect(result.facets?.["category"]).toBeDefined();
|
||||
const electronics = result.facets?.["category"]?.find(
|
||||
(b) => b.value === "electronics"
|
||||
);
|
||||
expect(electronics?.count).toBe(2);
|
||||
});
|
||||
|
||||
it("suggest returns matching terms", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const suggestions = await engine.suggest("lap");
|
||||
expect(suggestions.some((s) => s.startsWith("lap"))).toBe(true);
|
||||
});
|
||||
|
||||
it("suggest respects limit", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const suggestions = await engine.suggest("", { limit: 2 });
|
||||
expect(suggestions.length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("removes documents", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
await engine.remove(["1", "2"]);
|
||||
const result = await engine.search({});
|
||||
expect(result.total).toBe(2);
|
||||
});
|
||||
|
||||
it("clear empties index", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
await engine.clear();
|
||||
const result = await engine.search({});
|
||||
expect(result.total).toBe(0);
|
||||
});
|
||||
|
||||
it("fuzzy search finds approximate matches", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = createMemorySearchEngine<Doc>({
|
||||
fields: [{ name: "title", type: "text", searchable: true }],
|
||||
fuzzyThreshold: 0.1,
|
||||
});
|
||||
await engine.index([
|
||||
{
|
||||
id: "1",
|
||||
title: "Running Shoes",
|
||||
category: "sports",
|
||||
price: 100,
|
||||
active: true,
|
||||
},
|
||||
]);
|
||||
const result = await engine.search({ text: "runing", fuzzy: true });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("fuzzy with threshold object", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
text: "laptop",
|
||||
fuzzy: { threshold: 0.3 },
|
||||
});
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("replaces existing document on re-index", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index([docs[0]!]);
|
||||
await engine.index([{ ...docs[0]!, title: "Updated Title" }]);
|
||||
const result = await engine.search({ text: "updated" });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("handles non-string field in filter gracefully", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
// contains operator on non-string field returns false
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "price", op: "contains", value: "99" }],
|
||||
});
|
||||
expect(result.hits).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("handles in operator with non-array value", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
filters: [{ field: "category", op: "in", value: "electronics" }],
|
||||
});
|
||||
expect(result.hits).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("default pagination returns 10 results", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
const manyDocs = Array.from({ length: 20 }, (_, i) => ({
|
||||
id: `d${i}`,
|
||||
title: `Doc ${i}`,
|
||||
category: "test",
|
||||
price: i,
|
||||
active: true,
|
||||
}));
|
||||
await engine.index(manyDocs);
|
||||
const result = await engine.search({});
|
||||
expect(result.hits).toHaveLength(10);
|
||||
expect(result.total).toBe(20);
|
||||
});
|
||||
|
||||
it("cursor pagination returns correct page", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
pagination: { type: "cursor", limit: 2 },
|
||||
});
|
||||
expect(result.hits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns highlights for text matches", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index([docs[0]!]);
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
const hit = result.hits[0];
|
||||
expect(hit?.highlights?.["title"]).toBeDefined();
|
||||
});
|
||||
|
||||
it("highlight tag is configurable", { timeout: 30000 }, async () => {
|
||||
const engine = createMemorySearchEngine<Doc>({
|
||||
fields: [{ name: "title", type: "text", searchable: true }],
|
||||
highlightTag: "strong",
|
||||
});
|
||||
await engine.index([docs[0]!]);
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
const h = result.hits[0]?.highlights?.["title"];
|
||||
expect(h?.some((s) => s.includes("<strong>"))).toBe(true);
|
||||
});
|
||||
|
||||
it("query highlight tag overrides option", { timeout: 30000 }, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index([docs[0]!]);
|
||||
const result = await engine.search({
|
||||
text: "laptop",
|
||||
highlightTag: "em",
|
||||
});
|
||||
const h = result.hits[0]?.highlights?.["title"];
|
||||
expect(h?.some((s) => s.includes("<em>"))).toBe(true);
|
||||
});
|
||||
|
||||
it("no facets when no facetable fields", { timeout: 30000 }, async () => {
|
||||
const engine = createMemorySearchEngine<Doc>({
|
||||
fields: [{ name: "title", type: "text", searchable: true }],
|
||||
});
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({});
|
||||
expect(result.facets).toBeUndefined();
|
||||
});
|
||||
|
||||
it("sort comparator returns 0 when all sort fields are equal", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index([
|
||||
{
|
||||
id: "1",
|
||||
title: "Same Title",
|
||||
category: "a",
|
||||
price: 100,
|
||||
active: true,
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
title: "Same Title",
|
||||
category: "b",
|
||||
price: 100,
|
||||
active: true,
|
||||
},
|
||||
]);
|
||||
const result = await engine.search({
|
||||
sort: [{ field: "price", direction: "asc" }],
|
||||
});
|
||||
expect(result.hits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("sort comparator covers av > bv branch via 3-element insertion sort", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
// [150, 50, 100]: insertion sort compares (100, 50) → av=100 > bv=50
|
||||
await engine.index([
|
||||
{ id: "1", title: "High", category: "a", price: 150, active: true },
|
||||
{ id: "2", title: "Low", category: "a", price: 50, active: true },
|
||||
{ id: "3", title: "Mid", category: "a", price: 100, active: true },
|
||||
]);
|
||||
const result = await engine.search({
|
||||
sort: [{ field: "price", direction: "asc" }],
|
||||
});
|
||||
expect(result.hits[0]?.document.price).toBe(50);
|
||||
expect(result.hits[2]?.document.price).toBe(150);
|
||||
});
|
||||
|
||||
it("search wraps non-SearchError in catch block", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
await expect(
|
||||
engine.search({ text: "laptop" }, { signal: ac.signal })
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("search without options covers optional chain branches", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
// No options at all — covers options?.signal?.throwIfAborted optional chains
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("suggest without options uses default limit", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
// No options — covers limit ?? 10 branch
|
||||
const suggestions = await engine.suggest("la");
|
||||
expect(suggestions.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("engine with stemming and stopWords options covers conditional spreads", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = createMemorySearchEngine<Doc>({
|
||||
fields: [{ name: "title", type: "text", searchable: true }],
|
||||
stemming: true,
|
||||
stopWords: ["the", "a"],
|
||||
});
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({ text: "laptop" });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("non-string searchable field skipped in fuzzy computeFuzzyScore", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
// Use a field with type "text" but store a number to trigger non-string skip
|
||||
type MixedDoc = { id: string; title: string | number };
|
||||
const engine = createMemorySearchEngine<MixedDoc>({
|
||||
fields: [{ name: "title", type: "text", searchable: true }],
|
||||
fuzzyThreshold: 0.1,
|
||||
});
|
||||
await engine.index([{ id: "1", title: 42 }] as unknown as MixedDoc[]);
|
||||
const result = await engine.search({ text: "foo", fuzzy: true });
|
||||
expect(result.hits).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("non-string searchable field skipped in extractHighlights", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
// Doc with two text fields: one string, one non-string number.
|
||||
// The doc matches via "title", so extractHighlights is called on it.
|
||||
// When the "score" field (non-string value) is reached, continue is hit.
|
||||
type MixedDoc = { id: string; title: string; score: string | number };
|
||||
const engine = createMemorySearchEngine<MixedDoc>({
|
||||
fields: [
|
||||
{ name: "title", type: "text", searchable: true },
|
||||
{ name: "score", type: "text", searchable: true },
|
||||
],
|
||||
});
|
||||
await engine.index([{ id: "1", title: "hello", score: 42 }]);
|
||||
const result = await engine.search({ text: "hello" });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("cursor pagination offset branch is 0", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
const result = await engine.search({
|
||||
pagination: { type: "cursor", limit: 2 },
|
||||
});
|
||||
expect(result.hits).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("offset pagination without explicit offset defaults to 0", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
await engine.index(docs);
|
||||
// spec.offset ?? 0 branch: type is "offset" but offset field omitted
|
||||
const result = await engine.search({
|
||||
pagination: { type: "offset", limit: 2 } as {
|
||||
type: "offset";
|
||||
offset?: number;
|
||||
limit: number;
|
||||
},
|
||||
});
|
||||
expect(result.hits).toHaveLength(2);
|
||||
expect(result.total).toBe(4);
|
||||
});
|
||||
|
||||
it("facetable field with null value is skipped", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = createMemorySearchEngine<Doc>({
|
||||
fields: [
|
||||
{ name: "category", type: "keyword", facetable: true },
|
||||
{ name: "title", type: "text", searchable: true },
|
||||
],
|
||||
});
|
||||
await engine.index([
|
||||
{
|
||||
id: "1",
|
||||
title: "Doc",
|
||||
category: null as unknown as string,
|
||||
price: 10,
|
||||
active: true,
|
||||
},
|
||||
{ id: "2", title: "Doc2", category: "a", price: 20, active: true },
|
||||
]);
|
||||
const result = await engine.search({});
|
||||
expect(result.facets?.["category"]?.some((b) => b.value === "a")).toBe(
|
||||
true
|
||||
);
|
||||
});
|
||||
|
||||
it("getAvgFieldLength returns 1 when no docs indexed", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
class TestEngine extends MemorySearchEngine<Doc> {
|
||||
public testAvg() {
|
||||
return this.getAvgFieldLength();
|
||||
}
|
||||
}
|
||||
const engine = new TestEngine({ fields: [] });
|
||||
expect(engine.testAvg()).toBe(1);
|
||||
});
|
||||
|
||||
it("query token not in invertedIndex for a doc covers ?? 0 branch", {
|
||||
timeout: 30000,
|
||||
}, async () => {
|
||||
const engine = makeEngine();
|
||||
// One doc has "laptop", another has "phone". Search "laptop phone" — each token
|
||||
// appears in the invertedIndex but not for the other doc, hitting tf ?? 0.
|
||||
await engine.index([
|
||||
{
|
||||
id: "1",
|
||||
title: "Laptop Pro",
|
||||
category: "a",
|
||||
price: 100,
|
||||
active: true,
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
title: "Mobile Phone",
|
||||
category: "b",
|
||||
price: 200,
|
||||
active: true,
|
||||
},
|
||||
]);
|
||||
const result = await engine.search({ text: "laptop phone" });
|
||||
expect(result.hits.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,401 @@
|
||||
import { trace } from "@opentelemetry/api";
|
||||
import { SEARCH_CODES, SearchError } from "../../errors/search-error.js";
|
||||
import { bm25Score } from "../../text/scorer/scorer.js";
|
||||
import { tokenize } from "../../text/tokenizer/tokenizer.js";
|
||||
import { trigramSimilarity } from "../../text/trigram/trigram.js";
|
||||
import type {
|
||||
FacetBucket,
|
||||
FieldConfig,
|
||||
IndexOptions,
|
||||
MemorySearchEngineOptions,
|
||||
SearchDocument,
|
||||
SearchEngine,
|
||||
SearchFilter,
|
||||
SearchHit,
|
||||
SearchOptions,
|
||||
SearchQuery,
|
||||
SearchResult,
|
||||
SuggestOptions,
|
||||
} from "../../types/search.types.js";
|
||||
|
||||
const tracer = trace.getTracer("@suaveplan/search");
|
||||
|
||||
/**
|
||||
* In-memory search engine with inverted-index full-text search,
|
||||
* BM25 scoring, trigram fuzzy matching, faceted search, and highlights.
|
||||
*/
|
||||
export class MemorySearchEngine<T extends SearchDocument>
|
||||
implements SearchEngine<T>
|
||||
{
|
||||
public readonly name = "MemorySearchEngine";
|
||||
|
||||
protected readonly opts: MemorySearchEngineOptions<T>;
|
||||
protected readonly docs = new Map<string, T>();
|
||||
|
||||
// Inverted index: term -> Map<docId, tf>
|
||||
protected readonly invertedIndex = new Map<string, Map<string, number>>();
|
||||
|
||||
// Field lengths for BM25
|
||||
protected readonly fieldLengths = new Map<string, number>();
|
||||
|
||||
public constructor(opts: MemorySearchEngineOptions<T>) {
|
||||
this.opts = opts;
|
||||
}
|
||||
|
||||
public async index(documents: T[], options?: IndexOptions): Promise<void> {
|
||||
const span = tracer.startSpan("search.index");
|
||||
span.setAttribute("search.document_count", documents.length);
|
||||
try {
|
||||
for (const doc of documents) {
|
||||
if (options?.replace !== false && this.docs.has(doc.id)) {
|
||||
this.removeFromIndex(doc.id);
|
||||
}
|
||||
this.docs.set(doc.id, doc);
|
||||
this.addToIndex(doc);
|
||||
}
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
public async search(
|
||||
query: SearchQuery,
|
||||
options?: SearchOptions
|
||||
): Promise<SearchResult<T>> {
|
||||
const span = tracer.startSpan("search.search");
|
||||
const clock = this.opts.clock ?? (() => Date.now());
|
||||
const start = clock();
|
||||
|
||||
try {
|
||||
options?.signal?.throwIfAborted?.();
|
||||
|
||||
let candidates = [...this.docs.values()];
|
||||
|
||||
// Apply filters first
|
||||
if (query.filters?.length) {
|
||||
candidates = candidates.filter((doc) =>
|
||||
query.filters!.every((f) => this.matchFilter(doc, f))
|
||||
);
|
||||
}
|
||||
|
||||
// Scoring
|
||||
const scored: SearchHit<T>[] = [];
|
||||
|
||||
if (query.text) {
|
||||
const queryTokens = tokenize(query.text, {
|
||||
...(this.opts.stemming !== undefined
|
||||
? { stemming: this.opts.stemming }
|
||||
: {}),
|
||||
...(this.opts.stopWords !== undefined
|
||||
? { stopWords: this.opts.stopWords }
|
||||
: {}),
|
||||
});
|
||||
|
||||
const n = this.docs.size;
|
||||
const searchableFields = this.getSearchableFields();
|
||||
|
||||
// BM25 scoring per document
|
||||
const docScores = new Map<string, number>();
|
||||
|
||||
for (const token of queryTokens) {
|
||||
const df = this.invertedIndex.get(token)?.size ?? 0;
|
||||
|
||||
for (const doc of candidates) {
|
||||
const tf =
|
||||
this.invertedIndex.get(token)?.get(doc.id) ?? 0;
|
||||
const fieldLen =
|
||||
this.fieldLengths.get(doc.id) ??
|
||||
/* istanbul ignore next */ 1;
|
||||
const avgLen = this.getAvgFieldLength();
|
||||
|
||||
if (tf > 0) {
|
||||
const score = bm25Score({
|
||||
tf,
|
||||
df,
|
||||
n,
|
||||
fieldLength: fieldLen,
|
||||
avgFieldLength: avgLen,
|
||||
});
|
||||
docScores.set(
|
||||
doc.id,
|
||||
(docScores.get(doc.id) ?? 0) + score
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fuzzy matching for unmatched docs
|
||||
if (query.fuzzy) {
|
||||
const threshold = this.opts.fuzzyThreshold ?? 0.3;
|
||||
|
||||
for (const doc of candidates) {
|
||||
if (!docScores.has(doc.id)) {
|
||||
const fuzzyScore = this.computeFuzzyScore(
|
||||
doc,
|
||||
query.text,
|
||||
searchableFields
|
||||
);
|
||||
if (fuzzyScore >= threshold) {
|
||||
docScores.set(doc.id, fuzzyScore * 0.5); // fuzzy hits rank lower
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [docId, score] of docScores) {
|
||||
const doc = this.docs.get(docId)!;
|
||||
const highlights = this.extractHighlights(
|
||||
doc,
|
||||
queryTokens,
|
||||
searchableFields,
|
||||
query.highlightTag ?? this.opts.highlightTag ?? "mark"
|
||||
);
|
||||
scored.push({ document: doc, score, highlights });
|
||||
}
|
||||
} else {
|
||||
// No text query — return all candidates with score 1
|
||||
for (const doc of candidates) {
|
||||
scored.push({ document: doc, score: 1 });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort
|
||||
if (query.sort?.length) {
|
||||
scored.sort((a, b) => {
|
||||
for (const spec of query.sort!) {
|
||||
const av = a.document[spec.field];
|
||||
const bv = b.document[spec.field];
|
||||
const cmp = av! < bv! ? -1 : av! > bv! ? 1 : 0;
|
||||
if (cmp !== 0)
|
||||
return spec.direction === "asc" ? cmp : -cmp;
|
||||
}
|
||||
return 0;
|
||||
});
|
||||
} else {
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
|
||||
// Pagination
|
||||
const total = scored.length;
|
||||
const { offset, limit } = this.resolvePagination(query);
|
||||
const page = scored.slice(offset, offset + limit);
|
||||
|
||||
// Facets
|
||||
const facets = this.computeFacets(candidates);
|
||||
|
||||
const durationMs = clock() - start;
|
||||
span.setAttribute("search.hit_count", page.length);
|
||||
|
||||
return {
|
||||
hits: page,
|
||||
total,
|
||||
durationMs,
|
||||
...(facets !== undefined ? { facets } : {}),
|
||||
};
|
||||
} catch (err) {
|
||||
span.end();
|
||||
throw new SearchError(SEARCH_CODES.INDEX_FAILED, String(err), {
|
||||
cause: err as Error,
|
||||
});
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
public async suggest(
|
||||
prefix: string,
|
||||
options?: SuggestOptions
|
||||
): Promise<string[]> {
|
||||
const span = tracer.startSpan("search.suggest");
|
||||
span.setAttribute("search.prefix_length", prefix.length);
|
||||
try {
|
||||
options?.signal?.throwIfAborted?.();
|
||||
const limit = options?.limit ?? 10;
|
||||
const lower = prefix.toLowerCase();
|
||||
const matches: string[] = [];
|
||||
|
||||
for (const term of this.invertedIndex.keys()) {
|
||||
if (term.startsWith(lower)) {
|
||||
matches.push(term);
|
||||
if (matches.length >= limit) break;
|
||||
}
|
||||
}
|
||||
return matches.sort();
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
public async remove(ids: string[]): Promise<void> {
|
||||
for (const id of ids) {
|
||||
this.removeFromIndex(id);
|
||||
this.docs.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
public async clear(): Promise<void> {
|
||||
this.docs.clear();
|
||||
this.invertedIndex.clear();
|
||||
this.fieldLengths.clear();
|
||||
}
|
||||
|
||||
// ===== Private helpers =====
|
||||
|
||||
protected addToIndex(doc: T): void {
|
||||
const searchableFields = this.getSearchableFields();
|
||||
let totalTokens = 0;
|
||||
|
||||
for (const field of searchableFields) {
|
||||
const value = doc[field.name];
|
||||
if (typeof value !== "string") continue;
|
||||
const tokens = tokenize(value, {
|
||||
...(this.opts.stemming !== undefined
|
||||
? { stemming: this.opts.stemming }
|
||||
: {}),
|
||||
...(this.opts.stopWords !== undefined
|
||||
? { stopWords: this.opts.stopWords }
|
||||
: {}),
|
||||
});
|
||||
for (const token of tokens) {
|
||||
const postings = this.invertedIndex.get(token) ?? new Map();
|
||||
postings.set(doc.id, (postings.get(doc.id) ?? 0) + 1);
|
||||
this.invertedIndex.set(token, postings);
|
||||
}
|
||||
totalTokens += tokens.length;
|
||||
}
|
||||
|
||||
this.fieldLengths.set(doc.id, totalTokens);
|
||||
}
|
||||
|
||||
protected removeFromIndex(docId: string): void {
|
||||
for (const postings of this.invertedIndex.values()) {
|
||||
postings.delete(docId);
|
||||
}
|
||||
this.fieldLengths.delete(docId);
|
||||
}
|
||||
|
||||
protected getSearchableFields(): FieldConfig<T>[] {
|
||||
return this.opts.fields.filter(
|
||||
(f) => f.searchable !== false && f.type === "text"
|
||||
);
|
||||
}
|
||||
|
||||
protected getAvgFieldLength(): number {
|
||||
if (this.fieldLengths.size === 0) return /* istanbul ignore next */ 1;
|
||||
let total = 0;
|
||||
for (const len of this.fieldLengths.values()) total += len;
|
||||
return total / this.fieldLengths.size;
|
||||
}
|
||||
|
||||
protected matchFilter(doc: T, filter: SearchFilter): boolean {
|
||||
const value = doc[filter.field];
|
||||
const fv = filter.value;
|
||||
|
||||
switch (filter.op) {
|
||||
case "eq":
|
||||
return value === fv;
|
||||
case "neq":
|
||||
return value !== fv;
|
||||
case "gt":
|
||||
return typeof value === typeof fv && value! > fv!;
|
||||
case "gte":
|
||||
return typeof value === typeof fv && value! >= fv!;
|
||||
case "lt":
|
||||
return typeof value === typeof fv && value! < fv!;
|
||||
case "lte":
|
||||
return typeof value === typeof fv && value! <= fv!;
|
||||
case "in":
|
||||
return Array.isArray(fv) && fv.includes(value);
|
||||
case "contains":
|
||||
return typeof value === "string" && typeof fv === "string"
|
||||
? value.includes(fv)
|
||||
: false;
|
||||
}
|
||||
}
|
||||
|
||||
protected computeFuzzyScore(
|
||||
doc: T,
|
||||
queryText: string,
|
||||
fields: FieldConfig<T>[]
|
||||
): number {
|
||||
let maxScore = 0;
|
||||
for (const field of fields) {
|
||||
const value = doc[field.name];
|
||||
if (typeof value !== "string") continue;
|
||||
const score = trigramSimilarity(queryText, value);
|
||||
if (score > maxScore) maxScore = score;
|
||||
}
|
||||
return maxScore;
|
||||
}
|
||||
|
||||
protected extractHighlights(
|
||||
doc: T,
|
||||
tokens: string[],
|
||||
fields: FieldConfig<T>[],
|
||||
tag: string
|
||||
): Record<string, string[]> {
|
||||
const highlights: Record<string, string[]> = {};
|
||||
for (const field of fields) {
|
||||
const value = doc[field.name];
|
||||
if (typeof value !== "string") continue;
|
||||
const fragments: string[] = [];
|
||||
for (const token of tokens) {
|
||||
const re = new RegExp(`(${escapeRegex(token)})`, "gi");
|
||||
if (re.test(value)) {
|
||||
fragments.push(value.replace(re, `<${tag}>$1</${tag}>`));
|
||||
}
|
||||
}
|
||||
if (fragments.length > 0) {
|
||||
highlights[field.name] = [...new Set(fragments)];
|
||||
}
|
||||
}
|
||||
return highlights;
|
||||
}
|
||||
|
||||
protected resolvePagination(query: SearchQuery): {
|
||||
offset: number;
|
||||
limit: number;
|
||||
} {
|
||||
const spec = query.pagination;
|
||||
if (!spec) return { offset: 0, limit: 10 };
|
||||
const offset = spec.type === "offset" ? (spec.offset ?? 0) : 0;
|
||||
return { offset, limit: spec.limit };
|
||||
}
|
||||
|
||||
protected computeFacets(
|
||||
candidates: T[]
|
||||
): Record<string, FacetBucket[]> | undefined {
|
||||
const facetableFields = this.opts.fields.filter((f) => f.facetable);
|
||||
if (facetableFields.length === 0) return undefined;
|
||||
|
||||
const result: Record<string, FacetBucket[]> = {};
|
||||
for (const field of facetableFields) {
|
||||
const counts = new Map<string, number>();
|
||||
for (const doc of candidates) {
|
||||
const value = doc[field.name];
|
||||
if (value !== undefined && value !== null) {
|
||||
const key = String(value);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
result[field.name] = [...counts.entries()]
|
||||
.map(([value, count]) => ({ value, count }))
|
||||
.sort((a, b) => b.count - a.count)
|
||||
.slice(0, 100);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
function escapeRegex(s: string): string {
|
||||
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an in-memory `SearchEngine` for the given document type.
|
||||
*/
|
||||
export function createMemorySearchEngine<T extends SearchDocument>(
|
||||
options: MemorySearchEngineOptions<T>
|
||||
): SearchEngine<T> {
|
||||
return new MemorySearchEngine(options);
|
||||
}
|
||||
2
packages/persistence/search/search/src/errors/index.ts
Normal file
2
packages/persistence/search/search/src/errors/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export type { SearchCode } from "./search-error.js";
|
||||
export { SEARCH_CODES, SearchError } from "./search-error.js";
|
||||
@@ -0,0 +1,55 @@
|
||||
# search-error
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Defines the structured error type and stable error-code registry for `@suaveplan/search`. All runtime failures from indexing, querying, cursor pagination, and field validation surface as `SearchError` instances with a machine-readable `code` field.
|
||||
|
||||
## Features
|
||||
|
||||
- `SEARCH_CODES` registry via `@suaveplan/error`'s `defineCodes` — codes namespaced `SEARCH.*`
|
||||
- `SearchError` extends `SuaveplanError` for cause-chain traversal
|
||||
- Five codes: `INDEX_FAILED`, `QUERY_INVALID`, `ENGINE_NOT_READY`, `FIELD_UNKNOWN`, `CURSOR_INVALID`
|
||||
- `SearchCode` union type for exhaustive switch statements
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { SearchError, SEARCH_CODES } from "@suaveplan/search";
|
||||
|
||||
throw new SearchError(SEARCH_CODES.FIELD_UNKNOWN, `Field "price" is not indexed`);
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { SearchError, SEARCH_CODES } from "@suaveplan/search";
|
||||
|
||||
function handleSearchError(err: unknown): void {
|
||||
if (err instanceof SearchError) {
|
||||
if (err.code === SEARCH_CODES.CURSOR_INVALID) {
|
||||
// Reset pagination cursor
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `SEARCH_CODES`
|
||||
|
||||
- `SEARCH_CODES.INDEX_FAILED` — indexing operation failed
|
||||
- `SEARCH_CODES.QUERY_INVALID` — query syntax or parameter is invalid
|
||||
- `SEARCH_CODES.ENGINE_NOT_READY` — engine has no documents indexed
|
||||
- `SEARCH_CODES.FIELD_UNKNOWN` — referenced field not in schema
|
||||
- `SEARCH_CODES.CURSOR_INVALID` — pagination cursor is malformed or expired
|
||||
|
||||
### `SearchError`
|
||||
|
||||
Extends `SuaveplanError<SearchCode>`. Constructor: `(code, message, options?)`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
Codes are prefixed with `"SEARCH."` by `defineCodes`, producing strings like `"SEARCH.QUERY_INVALID"`.
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { SEARCH_CODES, type SearchCode, SearchError } from "./search-error.js";
|
||||
|
||||
describe("SearchError", () => {
|
||||
it("has correct name", { timeout: 5000 }, () => {
|
||||
expect(new SearchError(SEARCH_CODES.INDEX_FAILED, "test").name).toBe(
|
||||
"SearchError"
|
||||
);
|
||||
});
|
||||
|
||||
it("carries code", { timeout: 5000 }, () => {
|
||||
const err = new SearchError(SEARCH_CODES.QUERY_INVALID, "invalid");
|
||||
expect(err.code).toBe(SEARCH_CODES.QUERY_INVALID);
|
||||
});
|
||||
|
||||
it("all codes start with SEARCH.", { timeout: 5000 }, () => {
|
||||
const codes: SearchCode[] = [
|
||||
SEARCH_CODES.INDEX_FAILED,
|
||||
SEARCH_CODES.QUERY_INVALID,
|
||||
SEARCH_CODES.ENGINE_NOT_READY,
|
||||
SEARCH_CODES.FIELD_UNKNOWN,
|
||||
SEARCH_CODES.CURSOR_INVALID,
|
||||
];
|
||||
for (const c of codes) expect(c).toMatch(/^SEARCH\./);
|
||||
});
|
||||
|
||||
it("instanceof checks", { timeout: 5000 }, () => {
|
||||
const err = new SearchError(
|
||||
SEARCH_CODES.FIELD_UNKNOWN,
|
||||
"unknown field"
|
||||
);
|
||||
expect(err instanceof SearchError).toBe(true);
|
||||
expect(err instanceof Error).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { defineCodes, SuaveplanError } from "@suaveplan/error";
|
||||
|
||||
/**
|
||||
* Stable error-code registry for `@suaveplan/search`.
|
||||
*/
|
||||
export const SEARCH_CODES = defineCodes("SEARCH", [
|
||||
"INDEX_FAILED",
|
||||
"QUERY_INVALID",
|
||||
"ENGINE_NOT_READY",
|
||||
"FIELD_UNKNOWN",
|
||||
"CURSOR_INVALID",
|
||||
] as const);
|
||||
|
||||
export type SearchCode = (typeof SEARCH_CODES)[keyof typeof SEARCH_CODES];
|
||||
|
||||
/**
|
||||
* Typed error for all `@suaveplan/search` failures.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* throw new SearchError(SEARCH_CODES.QUERY_INVALID, "Empty query string");
|
||||
* ```
|
||||
*/
|
||||
export class SearchError extends SuaveplanError<SearchCode> {
|
||||
public constructor(
|
||||
code: SearchCode,
|
||||
message: string,
|
||||
options?: { cause?: Error; context?: Record<string, unknown> }
|
||||
) {
|
||||
super(code, message, options);
|
||||
this.name = "SearchError";
|
||||
}
|
||||
}
|
||||
54
packages/persistence/search/search/src/index.ts
Normal file
54
packages/persistence/search/search/src/index.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
// Errors
|
||||
|
||||
// Constants
|
||||
export {
|
||||
BM25_B,
|
||||
BM25_K1,
|
||||
DEFAULT_FUZZY_THRESHOLD,
|
||||
DEFAULT_PAGE_SIZE,
|
||||
} from "./constants/index.js";
|
||||
// Engine
|
||||
export {
|
||||
createMemorySearchEngine,
|
||||
MemorySearchEngine,
|
||||
} from "./engine/memory-engine/memory-engine.js";
|
||||
export type { SearchCode } from "./errors/search-error.js";
|
||||
export { SEARCH_CODES, SearchError } from "./errors/search-error.js";
|
||||
// Schemas
|
||||
export {
|
||||
FieldConfigSchema,
|
||||
MemorySearchEngineOptionsSchema,
|
||||
PaginationSpecSchema,
|
||||
SearchFilterSchema,
|
||||
SearchQuerySchema,
|
||||
SortSpecSchema,
|
||||
} from "./schemas/search.schemas.js";
|
||||
export { bm25Score, computeIdf, tfIdfScore } from "./text/scorer/scorer.js";
|
||||
export type { TokenizeOptions } from "./text/tokenizer/tokenizer.js";
|
||||
// Text utilities
|
||||
export {
|
||||
normalizeToken,
|
||||
simpleStem,
|
||||
tokenize,
|
||||
} from "./text/tokenizer/tokenizer.js";
|
||||
export { buildTrigrams, trigramSimilarity } from "./text/trigram/trigram.js";
|
||||
// Types
|
||||
export type {
|
||||
Bm25Options,
|
||||
FacetBucket,
|
||||
FieldConfig,
|
||||
FieldType,
|
||||
FilterOp,
|
||||
IndexOptions,
|
||||
MemorySearchEngineOptions,
|
||||
PaginationSpec,
|
||||
SearchDocument,
|
||||
SearchEngine,
|
||||
SearchFilter,
|
||||
SearchHit,
|
||||
SearchOptions,
|
||||
SearchQuery,
|
||||
SearchResult,
|
||||
SortSpec,
|
||||
SuggestOptions,
|
||||
} from "./types/search.types.js";
|
||||
8
packages/persistence/search/search/src/schemas/index.ts
Normal file
8
packages/persistence/search/search/src/schemas/index.ts
Normal file
@@ -0,0 +1,8 @@
|
||||
export {
|
||||
FieldConfigSchema,
|
||||
MemorySearchEngineOptionsSchema,
|
||||
PaginationSpecSchema,
|
||||
SearchFilterSchema,
|
||||
SearchQuerySchema,
|
||||
SortSpecSchema,
|
||||
} from "./search.schemas.js";
|
||||
@@ -0,0 +1,63 @@
|
||||
# search.schemas
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides Zod validation schemas for all public search query and configuration types in `@suaveplan/search`. Used at API boundaries to validate incoming queries, engine configurations, and filter specifications before they reach the in-memory engine.
|
||||
|
||||
## Features
|
||||
|
||||
- `SearchQuerySchema` — validates the full search query structure including text, filters, sort, pagination, fuzzy, and highlightTag
|
||||
- `SearchFilterSchema` — validates individual filter clauses with the 8-operator enum
|
||||
- `SortSpecSchema` — validates sort field and direction
|
||||
- `PaginationSpecSchema` — validates cursor/offset pagination with positive limit
|
||||
- `FieldConfigSchema` — validates field definitions
|
||||
- `MemorySearchEngineOptionsSchema` — validates engine construction options
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { SearchQuerySchema } from "@suaveplan/search";
|
||||
|
||||
const query = SearchQuerySchema.parse({
|
||||
text: "laptop",
|
||||
filters: [{ field: "price", op: "lte", value: 1000 }],
|
||||
pagination: { type: "offset", offset: 0, limit: 20 },
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { MemorySearchEngineOptionsSchema } from "@suaveplan/search";
|
||||
|
||||
const opts = MemorySearchEngineOptionsSchema.parse({
|
||||
fields: [{ name: "title", type: "text", searchable: true }],
|
||||
fuzzyThreshold: 0.4,
|
||||
stemming: true,
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `SearchQuerySchema`
|
||||
|
||||
Validates `{ text?, filters?, sort?, pagination?, fuzzy?, highlightTag? }`.
|
||||
|
||||
### `SearchFilterSchema`
|
||||
|
||||
Validates `{ field: string, op: FilterOp, value: unknown }`.
|
||||
|
||||
### `PaginationSpecSchema`
|
||||
|
||||
Validates `{ type: "cursor" | "offset", cursor?, offset?, limit: positive }`.
|
||||
|
||||
### `MemorySearchEngineOptionsSchema`
|
||||
|
||||
Validates `{ fields, stemming?, stopWords?, fuzzyThreshold? }`. Passthrough for injectable clock.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The `value` field in `SearchFilterSchema` uses `z.unknown()` since filter values can be any type depending on the field type. The `fuzzy` field accepts either a boolean or `{ threshold: number }` via a Zod union.
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
MemorySearchEngineOptionsSchema,
|
||||
PaginationSpecSchema,
|
||||
SearchFilterSchema,
|
||||
SearchQuerySchema,
|
||||
} from "./search.schemas.js";
|
||||
|
||||
describe("SearchQuerySchema", () => {
|
||||
it("parses empty query", { timeout: 5000 }, () => {
|
||||
expect(() => SearchQuerySchema.parse({})).not.toThrow();
|
||||
});
|
||||
|
||||
it("parses full query", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
SearchQuerySchema.parse({
|
||||
text: "hello",
|
||||
filters: [{ field: "price", op: "lte", value: 100 }],
|
||||
sort: [{ field: "price", direction: "asc" }],
|
||||
pagination: { type: "offset", offset: 0, limit: 10 },
|
||||
fuzzy: true,
|
||||
})
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects invalid op", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
SearchQuerySchema.parse({
|
||||
filters: [{ field: "f", op: "invalid", value: 1 }],
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("SearchFilterSchema", () => {
|
||||
it("parses valid filter", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
SearchFilterSchema.parse({
|
||||
field: "name",
|
||||
op: "eq",
|
||||
value: "Alice",
|
||||
})
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("accepts all operators", { timeout: 5000 }, () => {
|
||||
const ops = ["eq", "neq", "gt", "gte", "lt", "lte", "in", "contains"];
|
||||
for (const op of ops) {
|
||||
expect(() =>
|
||||
SearchFilterSchema.parse({ field: "f", op, value: null })
|
||||
).not.toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("PaginationSpecSchema", () => {
|
||||
it("parses offset pagination", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
PaginationSpecSchema.parse({ type: "offset", offset: 0, limit: 10 })
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects zero limit", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
PaginationSpecSchema.parse({ type: "offset", offset: 0, limit: 0 })
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("MemorySearchEngineOptionsSchema", () => {
|
||||
it("parses valid options", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
MemorySearchEngineOptionsSchema.parse({
|
||||
fields: [{ name: "title", type: "text" }],
|
||||
})
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects fuzzyThreshold > 1", { timeout: 5000 }, () => {
|
||||
expect(() =>
|
||||
MemorySearchEngineOptionsSchema.parse({
|
||||
fields: [],
|
||||
fuzzyThreshold: 1.5,
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import z from "zod";
|
||||
|
||||
const FilterOpSchema = z.enum([
|
||||
"eq",
|
||||
"neq",
|
||||
"gt",
|
||||
"gte",
|
||||
"lt",
|
||||
"lte",
|
||||
"in",
|
||||
"contains",
|
||||
]);
|
||||
|
||||
export const SearchFilterSchema = z.object({
|
||||
field: z.string(),
|
||||
op: FilterOpSchema,
|
||||
value: z.unknown(),
|
||||
});
|
||||
|
||||
export const SortSpecSchema = z.object({
|
||||
field: z.string(),
|
||||
direction: z.enum(["asc", "desc"]),
|
||||
});
|
||||
|
||||
export const PaginationSpecSchema = z.object({
|
||||
type: z.enum(["cursor", "offset"]),
|
||||
cursor: z.string().optional(),
|
||||
offset: z.number().int().nonnegative().optional(),
|
||||
limit: z.number().int().positive(),
|
||||
});
|
||||
|
||||
export const SearchQuerySchema = z.object({
|
||||
text: z.string().optional(),
|
||||
filters: z.array(SearchFilterSchema).optional(),
|
||||
sort: z.array(SortSpecSchema).optional(),
|
||||
pagination: PaginationSpecSchema.optional(),
|
||||
fuzzy: z
|
||||
.union([z.boolean(), z.object({ threshold: z.number() })])
|
||||
.optional(),
|
||||
highlightTag: z.string().optional(),
|
||||
});
|
||||
|
||||
export const FieldConfigSchema = z.object({
|
||||
name: z.string(),
|
||||
type: z.enum(["text", "keyword", "number", "date", "boolean"]),
|
||||
searchable: z.boolean().optional(),
|
||||
sortable: z.boolean().optional(),
|
||||
facetable: z.boolean().optional(),
|
||||
weight: z.number().optional(),
|
||||
});
|
||||
|
||||
export const MemorySearchEngineOptionsSchema = z
|
||||
.object({
|
||||
fields: z.array(FieldConfigSchema),
|
||||
stemming: z.boolean().optional(),
|
||||
stopWords: z.array(z.string()).optional(),
|
||||
fuzzyThreshold: z.number().min(0).max(1).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
@@ -0,0 +1 @@
|
||||
export { bm25Score, computeIdf, tfIdfScore } from "./scorer.js";
|
||||
69
packages/persistence/search/search/src/text/scorer/scorer.md
Normal file
69
packages/persistence/search/search/src/text/scorer/scorer.md
Normal file
@@ -0,0 +1,69 @@
|
||||
# scorer
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides BM25 and TF-IDF scoring functions used by the in-memory search engine to rank search results by relevance. BM25 (Best Match 25) is the de facto standard ranking algorithm used by Elasticsearch, Solr, and Lucene. TF-IDF is provided as a simpler alternative.
|
||||
|
||||
## Features
|
||||
|
||||
- `bm25Score(options)` — Okapi BM25 with configurable k1 (default 1.2) and b (default 0.75)
|
||||
- `tfIdfScore(tf, idf)` — classic TF × IDF product
|
||||
- `computeIdf(df, n)` — IDF computation with logarithmic smoothing
|
||||
- All functions are pure and stateless; no external dependencies
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { bm25Score } from "@suaveplan/search";
|
||||
|
||||
const score = bm25Score({
|
||||
tf: 3, // term appears 3 times in document
|
||||
df: 10, // term appears in 10 documents
|
||||
n: 100, // total documents in corpus
|
||||
fieldLength: 50,
|
||||
avgFieldLength: 30,
|
||||
});
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { bm25Score, tfIdfScore, computeIdf } from "@suaveplan/search";
|
||||
|
||||
// Custom k1/b for short-field optimization
|
||||
const titleScore = bm25Score({
|
||||
tf: 1, df: 5, n: 1000, fieldLength: 5, avgFieldLength: 8,
|
||||
k1: 0.9, b: 0.4,
|
||||
});
|
||||
|
||||
// TF-IDF comparison
|
||||
const idf = computeIdf(5, 1000);
|
||||
const tfidf = tfIdfScore(2, idf);
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `bm25Score(options: Bm25Options)`
|
||||
|
||||
- `tf` — term frequency
|
||||
- `df` — document frequency
|
||||
- `n` — corpus size
|
||||
- `fieldLength` — length of field being scored
|
||||
- `avgFieldLength` — average field length across corpus
|
||||
- `k1?` — saturation parameter (default 1.2)
|
||||
- `b?` — length normalization (default 0.75)
|
||||
|
||||
### `tfIdfScore(tf, idf)`
|
||||
|
||||
Returns `tf * idf`.
|
||||
|
||||
### `computeIdf(df, n)`
|
||||
|
||||
Returns `log(1 + n/df)`. Returns `0` for zero inputs.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
BM25 IDF uses the Robertson-Sparck Jones formula: `log(1 + (N - df + 0.5) / (df + 0.5))`. This variant is always non-negative. The field-length normalization parameter `b = 0.75` slightly penalises longer documents, which improves precision for titles and short-text fields.
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { bm25Score, computeIdf, tfIdfScore } from "./scorer.js";
|
||||
|
||||
describe("tfIdfScore", () => {
|
||||
it("multiplies tf and idf", { timeout: 5000 }, () => {
|
||||
expect(tfIdfScore(2, 3)).toBe(6);
|
||||
});
|
||||
|
||||
it("returns 0 for zero tf", { timeout: 5000 }, () => {
|
||||
expect(tfIdfScore(0, 5)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeIdf", () => {
|
||||
it("returns log(1 + n/df)", { timeout: 5000 }, () => {
|
||||
const result = computeIdf(10, 100);
|
||||
expect(result).toBeCloseTo(Math.log(1 + 100 / 10), 10);
|
||||
});
|
||||
|
||||
it("returns 0 for n=0", { timeout: 5000 }, () => {
|
||||
expect(computeIdf(5, 0)).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 0 for df=0", { timeout: 5000 }, () => {
|
||||
expect(computeIdf(0, 100)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("bm25Score", () => {
|
||||
it("returns positive score for valid inputs", { timeout: 5000 }, () => {
|
||||
const score = bm25Score({
|
||||
tf: 3,
|
||||
df: 10,
|
||||
n: 100,
|
||||
fieldLength: 50,
|
||||
avgFieldLength: 30,
|
||||
});
|
||||
expect(score).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("returns 0 for n=0", { timeout: 5000 }, () => {
|
||||
expect(
|
||||
bm25Score({
|
||||
tf: 1,
|
||||
df: 1,
|
||||
n: 0,
|
||||
fieldLength: 10,
|
||||
avgFieldLength: 10,
|
||||
})
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("returns 0 for df=0", { timeout: 5000 }, () => {
|
||||
expect(
|
||||
bm25Score({
|
||||
tf: 1,
|
||||
df: 0,
|
||||
n: 100,
|
||||
fieldLength: 10,
|
||||
avgFieldLength: 10,
|
||||
})
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("higher tf gives higher score (saturation)", { timeout: 5000 }, () => {
|
||||
const base = { df: 10, n: 100, fieldLength: 30, avgFieldLength: 30 };
|
||||
const s1 = bm25Score({ ...base, tf: 1 });
|
||||
const s5 = bm25Score({ ...base, tf: 5 });
|
||||
expect(s5).toBeGreaterThan(s1);
|
||||
});
|
||||
|
||||
it("uses custom k1 and b", { timeout: 5000 }, () => {
|
||||
const score = bm25Score({
|
||||
tf: 2,
|
||||
df: 5,
|
||||
n: 100,
|
||||
fieldLength: 20,
|
||||
avgFieldLength: 20,
|
||||
k1: 0.9,
|
||||
b: 0.4,
|
||||
});
|
||||
expect(score).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("handles avgFieldLength=0 gracefully", { timeout: 5000 }, () => {
|
||||
const score = bm25Score({
|
||||
tf: 1,
|
||||
df: 1,
|
||||
n: 10,
|
||||
fieldLength: 5,
|
||||
avgFieldLength: 0,
|
||||
});
|
||||
expect(Number.isFinite(score)).toBe(true);
|
||||
});
|
||||
});
|
||||
45
packages/persistence/search/search/src/text/scorer/scorer.ts
Normal file
45
packages/persistence/search/search/src/text/scorer/scorer.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import type { Bm25Options } from "../../types/search.types.js";
|
||||
|
||||
/**
|
||||
* Computes TF-IDF score.
|
||||
*
|
||||
* @param tf - Term frequency in the document.
|
||||
* @param idf - Inverse document frequency.
|
||||
*/
|
||||
export function tfIdfScore(tf: number, idf: number): number {
|
||||
return tf * idf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the Inverse Document Frequency (IDF) for a term.
|
||||
*
|
||||
* @param df - Number of documents containing the term.
|
||||
* @param n - Total number of documents.
|
||||
*/
|
||||
export function computeIdf(df: number, n: number): number {
|
||||
if (n === 0 || df === 0) return 0;
|
||||
return Math.log(1 + n / df);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes BM25 score for a term in a document.
|
||||
*
|
||||
* Uses the Okapi BM25 formula with configurable k1 and b parameters.
|
||||
* Default: k1 = 1.2, b = 0.75.
|
||||
*
|
||||
* @param options - BM25 parameters
|
||||
*/
|
||||
export function bm25Score(options: Bm25Options): number {
|
||||
const { tf, df, n, fieldLength, avgFieldLength } = options;
|
||||
const k1 = options.k1 ?? 1.2;
|
||||
const b = options.b ?? 0.75;
|
||||
|
||||
if (n === 0 || df === 0) return 0;
|
||||
|
||||
const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
|
||||
const normTf =
|
||||
(tf * (k1 + 1)) /
|
||||
(tf + k1 * (1 - b + b * (fieldLength / (avgFieldLength || 1))));
|
||||
|
||||
return idf * normTf;
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export type { TokenizeOptions } from "./tokenizer.js";
|
||||
export { normalizeToken, simpleStem, tokenize } from "./tokenizer.js";
|
||||
@@ -0,0 +1,59 @@
|
||||
# tokenizer
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides text tokenization and normalization utilities for the in-memory search engine. Tokenization converts raw text into a stream of normalized tokens used to build the inverted index and to process search queries. The tokenizer handles stop-word removal, case normalization, and optional simple English stemming.
|
||||
|
||||
## Features
|
||||
|
||||
- `tokenize(text, options?)` — splits text on non-word characters, removes stop words, optionally stems
|
||||
- `normalizeToken(token)` — lowercases and trims a single token
|
||||
- `simpleStem(word)` — strips common English suffixes: `-tion`, `-ing`, `-est`, `-ed`, `-er`, `-ly`, `-s`
|
||||
- Default stop-word list of 40+ common English words
|
||||
- Case sensitivity and stop-word list are configurable per call
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { tokenize } from "@suaveplan/search";
|
||||
|
||||
const tokens = tokenize("The quick brown fox jumps over the lazy dog");
|
||||
console.log(tokens); // ["quick", "brown", "fox", "jump", ...]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { tokenize, simpleStem } from "@suaveplan/search";
|
||||
|
||||
const tokens = tokenize("Running faster than expected", {
|
||||
stemming: true,
|
||||
stopWords: ["than"],
|
||||
});
|
||||
|
||||
console.log(simpleStem("computing")); // "comput"
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `tokenize(text, options?)`
|
||||
|
||||
- `options.caseSensitive?: boolean` (default `false`)
|
||||
- `options.stopWords?: string[]` — replaces default stop-word list
|
||||
- `options.stemming?: boolean` — applies `simpleStem` to each token (default `false`)
|
||||
- Returns `string[]`
|
||||
|
||||
### `normalizeToken(token)`
|
||||
|
||||
Lowercases and trims a single token. Used internally.
|
||||
|
||||
### `simpleStem(word)`
|
||||
|
||||
Returns the stem of an English word by stripping common suffixes. Words ≤3 characters are returned unchanged.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The stemmer is intentionally simple (suffix-stripping, no Snowball/Porter) to avoid external dependencies. For production use with high-accuracy stemming requirements, a custom tokenizer implementing the same interface can be passed via `MemorySearchEngineOptions`.
|
||||
@@ -0,0 +1,98 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { normalizeToken, simpleStem, tokenize } from "./tokenizer.js";
|
||||
|
||||
describe("tokenize", () => {
|
||||
it("splits on whitespace", { timeout: 5000 }, () => {
|
||||
const tokens = tokenize("hello world");
|
||||
expect(tokens).toContain("hello");
|
||||
expect(tokens).toContain("world");
|
||||
});
|
||||
|
||||
it("lowercases by default", { timeout: 5000 }, () => {
|
||||
expect(tokenize("Hello World")).toContain("hello");
|
||||
});
|
||||
|
||||
it("removes default stop words", { timeout: 5000 }, () => {
|
||||
const tokens = tokenize("the quick fox");
|
||||
expect(tokens).not.toContain("the");
|
||||
expect(tokens).toContain("quick");
|
||||
});
|
||||
|
||||
it("respects caseSensitive option", { timeout: 5000 }, () => {
|
||||
const tokens = tokenize("Hello World", { caseSensitive: true });
|
||||
expect(tokens).toContain("Hello");
|
||||
});
|
||||
|
||||
it("uses custom stop words", { timeout: 5000 }, () => {
|
||||
const tokens = tokenize("quick fox", { stopWords: ["fox"] });
|
||||
expect(tokens).not.toContain("fox");
|
||||
expect(tokens).toContain("quick");
|
||||
});
|
||||
|
||||
it("applies stemming when enabled", { timeout: 5000 }, () => {
|
||||
const tokens = tokenize("running faster", { stemming: true });
|
||||
expect(tokens).toContain("runn");
|
||||
});
|
||||
|
||||
it("handles empty string", { timeout: 5000 }, () => {
|
||||
expect(tokenize("")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("handles punctuation", { timeout: 5000 }, () => {
|
||||
const tokens = tokenize("hello, world!");
|
||||
expect(tokens).toContain("hello");
|
||||
expect(tokens).toContain("world");
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeToken", () => {
|
||||
it("lowercases", { timeout: 5000 }, () => {
|
||||
expect(normalizeToken("Hello")).toBe("hello");
|
||||
});
|
||||
|
||||
it("trims whitespace", { timeout: 5000 }, () => {
|
||||
expect(normalizeToken(" hello ")).toBe("hello");
|
||||
});
|
||||
});
|
||||
|
||||
describe("simpleStem", () => {
|
||||
it("strips -ing", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("running")).toBe("runn");
|
||||
});
|
||||
|
||||
it("strips -ed", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("jumped")).toBe("jump");
|
||||
});
|
||||
|
||||
it("strips -er", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("faster")).toBe("fast");
|
||||
});
|
||||
|
||||
it("strips -est", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("fastest")).toBe("fast");
|
||||
});
|
||||
|
||||
it("strips -ly", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("quickly")).toBe("quick");
|
||||
});
|
||||
|
||||
it("strips -s", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("cats")).toBe("cat");
|
||||
});
|
||||
|
||||
it("preserves -ss words", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("grass")).toBe("grass");
|
||||
});
|
||||
|
||||
it("does not stem short words", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("the")).toBe("the");
|
||||
});
|
||||
|
||||
it("strips -tion", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("creation")).toBe("crea");
|
||||
});
|
||||
|
||||
it("returns unchanged for no matching suffix", { timeout: 5000 }, () => {
|
||||
expect(simpleStem("hello")).toBe("hello");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* Options for tokenization.
|
||||
*/
|
||||
export interface TokenizeOptions {
|
||||
caseSensitive?: boolean;
|
||||
stopWords?: string[];
|
||||
stemming?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_STOP_WORDS = new Set([
|
||||
"a",
|
||||
"an",
|
||||
"the",
|
||||
"and",
|
||||
"or",
|
||||
"but",
|
||||
"in",
|
||||
"on",
|
||||
"at",
|
||||
"to",
|
||||
"for",
|
||||
"of",
|
||||
"with",
|
||||
"by",
|
||||
"from",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"will",
|
||||
"would",
|
||||
"could",
|
||||
"should",
|
||||
"may",
|
||||
"might",
|
||||
"shall",
|
||||
"it",
|
||||
"its",
|
||||
"this",
|
||||
"that",
|
||||
"these",
|
||||
"those",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Tokenizes text into an array of normalized tokens.
|
||||
*
|
||||
* Splits on non-word characters, lowercases by default, removes stop words,
|
||||
* and optionally applies simple English stemming.
|
||||
*/
|
||||
export function tokenize(text: string, options?: TokenizeOptions): string[] {
|
||||
const caseSensitive = options?.caseSensitive ?? false;
|
||||
const stopWords = options?.stopWords
|
||||
? new Set(options.stopWords.map((s) => s.toLowerCase()))
|
||||
: DEFAULT_STOP_WORDS;
|
||||
const doStemming = options?.stemming ?? false;
|
||||
|
||||
const normalized = caseSensitive ? text : text.toLowerCase();
|
||||
const tokens = normalized.split(/\W+/).filter(Boolean);
|
||||
|
||||
return tokens
|
||||
.filter((t) => t.length > 0 && !stopWords.has(t.toLowerCase()))
|
||||
.map((t) => (doStemming ? simpleStem(t) : t));
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes a single token (lowercase, trim).
|
||||
*/
|
||||
export function normalizeToken(token: string): string {
|
||||
return token.toLowerCase().trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Very simple suffix-stripping English stemmer.
|
||||
* Handles common suffixes: -ing, -ed, -er, -est, -ly, -tion, -s.
|
||||
*/
|
||||
export function simpleStem(word: string): string {
|
||||
if (word.length <= 3) return word;
|
||||
if (word.endsWith("tion") && word.length > 6) return word.slice(0, -4);
|
||||
if (word.endsWith("ing") && word.length > 5) return word.slice(0, -3);
|
||||
if (word.endsWith("est") && word.length > 5) return word.slice(0, -3);
|
||||
if (word.endsWith("ed") && word.length > 4) return word.slice(0, -2);
|
||||
if (word.endsWith("er") && word.length > 4) return word.slice(0, -2);
|
||||
if (word.endsWith("ly") && word.length > 4) return word.slice(0, -2);
|
||||
if (word.endsWith("s") && !word.endsWith("ss") && word.length > 3) {
|
||||
return word.slice(0, -1);
|
||||
}
|
||||
return word;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { buildTrigrams, trigramSimilarity } from "./trigram.js";
|
||||
@@ -0,0 +1,51 @@
|
||||
# trigram
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
## Purpose
|
||||
|
||||
Provides trigram-based fuzzy matching utilities for the in-memory search engine. Trigrams are overlapping 3-character substrings that capture structural similarity between strings — they are language-agnostic, robust to minor typos, and require no external libraries.
|
||||
|
||||
## Features
|
||||
|
||||
- `buildTrigrams(text)` — generates a `Set<string>` of all 3-character substrings, padded with `##` for start/end boundary capture
|
||||
- `trigramSimilarity(a, b)` — computes Jaccard similarity (intersection / union) over the trigram sets, returning a value in `[0, 1]`
|
||||
- Case-insensitive: both functions normalize input to lowercase before processing
|
||||
- Exact matches always return `1.0`
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { trigramSimilarity } from "@suaveplan/search";
|
||||
|
||||
const score = trigramSimilarity("hello", "helo");
|
||||
console.log(score); // ~0.71
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { buildTrigrams, trigramSimilarity } from "@suaveplan/search";
|
||||
|
||||
const threshold = 0.5;
|
||||
const candidates = ["hello", "help", "world", "dog"];
|
||||
const matches = candidates.filter(
|
||||
(c) => trigramSimilarity("helo", c) >= threshold
|
||||
);
|
||||
console.log(matches); // ["hello", "help"]
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `buildTrigrams(text)`
|
||||
|
||||
Returns `Set<string>` of all trigrams in `text` (case-insensitive, `##`-padded).
|
||||
|
||||
### `trigramSimilarity(a, b)`
|
||||
|
||||
Returns `number` in `[0, 1]`. Uses Jaccard coefficient: `|intersection| / |union|`. Returns `1` for identical strings or empty trigram sets.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The `##` padding ensures that short strings (1-2 characters) still generate meaningful trigrams that capture their start and end positions. The Jaccard coefficient is symmetric: `trigramSimilarity(a, b) === trigramSimilarity(b, a)`. Complexity is O(n + m) where n and m are the trigram set sizes.
|
||||
@@ -0,0 +1,63 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { buildTrigrams, trigramSimilarity } from "./trigram.js";
|
||||
|
||||
describe("buildTrigrams", () => {
|
||||
it("returns set of trigrams", { timeout: 5000 }, () => {
|
||||
const t = buildTrigrams("cat");
|
||||
expect(t.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("includes padded trigrams", { timeout: 5000 }, () => {
|
||||
const t = buildTrigrams("cat");
|
||||
expect(t.has("##c")).toBe(true);
|
||||
expect(t.has("t##")).toBe(true);
|
||||
});
|
||||
|
||||
it("handles empty string", { timeout: 5000 }, () => {
|
||||
const t = buildTrigrams("");
|
||||
// "####" -> "###", "##" -- only "###" is length 3
|
||||
expect(t.size).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it("is case insensitive", { timeout: 5000 }, () => {
|
||||
const t1 = buildTrigrams("Cat");
|
||||
const t2 = buildTrigrams("cat");
|
||||
expect([...t1].sort()).toEqual([...t2].sort());
|
||||
});
|
||||
});
|
||||
|
||||
describe("trigramSimilarity", () => {
|
||||
it("identical strings return 1", { timeout: 5000 }, () => {
|
||||
expect(trigramSimilarity("hello", "hello")).toBe(1);
|
||||
});
|
||||
|
||||
it("completely different strings return low score", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expect(trigramSimilarity("cat", "xyz")).toBeLessThan(0.3);
|
||||
});
|
||||
|
||||
it("similar strings return high score", { timeout: 5000 }, () => {
|
||||
expect(trigramSimilarity("hello", "helo")).toBeGreaterThan(0.5);
|
||||
});
|
||||
|
||||
it("is symmetric", { timeout: 5000 }, () => {
|
||||
const a = trigramSimilarity("hello", "world");
|
||||
const b = trigramSimilarity("world", "hello");
|
||||
expect(a).toBeCloseTo(b, 10);
|
||||
});
|
||||
|
||||
it("returns value between 0 and 1", { timeout: 5000 }, () => {
|
||||
const score = trigramSimilarity("foo", "bar");
|
||||
expect(score).toBeGreaterThanOrEqual(0);
|
||||
expect(score).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
it("case insensitive comparison", { timeout: 5000 }, () => {
|
||||
expect(trigramSimilarity("Hello", "hello")).toBe(1);
|
||||
});
|
||||
|
||||
it("both empty strings return 1", { timeout: 5000 }, () => {
|
||||
expect(trigramSimilarity("", "")).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Builds a set of character trigrams from a string.
|
||||
*
|
||||
* Pads the string with `#` on both sides to capture start/end bigrams.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* buildTrigrams("cat") // Set {"##c", "#ca", "cat", "at#", "t##"}
|
||||
* ```
|
||||
*/
|
||||
export function buildTrigrams(text: string): Set<string> {
|
||||
const padded = `##${text.toLowerCase()}##`;
|
||||
const trigrams = new Set<string>();
|
||||
for (let i = 0; i <= padded.length - 3; i++) {
|
||||
trigrams.add(padded.slice(i, i + 3));
|
||||
}
|
||||
return trigrams;
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the trigram similarity between two strings.
|
||||
*
|
||||
* Returns a value in [0, 1] where 1.0 means identical.
|
||||
* Based on the Jaccard coefficient of the trigram sets.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* trigramSimilarity("hello", "hello") // 1.0
|
||||
* trigramSimilarity("hello", "helo") // ~0.7
|
||||
* trigramSimilarity("cat", "dog") // ~0.0
|
||||
* ```
|
||||
*/
|
||||
export function trigramSimilarity(a: string, b: string): number {
|
||||
if (a === b) return 1;
|
||||
const ta = buildTrigrams(a);
|
||||
const tb = buildTrigrams(b);
|
||||
let intersection = 0;
|
||||
for (const t of ta) {
|
||||
if (tb.has(t)) intersection++;
|
||||
}
|
||||
const union = ta.size + tb.size - intersection;
|
||||
return union === 0 ? /* istanbul ignore next */ 1 : intersection / union;
|
||||
}
|
||||
19
packages/persistence/search/search/src/types/index.ts
Normal file
19
packages/persistence/search/search/src/types/index.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
export type {
|
||||
Bm25Options,
|
||||
FacetBucket,
|
||||
FieldConfig,
|
||||
FieldType,
|
||||
FilterOp,
|
||||
IndexOptions,
|
||||
MemorySearchEngineOptions,
|
||||
PaginationSpec,
|
||||
SearchDocument,
|
||||
SearchEngine,
|
||||
SearchFilter,
|
||||
SearchHit,
|
||||
SearchOptions,
|
||||
SearchQuery,
|
||||
SearchResult,
|
||||
SortSpec,
|
||||
SuggestOptions,
|
||||
} from "./search.types.js";
|
||||
70
packages/persistence/search/search/src/types/search.types.md
Normal file
70
packages/persistence/search/search/src/types/search.types.md
Normal file
@@ -0,0 +1,70 @@
|
||||
# search.types
|
||||
|
||||
## Purpose
|
||||
|
||||
Shared TypeScript type definitions for the `@suaveplan/search` package. Defines all public contracts: search documents, queries, filters, results, field configurations, and engine interfaces. Centralising types here prevents circular imports between the engine, text utilities, and schema modules.
|
||||
|
||||
## Features
|
||||
|
||||
- `SearchDocument` — base interface requiring `id: string` plus arbitrary fields
|
||||
- `SearchQuery` — full query structure with text, filters, sort, pagination, and fuzzy options
|
||||
- `SearchFilter` — structured field filter with 8 operators (`eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `in`, `contains`)
|
||||
- `SearchResult<T>` / `SearchHit<T>` — typed result container with score, highlights, and facets
|
||||
- `FieldConfig<T>` — per-field schema: type, searchable, sortable, facetable, weight
|
||||
- `MemorySearchEngineOptions<T>` — full configuration for the in-memory engine
|
||||
- `SearchEngine<T>` — interface for `index`, `search`, `suggest`, `remove`, `clear`
|
||||
- `Bm25Options` — parameters for BM25 scoring function
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import type { SearchDocument, SearchQuery, FieldConfig } from "@suaveplan/search";
|
||||
|
||||
interface Product extends SearchDocument {
|
||||
name: string;
|
||||
price: number;
|
||||
}
|
||||
|
||||
const field: FieldConfig<Product> = {
|
||||
name: "name",
|
||||
type: "text",
|
||||
searchable: true,
|
||||
facetable: false,
|
||||
};
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import type { SearchQuery, SearchFilter } from "@suaveplan/search";
|
||||
|
||||
const query: SearchQuery = {
|
||||
text: "laptop",
|
||||
filters: [{ field: "price", op: "lte", value: 1000 }],
|
||||
sort: [{ field: "price", direction: "asc" }],
|
||||
pagination: { type: "offset", offset: 0, limit: 20 },
|
||||
fuzzy: true,
|
||||
};
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `SearchDocument`
|
||||
|
||||
Base interface: `{ id: string; [field: string]: unknown }`.
|
||||
|
||||
### `SearchQuery`
|
||||
|
||||
Full query: `text?`, `filters?`, `sort?`, `pagination?`, `fuzzy?`, `highlightTag?`.
|
||||
|
||||
### `FieldConfig<T>`
|
||||
|
||||
`name: keyof T & string`, `type: FieldType`, `searchable?`, `sortable?`, `facetable?`, `weight?`.
|
||||
|
||||
### `SearchEngine<T>`
|
||||
|
||||
`index(docs, opts?)`, `search(query, opts?)`, `suggest(prefix, opts?)`, `remove(ids)`, `clear()`.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The `FilterOp` union and `FieldType` union are intentionally narrow — they define the supported operation set for the in-memory engine. Engines backed by external systems may support additional operators through extension.
|
||||
168
packages/persistence/search/search/src/types/search.types.ts
Normal file
168
packages/persistence/search/search/src/types/search.types.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
import type { Clock } from "@suaveplan/types";
|
||||
|
||||
/**
|
||||
* A document that can be indexed and searched.
|
||||
*/
|
||||
export interface SearchDocument {
|
||||
id: string;
|
||||
[field: string]: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* A single search result hit.
|
||||
*/
|
||||
export interface SearchHit<T extends SearchDocument> {
|
||||
document: T;
|
||||
score: number;
|
||||
highlights?: Record<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A facet bucket for aggregated counts.
|
||||
*/
|
||||
export interface FacetBucket {
|
||||
value: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full search result including hits, total, cursor, and facets.
|
||||
*/
|
||||
export interface SearchResult<T extends SearchDocument> {
|
||||
hits: SearchHit<T>[];
|
||||
total: number;
|
||||
cursor?: string;
|
||||
durationMs: number;
|
||||
facets?: Record<string, FacetBucket[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter operator for structured field queries.
|
||||
*/
|
||||
export type FilterOp =
|
||||
| "eq"
|
||||
| "neq"
|
||||
| "gt"
|
||||
| "gte"
|
||||
| "lt"
|
||||
| "lte"
|
||||
| "in"
|
||||
| "contains";
|
||||
|
||||
/**
|
||||
* A single filter clause.
|
||||
*/
|
||||
export interface SearchFilter {
|
||||
field: string;
|
||||
op: FilterOp;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort specification for result ordering.
|
||||
*/
|
||||
export interface SortSpec {
|
||||
field: string;
|
||||
direction: "asc" | "desc";
|
||||
}
|
||||
|
||||
/**
|
||||
* Pagination spec for search results.
|
||||
*/
|
||||
export interface PaginationSpec {
|
||||
type: "cursor" | "offset";
|
||||
cursor?: string;
|
||||
offset?: number;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full search query.
|
||||
*/
|
||||
export interface SearchQuery {
|
||||
text?: string;
|
||||
filters?: SearchFilter[];
|
||||
sort?: SortSpec[];
|
||||
pagination?: PaginationSpec;
|
||||
fuzzy?: boolean;
|
||||
highlightTag?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for indexing.
|
||||
*/
|
||||
export interface IndexOptions {
|
||||
replace?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for search operations.
|
||||
*/
|
||||
export interface SearchOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for suggest operations.
|
||||
*/
|
||||
export interface SuggestOptions {
|
||||
limit?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Field type in the search schema.
|
||||
*/
|
||||
export type FieldType = "text" | "keyword" | "number" | "date" | "boolean";
|
||||
|
||||
/**
|
||||
* Per-field configuration.
|
||||
*/
|
||||
export interface FieldConfig<T extends SearchDocument> {
|
||||
name: keyof T & string;
|
||||
type: FieldType;
|
||||
searchable?: boolean;
|
||||
sortable?: boolean;
|
||||
facetable?: boolean;
|
||||
weight?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for the in-memory search engine.
|
||||
*/
|
||||
export interface MemorySearchEngineOptions<T extends SearchDocument> {
|
||||
fields: FieldConfig<T>[];
|
||||
stemming?: boolean;
|
||||
stopWords?: string[];
|
||||
fuzzyThreshold?: number;
|
||||
highlightTag?: string;
|
||||
clock?: Clock;
|
||||
}
|
||||
|
||||
/**
|
||||
* BM25 scoring options.
|
||||
*/
|
||||
export interface Bm25Options {
|
||||
tf: number;
|
||||
df: number;
|
||||
n: number;
|
||||
fieldLength: number;
|
||||
avgFieldLength: number;
|
||||
k1?: number;
|
||||
b?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface all search engine implementations must satisfy.
|
||||
*/
|
||||
export interface SearchEngine<T extends SearchDocument> {
|
||||
readonly name: string;
|
||||
index(documents: T[], options?: IndexOptions): Promise<void>;
|
||||
search(
|
||||
query: SearchQuery,
|
||||
options?: SearchOptions
|
||||
): Promise<SearchResult<T>>;
|
||||
suggest(prefix: string, options?: SuggestOptions): Promise<string[]>;
|
||||
remove(ids: string[]): Promise<void>;
|
||||
clear(): Promise<void>;
|
||||
}
|
||||
12
packages/persistence/search/search/tests/dep-graph.test.ts
Normal file
12
packages/persistence/search/search/tests/dep-graph.test.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
|
||||
describe("@suaveplan/search dep-graph", () => {
|
||||
it("imports without error", { timeout: 30000 }, async () => {
|
||||
const mod = await import("../src/index.js");
|
||||
expect(typeof mod.createMemorySearchEngine).toBe("function");
|
||||
expect(typeof mod.tokenize).toBe("function");
|
||||
expect(typeof mod.trigramSimilarity).toBe("function");
|
||||
expect(typeof mod.bm25Score).toBe("function");
|
||||
expect(typeof mod.SearchError).toBe("function");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
// Polyfills for search tests
|
||||
12
packages/persistence/search/search/tsconfig.lib.json
Normal file
12
packages/persistence/search/search/tsconfig.lib.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "src/**/*.bench.ts", "dist", "node_modules"]
|
||||
}
|
||||
38
packages/persistence/search/search/vite.config.ts
Normal file
38
packages/persistence/search/search/vite.config.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/search",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/types",
|
||||
"@suaveplan/pagination",
|
||||
"tslib",
|
||||
"zod",
|
||||
"@opentelemetry/api",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
26
packages/persistence/search/search/vitest.config.ts
Normal file
26
packages/persistence/search/search/vitest.config.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
setupFiles: ["./tests/setup/polyfills.ts"],
|
||||
include: ["src/**/*.test.ts", "tests/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/index.ts",
|
||||
"src/**/__benchmarks__/**",
|
||||
"tests/**",
|
||||
],
|
||||
thresholds: { statements: 100, branches: 100, functions: 100, lines: 100 },
|
||||
},
|
||||
testTimeout: 30000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user