Initial Commit
This commit is contained in:
265
openspec/archive/2026-04-24-enterprise-search/proposal.md
Normal file
265
openspec/archive/2026-04-24-enterprise-search/proposal.md
Normal file
@@ -0,0 +1,265 @@
|
||||
# OpenSpec Change: enterprise-search
|
||||
|
||||
- **Change ID**: enterprise-search
|
||||
- **Package**: `@suaveplan/search`
|
||||
- **Category**: universal
|
||||
- **Target path**: `universal/data/search`
|
||||
- **Status**: PROPOSED
|
||||
- **Date**: 2026-04-24
|
||||
- **Reference spec**: `/mnt/s/web/suaveplan/enterprise/search.md`
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Deliver `@suaveplan/search` on the genesis branch from scratch at enterprise quality. The package provides universal full-text and fuzzy search primitives: a pluggable `SearchEngine` interface, built-in in-memory engine with inverted-index full-text and trigram-based fuzzy matching, cursor and offset pagination via `@suaveplan/pagination`, structured errors, OTel instrumentation, Zod-validated query schemas, and property-based correctness guarantees.
|
||||
|
||||
---
|
||||
|
||||
## Current-state audit (genesis)
|
||||
|
||||
The package does not exist on genesis yet — fresh build. No prior source, no test results, no coverage data.
|
||||
|
||||
---
|
||||
|
||||
## Available workspace deps on genesis
|
||||
|
||||
| Package | Role |
|
||||
|---|---|
|
||||
| `@suaveplan/error` (`workspace:^`) | Base class for `SearchError`; `defineCodes` for `SEARCH.*` codes |
|
||||
| `@suaveplan/types` (`workspace:^`) | `LoggerLike`, `Clock` injection interfaces |
|
||||
| `@suaveplan/pagination` (`workspace:^`) | Cursor and offset pagination helpers for search results |
|
||||
| `zod` (`catalog:forms`) | Zod schemas for query, filter, and result types |
|
||||
| `@opentelemetry/api` (`catalog:observability`) | Spans, counters, histograms on search, index, and suggest operations |
|
||||
| `tslib` (`catalog:build`) | Runtime helpers |
|
||||
|
||||
Dev deps: `vitest`, `@vitest/coverage-istanbul`, `fast-check`, `mitata` (all `catalog:testing`).
|
||||
|
||||
---
|
||||
|
||||
## Public API surface
|
||||
|
||||
### Engine interface
|
||||
|
||||
```ts
|
||||
interface SearchEngine<T extends SearchDocument> {
|
||||
readonly name: string
|
||||
index(documents: T[], options?: IndexOptions): Promise<void>
|
||||
search(query: SearchQuery, options?: SearchOptions): Promise<SearchResult<T>>
|
||||
suggest(prefix: string, options?: SuggestOptions): Promise<string[]>
|
||||
remove(ids: string[]): Promise<void>
|
||||
clear(): Promise<void>
|
||||
}
|
||||
|
||||
interface SearchDocument {
|
||||
id: string
|
||||
[field: string]: unknown
|
||||
}
|
||||
|
||||
interface SearchQuery {
|
||||
text?: string
|
||||
filters?: SearchFilter[]
|
||||
sort?: SortSpec[]
|
||||
pagination?: PaginationSpec
|
||||
}
|
||||
|
||||
interface SearchFilter {
|
||||
field: string
|
||||
op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "in" | "contains"
|
||||
value: unknown
|
||||
}
|
||||
|
||||
interface SortSpec {
|
||||
field: string
|
||||
direction: "asc" | "desc"
|
||||
}
|
||||
|
||||
interface PaginationSpec {
|
||||
type: "cursor" | "offset"
|
||||
cursor?: string
|
||||
offset?: number
|
||||
limit: number
|
||||
}
|
||||
|
||||
interface SearchResult<T extends SearchDocument> {
|
||||
hits: SearchHit<T>[]
|
||||
total: number
|
||||
cursor?: string
|
||||
durationMs: number
|
||||
facets?: Record<string, FacetBucket[]>
|
||||
}
|
||||
|
||||
interface SearchHit<T extends SearchDocument> {
|
||||
document: T
|
||||
score: number
|
||||
highlights?: Record<string, string[]>
|
||||
}
|
||||
|
||||
interface FacetBucket {
|
||||
value: string
|
||||
count: number
|
||||
}
|
||||
```
|
||||
|
||||
### In-memory engine
|
||||
|
||||
```ts
|
||||
interface MemorySearchEngineOptions<T extends SearchDocument> {
|
||||
fields: FieldConfig<T>[]
|
||||
stemming?: boolean
|
||||
stopWords?: string[]
|
||||
fuzzyThreshold?: number
|
||||
logger?: LoggerLike
|
||||
clock?: Clock
|
||||
}
|
||||
|
||||
interface FieldConfig<T extends SearchDocument> {
|
||||
name: keyof T & string
|
||||
type: "text" | "keyword" | "number" | "date" | "boolean"
|
||||
searchable?: boolean
|
||||
sortable?: boolean
|
||||
facetable?: boolean
|
||||
weight?: number
|
||||
}
|
||||
|
||||
function createMemorySearchEngine<T extends SearchDocument>(
|
||||
options: MemorySearchEngineOptions<T>
|
||||
): SearchEngine<T>
|
||||
```
|
||||
|
||||
### Text utilities
|
||||
|
||||
```ts
|
||||
// Tokenization, normalization, stemming
|
||||
function tokenize(text: string, options?: TokenizeOptions): string[]
|
||||
function normalize(token: string): string
|
||||
function buildTrigrams(text: string): Set<string>
|
||||
function trigramSimilarity(a: string, b: string): number
|
||||
|
||||
// Scoring
|
||||
function tfIdfScore(tf: number, idf: number): number
|
||||
function bm25Score(options: Bm25Options): number
|
||||
```
|
||||
|
||||
### Error handling
|
||||
|
||||
```ts
|
||||
class SearchError extends SuaveplanError<SearchErrorCode>
|
||||
|
||||
type SearchErrorCode =
|
||||
| "SEARCH_INDEX_FAILED"
|
||||
| "SEARCH_QUERY_INVALID"
|
||||
| "SEARCH_ENGINE_NOT_READY"
|
||||
| "SEARCH_FIELD_UNKNOWN"
|
||||
| "SEARCH_CURSOR_INVALID"
|
||||
|
||||
function registerSearchErrorCodes(): void
|
||||
```
|
||||
|
||||
### Zod schemas
|
||||
|
||||
```ts
|
||||
const SearchQuerySchema: ZodSchema
|
||||
const SearchFilterSchema: ZodSchema
|
||||
const SearchResultSchema: ZodSchema
|
||||
const MemorySearchEngineOptionsSchema: ZodSchema
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## New additions for genesis
|
||||
|
||||
### 1. BM25 scoring
|
||||
|
||||
The in-memory engine uses BM25 (k1=1.2, b=0.75) as its default ranking algorithm, configurable via `scoringOptions` in `MemorySearchEngineOptions`.
|
||||
|
||||
### 2. Faceted search
|
||||
|
||||
When `FieldConfig.facetable = true`, search results include `facets` with bucket counts per facetable field. Supports up to 100 distinct values per field in memory.
|
||||
|
||||
### 3. Highlight extraction
|
||||
|
||||
Matched terms are highlighted in the original field text using a configurable `<mark>` wrapper (or caller-supplied `highlightTag`). Applied to all `searchable` text fields.
|
||||
|
||||
### 4. OTel instrumentation
|
||||
|
||||
- `engine.index` — span `search.index`, attributes: `search.document_count`
|
||||
- `engine.search` — span `search.search`, attributes: `search.query_text`, `search.filter_count`, `search.hit_count`; histogram `search.query_duration_ms`
|
||||
- `engine.suggest` — span `search.suggest`, attributes: `search.prefix_length`
|
||||
- Counters: `search.queries_total`, `search.index_operations_total`
|
||||
|
||||
### 5. Property-based tests (fast-check)
|
||||
|
||||
- Index then search: every indexed document is retrievable by exact ID match
|
||||
- Filter monotonicity: adding a filter never increases the result count
|
||||
- Trigram similarity symmetry: `sim(a, b) === sim(b, a)` for all strings
|
||||
- Pagination completeness: paginating through all results yields the same total as a single unbounded query
|
||||
|
||||
### 6. Benchmark suite (`src/__benchmarks__/`)
|
||||
|
||||
- Index 1k, 10k, 100k documents
|
||||
- Search with single text query on indexed dataset
|
||||
- Suggest with 3-character prefix
|
||||
|
||||
---
|
||||
|
||||
## File layout
|
||||
|
||||
```text
|
||||
universal/data/search/
|
||||
package.json
|
||||
tsconfig.json
|
||||
vitest.config.ts
|
||||
src/
|
||||
index.ts
|
||||
error-codes.ts
|
||||
constants/
|
||||
index.ts
|
||||
types/
|
||||
common.ts
|
||||
common.md
|
||||
engine/
|
||||
search-engine/
|
||||
search-engine.ts
|
||||
search-engine.md
|
||||
search-engine.types.ts
|
||||
search-engine.test.ts
|
||||
index.ts
|
||||
memory-engine/
|
||||
memory-engine.ts
|
||||
memory-engine.md
|
||||
memory-engine.types.ts
|
||||
memory-engine.test.ts
|
||||
index.ts
|
||||
text/
|
||||
tokenizer/
|
||||
tokenizer.ts
|
||||
tokenizer.md
|
||||
tokenizer.test.ts
|
||||
index.ts
|
||||
trigram/
|
||||
trigram.ts
|
||||
trigram.md
|
||||
trigram.test.ts
|
||||
index.ts
|
||||
scorer/
|
||||
scorer.ts
|
||||
scorer.md
|
||||
scorer.test.ts
|
||||
index.ts
|
||||
__benchmarks__/
|
||||
search.bench.ts
|
||||
tests/
|
||||
integration.test.ts
|
||||
property.test.ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Impact
|
||||
|
||||
**Downstream packages**: any package needing full-text or fuzzy search on in-memory datasets. No existing genesis package depends on it yet.
|
||||
|
||||
**Workspace dep prerequisites**: `@suaveplan/error`, `@suaveplan/types`, `@suaveplan/pagination` must be published to Verdaccio before this package can be built.
|
||||
|
||||
**Breaking changes**: none (new package).
|
||||
87
openspec/archive/2026-04-24-enterprise-search/tasks.md
Normal file
87
openspec/archive/2026-04-24-enterprise-search/tasks.md
Normal file
@@ -0,0 +1,87 @@
|
||||
# Tasks: enterprise-search
|
||||
|
||||
- **Change ID**: enterprise-search
|
||||
- **Package**: `@suaveplan/search`
|
||||
- **Date**: 2026-04-24
|
||||
|
||||
## Prerequisites (must be published to Verdaccio first)
|
||||
|
||||
- [x] `@suaveplan/error`
|
||||
- [x] `@suaveplan/types`
|
||||
- [x] `@suaveplan/pagination`
|
||||
|
||||
## Tasks
|
||||
|
||||
### 1. Scaffold package
|
||||
|
||||
- [x] Create `universal/data/search/` directory tree per file layout in `proposal.md`
|
||||
- [x] Write `package.json` (name `@suaveplan/search`, version `0.1.0`, `sideEffects: false`, correct workspace deps and catalog externals)
|
||||
- [x] Write `tsconfig.json` extending `@suaveplan/typescript-config/library.json`
|
||||
- [x] Write `vitest.config.ts` using `createTestConfig()` from `@suaveplan/testing/config`
|
||||
|
||||
### 2. Error codes (`src/error-codes.ts`)
|
||||
|
||||
- [x] Define `SearchErrorCode` union type
|
||||
- [x] Call `defineCodes` to register all `SEARCH.*` codes
|
||||
- [x] Export `registerSearchErrorCodes()` and `SearchError`
|
||||
|
||||
### 3. Types (`src/types/common.ts`)
|
||||
|
||||
- [x] Define and export all public interfaces: `SearchEngine`, `SearchDocument`, `SearchQuery`, `SearchFilter`, `SortSpec`, `PaginationSpec`, `SearchResult`, `SearchHit`, `FacetBucket`, `FieldConfig`, `MemorySearchEngineOptions`
|
||||
- [x] Write `common.md`
|
||||
|
||||
### 4. Text utilities
|
||||
|
||||
- [x] Implement `tokenize`, `normalize` in `src/text/tokenizer/`
|
||||
- [x] Implement `buildTrigrams`, `trigramSimilarity` in `src/text/trigram/`
|
||||
- [x] Implement `tfIdfScore`, `bm25Score` in `src/text/scorer/`
|
||||
- [x] Each module: `.ts`, `.md`, `.test.ts`, `index.ts`
|
||||
|
||||
### 5. In-memory engine (`src/engine/memory-engine/`)
|
||||
|
||||
- [x] Implement inverted-index full-text search with configurable `FieldConfig`
|
||||
- [x] Implement BM25 scoring
|
||||
- [x] Implement trigram-based fuzzy matching
|
||||
- [x] Implement faceted search (`facetable` fields → bucket counts)
|
||||
- [x] Implement highlight extraction
|
||||
- [x] Implement `suggest` prefix search
|
||||
- [x] Integrate `@suaveplan/pagination` cursor and offset pagination on results
|
||||
- [x] Write `memory-engine.md`, `.types.ts`, `.test.ts`, `index.ts`
|
||||
|
||||
### 6. `SearchEngine` interface (`src/engine/search-engine/`)
|
||||
|
||||
- [x] Define and export the generic `SearchEngine<T>` interface
|
||||
- [x] Write `search-engine.md`, `.types.ts`, `.test.ts`, `index.ts`
|
||||
|
||||
### 7. OTel instrumentation
|
||||
|
||||
- [x] Add spans, counters, histograms per `proposal.md` §4
|
||||
|
||||
### 8. Tests
|
||||
|
||||
- [x] Unit tests co-located with each feature module (100% coverage target)
|
||||
- [x] `tests/integration.test.ts` — index → search → paginate round-trip
|
||||
- [x] `tests/property.test.ts` — fast-check properties per `proposal.md` §5
|
||||
|
||||
### 9. Benchmarks (`src/__benchmarks__/search.bench.ts`)
|
||||
|
||||
- [x] Implement benchmark cases per `proposal.md` §6
|
||||
|
||||
### 10. Docs
|
||||
|
||||
- [x] Every `.ts` implementation file has a sibling `.md`
|
||||
- [x] Root `README.md`
|
||||
|
||||
### 11. Pre-publish gate
|
||||
|
||||
- [x] `bunx turbo lint typecheck test --filter=@suaveplan/search` green
|
||||
- [x] `bunx turbo test:coverage --filter=@suaveplan/search` ≥ 100% on touched modules
|
||||
|
||||
### 12. Publish
|
||||
|
||||
- [x] `bun scripts/publish.ts --package @suaveplan/search`
|
||||
|
||||
### 13. Archive
|
||||
|
||||
- [x] Fold spec delta into `openspec/specs/data/spec.md`
|
||||
- [x] `mv openspec/changes/enterprise-search openspec/archive/2026-04-24-enterprise-search`
|
||||
Reference in New Issue
Block a user