Initial Commit
This commit is contained in:
111
packages/localization/i18n/i18n-formatter-segmenter/README.md
Normal file
111
packages/localization/i18n/i18n-formatter-segmenter/README.md
Normal file
@@ -0,0 +1,111 @@
|
||||
# @suaveplan/i18n-formatter-segmenter - Intl.Segmenter wrapper for the @suaveplan/i18n family — locale-aware text segmentation by grapheme, word, or sentence (compound-emoji-aware grapheme boundaries, ICU word-boundary rules, sentence segmentation). Per-(locale, granularity) cache, Zod-validated options, OTel histogram + cache-hit counter, structured I18nError mapping. Brand-new formatter — closes the gap from the original i18n-formatters change scope. Universal leaf package.
|
||||
|
||||
## Summary
|
||||
|
||||
Wrapper around `Intl.Segmenter` for the `@suaveplan/i18n` family —
|
||||
locale-aware text segmentation by grapheme, word, or sentence.
|
||||
Compound emoji count as a single grapheme; word boundaries follow
|
||||
ICU rules (handles non-space-separated scripts like Thai / Khmer /
|
||||
Japanese correctly); sentence boundaries follow ICU sentence-break
|
||||
rules.
|
||||
|
||||
This is a **brand-new formatter** in the i18n family — `Intl.Segmenter`
|
||||
was listed in the original `i18n-formatters` change scope but was
|
||||
never implemented in the monolith. This package closes that gap.
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
## Documentation tree
|
||||
|
||||
- [formatter](./src/formatter/formatter.md) — `SegmenterFormatter` and the `segmentText` / `formatSegments` standalone helpers expose `Intl.Segmenter` for the `@suav…
|
||||
- [formatter](./src/formatter/formatter.md) — `SegmenterFormatter` and the `segmentText` / `formatSegments` standalone helpers expose `Intl.Segmenter` for the `@suav…
|
||||
- [helpers](./src/helpers/helpers.md) — The three ergonomic helpers (`countGraphemes`, `splitWords`, `splitSentences`) pre-bind `segmentText` and `formatSegmen…
|
||||
- [helpers](./src/helpers/helpers.md) — The three ergonomic helpers (`countGraphemes`, `splitWords`, `splitSentences`) pre-bind `segmentText` and `formatSegmen…
|
||||
- [observability](./src/observability/observability.md) — The Segmenter formatter exports two OpenTelemetry instruments and a small in-memory state ring so spec-conformance test…
|
||||
- [observability](./src/observability/observability.md) — The Segmenter formatter exports two OpenTelemetry instruments and a small in-memory state ring so spec-conformance test…
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Why this exists
|
||||
|
||||
The 14-package i18n modular split (cross-cutting design
|
||||
`2026-05-05-i18n-modular-split.md`) lifts each `Intl.*` wrapper into
|
||||
its own sibling package and adds the two formatters
|
||||
(`Collator` + `Segmenter`) that the original change scope listed but
|
||||
never shipped. Locale-aware segmentation is the right primitive for
|
||||
character-limited inputs (tweet-style counters), full-text search
|
||||
tokenisation, and sentence-aware UI truncation.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bun add @suaveplan/i18n-formatter-segmenter
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```ts
|
||||
import {
|
||||
SegmenterFormatter,
|
||||
segmentText,
|
||||
formatSegments,
|
||||
countGraphemes,
|
||||
splitWords,
|
||||
splitSentences,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
|
||||
segmentText("ab", "en-GB"); // ["a", "b"]
|
||||
segmentText("hello world", "en-GB", { granularity: "word" });
|
||||
// ["hello", " ", "world"]
|
||||
|
||||
countGraphemes("👨👩👧", "en-GB"); // 1 (single grapheme)
|
||||
splitWords("hello, world!", "en-GB"); // ["hello", "world"]
|
||||
splitSentences("Hi. How are you?", "en-GB");
|
||||
// ["Hi.", "How are you?"]
|
||||
|
||||
const f = new SegmenterFormatter({ granularity: "word" });
|
||||
f.segmentWithMetadata("hi!", "en-GB");
|
||||
// [
|
||||
// { segment: "hi", index: 0, isWordLike: true },
|
||||
// { segment: "!", index: 2, isWordLike: false },
|
||||
// ]
|
||||
```
|
||||
|
||||
## Modules
|
||||
|
||||
| Subpath | Description |
|
||||
|---|---|
|
||||
| `@suaveplan/i18n-formatter-segmenter` | Root barrel — class + standalones + helpers + observability |
|
||||
| `@suaveplan/i18n-formatter-segmenter/formatter` | `SegmenterFormatter`, `segmentText`, `formatSegments`, types, schema |
|
||||
| `@suaveplan/i18n-formatter-segmenter/helpers` | `countGraphemes`, `splitWords`, `splitSentences` |
|
||||
|
||||
## Modules Documentation
|
||||
|
||||
- [formatter.md](src/formatter.md) — `SegmenterFormatter`, `segmentText`, `formatSegments`, `clearSegmenterCache`
|
||||
- [helpers.md](src/helpers.md) — three ergonomic helpers
|
||||
- [observability.md](src/observability.md) — OTel histogram + cache-hit counter
|
||||
|
||||
## Public Surface (snapshot)
|
||||
|
||||
| Export | Kind | Origin |
|
||||
|---|---|---|
|
||||
| `SegmenterFormatter` | class | `formatter.ts` |
|
||||
| `segmentText` | function | `formatter.ts` |
|
||||
| `formatSegments` | function | `formatter.ts` |
|
||||
| `clearSegmenterCache` | function | `formatter.ts` |
|
||||
| `SegmenterFormatterOptionsSchema` | Zod schema | `formatter.schemas.ts` |
|
||||
| `SegmenterFormatterOptions` | type | `formatter.types.ts` |
|
||||
| `SegmenterGranularity` | type | `formatter.types.ts` |
|
||||
| `SegmentData` | type | `formatter.types.ts` |
|
||||
| `SegmenterFormatterCacheKey` | type | `formatter.types.ts` |
|
||||
| `countGraphemes` | function | `helpers.ts` |
|
||||
| `splitWords` | function | `helpers.ts` |
|
||||
| `splitSentences` | function | `helpers.ts` |
|
||||
| `recordSegmenterFormat` | function | `observability.ts` |
|
||||
| `recordSegmenterCacheHit` | function | `observability.ts` |
|
||||
| `getSegmenterMetricsState` | function | `observability.ts` |
|
||||
| `resetSegmenterMetricsState` | function | `observability.ts` |
|
||||
|
||||
## Versioning
|
||||
|
||||
Part of the 14-package i18n modular-split rollout. Stays at `0.1.0`
|
||||
until the slim core (Wave 6) cuts the family `1.0.0` release.
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"name": "@suaveplan/i18n-formatter-segmenter",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"sideEffects": false,
|
||||
"description": "Intl.Segmenter wrapper for the @suaveplan/i18n family — locale-aware text segmentation by grapheme, word, or sentence (compound-emoji-aware grapheme boundaries, ICU word-boundary rules, sentence segmentation). Per-(locale, granularity) cache, Zod-validated options, OTel histogram + cache-hit counter, structured I18nError mapping. Brand-new formatter — closes the gap from the original i18n-formatters change scope. Universal leaf package.",
|
||||
"license": "MIT",
|
||||
"author": "Suaveplan Team",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
"./package.json": "./package.json",
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"./formatter": {
|
||||
"import": {
|
||||
"types": "./dist/formatter/index.d.ts",
|
||||
"default": "./dist/formatter/index.js"
|
||||
}
|
||||
},
|
||||
"./helpers": {
|
||||
"import": {
|
||||
"types": "./dist/helpers/index.d.ts",
|
||||
"default": "./dist/helpers/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"test": "vitest run --passWithNoTests",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage --passWithNoTests",
|
||||
"lint": "bunx biome check src/",
|
||||
"lint:fix": "bunx biome lint --fix --unsafe --max-diagnostics 500 src/",
|
||||
"format": "bunx biome format --write --max-diagnostics 500 src/",
|
||||
"check": "bunx biome check --write --unsafe --max-diagnostics 500 src/",
|
||||
"typecheck": "bunx tsc --noEmit --composite false -p tsconfig.lib.json",
|
||||
"docs:check": "bun run ../../../../scripts/docs-check.ts ."
|
||||
},
|
||||
"dependencies": {
|
||||
"@suaveplan/error": "workspace:^",
|
||||
"@suaveplan/i18n-shared": "workspace:^",
|
||||
"@suaveplan/types": "workspace:^",
|
||||
"@opentelemetry/api": "catalog:observability",
|
||||
"tslib": "catalog:build",
|
||||
"zod": "catalog:forms"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@suaveplan/testing": "workspace:^",
|
||||
"@suaveplan/typescript-config": "workspace:^",
|
||||
"@types/node": "catalog:",
|
||||
"@vitest/coverage-istanbul": "catalog:testing",
|
||||
"typescript": "catalog:",
|
||||
"vite": "catalog:build",
|
||||
"vite-plugin-dts": "catalog:build",
|
||||
"vitest": "catalog:testing"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://npmjs.fq.io/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"src",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import z from "zod";
|
||||
|
||||
/**
|
||||
* Zod schema for `SegmenterFormatterOptions`. `granularity` validates
|
||||
* against the canonical `Intl.Segmenter` enum.
|
||||
*/
|
||||
export const SegmenterFormatterOptionsSchema = z.object({
|
||||
granularity: z.enum(["grapheme", "word", "sentence"]).optional(),
|
||||
});
|
||||
|
||||
export type SegmenterFormatterOptionsInput = z.input<
|
||||
typeof SegmenterFormatterOptionsSchema
|
||||
>;
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* Spec-conformance pins for SegmenterFormatter — Zod rejection paths,
|
||||
* Intl.RangeError mapping, OTel emission, and cache reuse semantics.
|
||||
*/
|
||||
|
||||
import { I18N_CODES, I18nError } from "@suaveplan/i18n-shared";
|
||||
import { describe, expect, expectTypeOf, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
clearSegmenterCache,
|
||||
SegmenterFormatter,
|
||||
segmentText,
|
||||
} from "./formatter/formatter.js";
|
||||
import type {
|
||||
SegmenterFormatterOptions,
|
||||
SegmenterGranularity,
|
||||
} from "./formatter.types.js";
|
||||
import {
|
||||
getSegmenterMetricsState,
|
||||
resetSegmenterMetricsState,
|
||||
} from "./observability/observability.js";
|
||||
|
||||
describe("SegmenterFormatter spec-conformance", () => {
|
||||
it("rejects non-object options shape (no path → field='options')", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
// @ts-expect-error wrong type for safeParse
|
||||
segmentText("a", "en-GB", "not-an-object");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(I18nError);
|
||||
expect((err as I18nError).code).toBe(
|
||||
I18N_CODES.INVALID_FORMATTER_OPTIONS
|
||||
);
|
||||
expect((err as I18nError).context).toMatchObject({
|
||||
field: "options",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects unknown granularity with INVALID_FORMATTER_OPTIONS", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
new SegmenterFormatter({ granularity: "magic" as never });
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(I18nError);
|
||||
expect((err as I18nError).code).toBe(
|
||||
I18N_CODES.INVALID_FORMATTER_OPTIONS
|
||||
);
|
||||
expect((err as I18nError).context).toMatchObject({
|
||||
field: "granularity",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("maps Intl RangeError to UNSUPPORTED_LOCALE", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
try {
|
||||
segmentText("a", "en US");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(I18nError);
|
||||
expect((err as I18nError).code).toBe(I18N_CODES.UNSUPPORTED_LOCALE);
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses cached Intl.Segmenter on identical key", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("a", "en-GB");
|
||||
segmentText("a", "en-GB");
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.cacheHitSamples).toHaveLength(1);
|
||||
expect(state.cacheHitSamples[0]?.attributes.formatter).toBe(
|
||||
"segmenter"
|
||||
);
|
||||
});
|
||||
|
||||
it("does not reuse cache across distinct granularities", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("a b", "en-GB", { granularity: "grapheme" });
|
||||
segmentText("a b", "en-GB", { granularity: "word" });
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.cacheHitSamples).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("records a duration_ms sample per call with locale + granularity", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("hello", "en-GB", { granularity: "word" });
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.durationSamples).toHaveLength(1);
|
||||
expect(state.durationSamples[0]?.attributes).toMatchObject({
|
||||
locale: "en-GB",
|
||||
granularity: "word",
|
||||
});
|
||||
});
|
||||
|
||||
it("defaults granularity to 'grapheme' when omitted", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("ab", "en-GB");
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.durationSamples[0]?.attributes.granularity).toBe(
|
||||
"grapheme"
|
||||
);
|
||||
});
|
||||
|
||||
it("granularity field of options matches SegmenterGranularity", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
expectTypeOf<SegmenterFormatterOptions["granularity"]>().toEqualTypeOf<
|
||||
SegmenterGranularity | undefined
|
||||
>();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
/** Segmenter formatter type definitions per ECMA-402 Intl.Segmenter. */
|
||||
|
||||
/** Segmentation granularity. */
|
||||
export type SegmenterGranularity = "grapheme" | "word" | "sentence";
|
||||
|
||||
/**
|
||||
* Public options shape consumed by `SegmenterFormatter`,
|
||||
* `segmentText`, and `formatSegments`. Every field is optional;
|
||||
* `granularity` defaults to `"grapheme"` per `Intl.Segmenter` defaults.
|
||||
*/
|
||||
export interface SegmenterFormatterOptions {
|
||||
readonly granularity?: SegmenterGranularity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-segment metadata returned by `formatSegments`. Mirrors
|
||||
* `Intl.SegmentData`:
|
||||
*
|
||||
* - `segment` — the substring text
|
||||
* - `index` — UTF-16 index into the source string
|
||||
* - `isWordLike` — present only for `granularity: "word"`; `true` for
|
||||
* word-like segments and `false` for whitespace / punctuation
|
||||
*/
|
||||
export interface SegmentData {
|
||||
readonly segment: string;
|
||||
readonly index: number;
|
||||
readonly isWordLike?: boolean;
|
||||
}
|
||||
|
||||
/** Cache key shape: `${locale}|${granularity}`. */
|
||||
export type SegmenterFormatterCacheKey = string;
|
||||
@@ -0,0 +1,127 @@
|
||||
# SegmenterFormatter
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
`SegmenterFormatter` and the `segmentText` / `formatSegments`
|
||||
standalone helpers expose `Intl.Segmenter` for the `@suaveplan/i18n`
|
||||
family at the canonical Wave-3 formatter surface. Locale-aware text
|
||||
segmentation by grapheme, word, or sentence is the right primitive
|
||||
for emoji-aware character counts, ICU word-boundary tokenisation,
|
||||
and sentence splitting.
|
||||
|
||||
This is a brand-new formatter — `Intl.Segmenter` was listed in the
|
||||
original `i18n-formatters` change scope but never implemented in the
|
||||
monolith. The split package closes that gap with the canonical Wave-3
|
||||
shape.
|
||||
|
||||
## Features
|
||||
|
||||
- `SegmenterFormatter` class with `segment(input, locale)` and `segmentWithMetadata(input, locale)` methods
|
||||
- `segmentText(input, locale, options?)` standalone — returns plain `string[]`
|
||||
- `formatSegments(input, locale, options?)` standalone — returns `SegmentData[]` with `segment` / `index` / `isWordLike` metadata
|
||||
- Three granularity values: `"grapheme"` (default), `"word"`, `"sentence"`
|
||||
- Construction-time Zod validation with `INVALID_FORMATTER_OPTIONS` mapping
|
||||
- `Intl.Segmenter` `RangeError` mapped to `UNSUPPORTED_LOCALE` with original error attached as `cause`
|
||||
- Runtime-availability check: when the host runtime lacks `Intl.Segmenter`, throws `I18nError(UNSUPPORTED_LOCALE)` with message key `intl.segmenter.unavailable`
|
||||
- Per-(locale, granularity) cache — repeated calls reuse the same `Intl.Segmenter`
|
||||
- `clearSegmenterCache()` test-only helper
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
formatSegments,
|
||||
segmentText,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
|
||||
segmentText("ab", "en-GB"); // ["a", "b"]
|
||||
segmentText("hello world", "en-GB", { granularity: "word" });
|
||||
// ["hello", " ", "world"]
|
||||
|
||||
formatSegments("hi!", "en-GB", { granularity: "word" });
|
||||
// [
|
||||
// { segment: "hi", index: 0, isWordLike: true },
|
||||
// { segment: "!", index: 2, isWordLike: false },
|
||||
// ]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
SegmenterFormatter,
|
||||
type SegmenterFormatterOptions,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
|
||||
// Emoji-aware character counter — handles compound emoji
|
||||
// (family-of-three ZWJ sequence is ONE grapheme, not 7+ code units).
|
||||
const f = new SegmenterFormatter({ granularity: "grapheme" });
|
||||
const count = f.segment("👨👩👧 hello", "en-GB").length;
|
||||
// count === 8 (1 emoji + space + 5 letters + ... locale-correct)
|
||||
|
||||
// Build a word-level token stream for full-text search indexing.
|
||||
const search = new SegmenterFormatter({ granularity: "word" });
|
||||
const tokens = search
|
||||
.segmentWithMetadata("Quick brown fox.", "en-GB")
|
||||
.filter((s) => s.isWordLike === true)
|
||||
.map((s) => s.segment.toLowerCase());
|
||||
// ["quick", "brown", "fox"]
|
||||
|
||||
// Sentence-aware truncation for chat previews.
|
||||
const sentences = new SegmenterFormatter({ granularity: "sentence" });
|
||||
const first = sentences.segment(
|
||||
"Hello there. How are you today?",
|
||||
"en-GB"
|
||||
)[0];
|
||||
// "Hello there. "
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `segmentText(input, locale, options?): string[]`
|
||||
|
||||
Standalone helper. Returns a plain `string[]` of segments. Default
|
||||
`granularity` is `"grapheme"`. Throws
|
||||
`I18nError(INVALID_FORMATTER_OPTIONS)` on Zod failure;
|
||||
`I18nError(UNSUPPORTED_LOCALE)` on runtime unavailability or
|
||||
`Intl.Segmenter` `RangeError`.
|
||||
|
||||
### `formatSegments(input, locale, options?): SegmentData[]`
|
||||
|
||||
Standalone helper. Returns `SegmentData[]` preserving each segment's
|
||||
`index` and (for `granularity: "word"`) `isWordLike` flag. The
|
||||
`isWordLike` field is omitted on `grapheme` and `sentence`
|
||||
granularities (matches `Intl.Segmenter` semantics).
|
||||
|
||||
### `class SegmenterFormatter`
|
||||
|
||||
- `constructor(options?: SegmenterFormatterOptions)` — Zod-validates options
|
||||
- `segment(input, locale): string[]`
|
||||
- `segmentWithMetadata(input, locale): SegmentData[]`
|
||||
|
||||
### `clearSegmenterCache(): void`
|
||||
|
||||
Test-only helper that empties the per-(locale, granularity) cache.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The cache key is `${locale}|${granularity}` — only two dimensions
|
||||
because `Intl.Segmenter` accepts only `granularity` as a configurable
|
||||
option. Cache size stays small even with many locales.
|
||||
- Runtime-availability guard checks `typeof Intl.Segmenter === "function"`
|
||||
before construction. This catches Node < 16 / very-old browsers /
|
||||
test stubs that null out `Intl.Segmenter`. The error code reuses
|
||||
`UNSUPPORTED_LOCALE` (the closest fit in the `I18N_CODES`
|
||||
registry) with a discriminating message key
|
||||
`intl.segmenter.unavailable` so consumers can branch on the message.
|
||||
- `Intl.Segmenter` shipped in Node 16+, Bun, and every current major
|
||||
browser. The package does not provide a polyfill.
|
||||
- The `segmentWithMetadata` method exists on the class as the
|
||||
programmatic-friendly companion to the standalone `formatSegments`
|
||||
— both produce identical output.
|
||||
- Compound emoji (e.g. `"👨👩👧"`, the family-of-three ZWJ sequence)
|
||||
count as a single grapheme per ECMA-402.
|
||||
@@ -0,0 +1,196 @@
|
||||
import { I18nError } from "@suaveplan/i18n-shared";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
clearSegmenterCache,
|
||||
formatSegments,
|
||||
SegmenterFormatter,
|
||||
segmentText,
|
||||
} from "./formatter.js";
|
||||
|
||||
describe("segmentText", () => {
|
||||
it("default granularity (grapheme) splits ascii by character", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
expect(segmentText("ab", "en-GB")).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("segmentText word granularity returns word segments", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const out = segmentText("hello world", "en-GB", {
|
||||
granularity: "word",
|
||||
});
|
||||
expect(out).toContain("hello");
|
||||
expect(out).toContain("world");
|
||||
});
|
||||
|
||||
it("sentence granularity splits on period", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
const out = segmentText("Hi. How are you?", "en-GB", {
|
||||
granularity: "sentence",
|
||||
});
|
||||
expect(out.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("compound emoji is a single grapheme segment", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
const out = segmentText("👨👩👧", "en-GB", { granularity: "grapheme" });
|
||||
expect(out).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("segmentText output twins SegmenterFormatter.segment output", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const opts = { granularity: "word" as const };
|
||||
const standalone = segmentText("hello world", "en-GB", opts);
|
||||
const klass = new SegmenterFormatter(opts).segment(
|
||||
"hello world",
|
||||
"en-GB"
|
||||
);
|
||||
expect(standalone).toEqual(klass);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SegmenterFormatter class", () => {
|
||||
it("SegmenterFormatter segments with default options", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const f = new SegmenterFormatter();
|
||||
expect(f.segment("ab", "en-GB")).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("SegmenterFormatter rejects invalid options with INVALID_FORMATTER_OPTIONS", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
try {
|
||||
new SegmenterFormatter({ granularity: "invalid" as never });
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(I18nError);
|
||||
expect((err as I18nError).code).toBe(
|
||||
"I18N.INVALID_FORMATTER_OPTIONS"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("SegmenterFormatter.segmentWithMetadata returns SegmentData entries", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const f = new SegmenterFormatter({ granularity: "word" });
|
||||
const out = f.segmentWithMetadata("hi!", "en-GB");
|
||||
expect(out.length).toBeGreaterThan(0);
|
||||
expect(typeof out[0]?.segment).toBe("string");
|
||||
expect(typeof out[0]?.index).toBe("number");
|
||||
});
|
||||
|
||||
it("SegmenterFormatter constructed with no options", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const f = new SegmenterFormatter();
|
||||
const out = f.segmentWithMetadata("ab", "en-GB");
|
||||
expect(out).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatSegments", () => {
|
||||
it("formatSegments preserves index and isWordLike metadata", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const out = formatSegments("hi!", "en-GB", { granularity: "word" });
|
||||
const word = out.find((s) => s.segment === "hi");
|
||||
const punct = out.find((s) => s.segment === "!");
|
||||
expect(word).toBeDefined();
|
||||
expect(word?.index).toBe(0);
|
||||
expect(word?.isWordLike).toBe(true);
|
||||
expect(punct).toBeDefined();
|
||||
expect(punct?.isWordLike).toBe(false);
|
||||
});
|
||||
|
||||
it("formatSegments grapheme returns no isWordLike field", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const out = formatSegments("ab", "en-GB", { granularity: "grapheme" });
|
||||
expect(out[0]?.isWordLike).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Intl.Segmenter RangeError mapping", () => {
|
||||
it("unsupported locale throws UNSUPPORTED_LOCALE with cause", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
try {
|
||||
segmentText("a", "xx-bogus-tag-!!!");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(I18nError);
|
||||
expect((err as I18nError).code).toBe("I18N.UNSUPPORTED_LOCALE");
|
||||
expect((err as I18nError).cause).toBeInstanceOf(RangeError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("Intl.Segmenter availability check", () => {
|
||||
it("throws UNSUPPORTED_LOCALE when Intl.Segmenter is not a function", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const original = Intl.Segmenter;
|
||||
// Replace Intl.Segmenter with a non-function value to exercise
|
||||
// the runtime-availability guard.
|
||||
(Intl as unknown as { Segmenter: unknown }).Segmenter = undefined;
|
||||
try {
|
||||
segmentText("a", "en-GB");
|
||||
expect.fail("expected throw");
|
||||
} catch (err) {
|
||||
expect(err).toBeInstanceOf(I18nError);
|
||||
expect((err as I18nError).code).toBe("I18N.UNSUPPORTED_LOCALE");
|
||||
expect((err as I18nError).message).toContain(
|
||||
"intl.segmenter.unavailable"
|
||||
);
|
||||
} finally {
|
||||
(
|
||||
Intl as unknown as { Segmenter: typeof Intl.Segmenter }
|
||||
).Segmenter = original;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("Per-(locale, granularity) cache", () => {
|
||||
it("identical key reuses cached Intl.Segmenter", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
const a = segmentText("a", "en-GB");
|
||||
const b = segmentText("a", "en-GB");
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it("distinct granularity creates distinct cache entries", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const grapheme = segmentText("hello", "en-GB", {
|
||||
granularity: "grapheme",
|
||||
});
|
||||
const word = segmentText("hello", "en-GB", { granularity: "word" });
|
||||
// grapheme splits "hello" into 5 single-char graphemes;
|
||||
// word treats it as a single word segment.
|
||||
expect(grapheme).toHaveLength(5);
|
||||
expect(word).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("clearSegmenterCache drops cached entries", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
segmentText("a", "en-GB");
|
||||
clearSegmenterCache();
|
||||
// Subsequent call still works.
|
||||
expect(segmentText("a", "en-GB")).toEqual(["a"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { I18N_CODES, I18nError } from "@suaveplan/i18n-shared";
|
||||
import { SegmenterFormatterOptionsSchema } from "../formatter.schemas.js";
|
||||
import type {
|
||||
SegmentData,
|
||||
SegmenterFormatterCacheKey,
|
||||
SegmenterFormatterOptions,
|
||||
SegmenterGranularity,
|
||||
} from "../formatter.types.js";
|
||||
import {
|
||||
recordSegmenterCacheHit,
|
||||
recordSegmenterFormat,
|
||||
} from "../observability/observability.js";
|
||||
|
||||
const cache = new Map<SegmenterFormatterCacheKey, Intl.Segmenter>();
|
||||
|
||||
/**
|
||||
* Probe `Intl.Segmenter` availability. Throws
|
||||
* `I18nError(UNSUPPORTED_LOCALE)` (the closest fit in the
|
||||
* `I18N_CODES` registry) with the message key
|
||||
* `intl.segmenter.unavailable` when the host runtime lacks
|
||||
* `Intl.Segmenter`. Older runtimes (Node < 16) hit this path.
|
||||
*/
|
||||
function ensureSegmenterAvailable(): void {
|
||||
if (typeof Intl.Segmenter === "function") return;
|
||||
throw new I18nError(
|
||||
I18N_CODES.UNSUPPORTED_LOCALE,
|
||||
"intl.segmenter.unavailable",
|
||||
{
|
||||
context: {
|
||||
reason: "Intl.Segmenter is not available in this runtime",
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function buildKey(
|
||||
locale: string,
|
||||
granularity: SegmenterGranularity
|
||||
): SegmenterFormatterCacheKey {
|
||||
return `${locale}|${granularity}`;
|
||||
}
|
||||
|
||||
function validateOptions(options: SegmenterFormatterOptions | undefined): void {
|
||||
if (options === undefined) return;
|
||||
const parsed = SegmenterFormatterOptionsSchema.safeParse(options);
|
||||
if (parsed.success) return;
|
||||
const field = String(parsed.error.issues[0]?.path[0] ?? "options");
|
||||
throw new I18nError(
|
||||
I18N_CODES.INVALID_FORMATTER_OPTIONS,
|
||||
`SegmenterFormatter: invalid options (field: ${field})`,
|
||||
{ context: { field, issues: parsed.error.issues } }
|
||||
);
|
||||
}
|
||||
|
||||
function buildIntlSegmenter(
|
||||
locale: string,
|
||||
granularity: SegmenterGranularity
|
||||
): Intl.Segmenter {
|
||||
try {
|
||||
return new Intl.Segmenter(locale, { granularity });
|
||||
} catch (cause) {
|
||||
throw new I18nError(
|
||||
I18N_CODES.UNSUPPORTED_LOCALE,
|
||||
`SegmenterFormatter: unsupported locale '${locale}'`,
|
||||
{ cause, context: { locale } }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function getOrCreate(
|
||||
locale: string,
|
||||
granularity: SegmenterGranularity
|
||||
): { segmenter: Intl.Segmenter; cached: boolean } {
|
||||
const key = buildKey(locale, granularity);
|
||||
const existing = cache.get(key);
|
||||
if (existing) return { segmenter: existing, cached: true };
|
||||
const segmenter = buildIntlSegmenter(locale, granularity);
|
||||
cache.set(key, segmenter);
|
||||
return { segmenter, cached: false };
|
||||
}
|
||||
|
||||
function emit(
|
||||
locale: string,
|
||||
granularity: SegmenterGranularity,
|
||||
cachedHit: boolean,
|
||||
t0: number
|
||||
): void {
|
||||
if (cachedHit) recordSegmenterCacheHit();
|
||||
recordSegmenterFormat(locale, granularity, performance.now() - t0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Locale-aware segmentation that returns the segments as a plain
|
||||
* `string[]`. `granularity` defaults to `"grapheme"` (per
|
||||
* `Intl.Segmenter` defaults).
|
||||
*/
|
||||
export function segmentText(
|
||||
input: string,
|
||||
locale: string,
|
||||
options?: SegmenterFormatterOptions
|
||||
): string[] {
|
||||
ensureSegmenterAvailable();
|
||||
validateOptions(options);
|
||||
const granularity: SegmenterGranularity =
|
||||
options?.granularity ?? "grapheme";
|
||||
const t0 = performance.now();
|
||||
const { segmenter, cached: hit } = getOrCreate(locale, granularity);
|
||||
const out: string[] = [];
|
||||
for (const seg of segmenter.segment(input)) {
|
||||
out.push(seg.segment);
|
||||
}
|
||||
emit(locale, granularity, hit, t0);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Locale-aware segmentation that returns full `SegmentData[]` with
|
||||
* `index` and `isWordLike` metadata preserved.
|
||||
*/
|
||||
export function formatSegments(
|
||||
input: string,
|
||||
locale: string,
|
||||
options?: SegmenterFormatterOptions
|
||||
): SegmentData[] {
|
||||
ensureSegmenterAvailable();
|
||||
validateOptions(options);
|
||||
const granularity: SegmenterGranularity =
|
||||
options?.granularity ?? "grapheme";
|
||||
const t0 = performance.now();
|
||||
const { segmenter, cached: hit } = getOrCreate(locale, granularity);
|
||||
const out: SegmentData[] = [];
|
||||
for (const seg of segmenter.segment(input)) {
|
||||
const entry: SegmentData = {
|
||||
segment: seg.segment,
|
||||
index: seg.index,
|
||||
...(typeof (seg as { isWordLike?: boolean }).isWordLike ===
|
||||
"boolean"
|
||||
? { isWordLike: (seg as { isWordLike: boolean }).isWordLike }
|
||||
: {}),
|
||||
};
|
||||
out.push(entry);
|
||||
}
|
||||
emit(locale, granularity, hit, t0);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Class wrapper for `segmentText` and `formatSegments` with
|
||||
* construction-time option validation and stored defaults.
|
||||
*/
|
||||
export class SegmenterFormatter {
|
||||
private readonly options: SegmenterFormatterOptions | undefined;
|
||||
|
||||
public constructor(options?: SegmenterFormatterOptions) {
|
||||
validateOptions(options);
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
public segment(input: string, locale: string): string[] {
|
||||
return segmentText(input, locale, this.options);
|
||||
}
|
||||
|
||||
public segmentWithMetadata(input: string, locale: string): SegmentData[] {
|
||||
return formatSegments(input, locale, this.options);
|
||||
}
|
||||
}
|
||||
|
||||
/** Test-only: clear the per-(locale, granularity) cache. */
|
||||
export function clearSegmenterCache(): void {
|
||||
cache.clear();
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
clearSegmenterCache,
|
||||
formatSegments,
|
||||
SegmenterFormatter,
|
||||
segmentText,
|
||||
} from "./formatter.js";
|
||||
@@ -0,0 +1,105 @@
|
||||
# Segmenter ergonomic helpers
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
The three ergonomic helpers (`countGraphemes`, `splitWords`,
|
||||
`splitSentences`) pre-bind `segmentText` and `formatSegments` with
|
||||
the appropriate granularity for the most common UI patterns. They
|
||||
exist because counting graphemes (for character limits) and splitting
|
||||
words (for token streams) are universal needs that should not require
|
||||
remembering the `granularity` discriminant.
|
||||
|
||||
## Features
|
||||
|
||||
- `countGraphemes(input, locale)` — emoji-aware grapheme count
|
||||
- `splitWords(input, locale)` — word-segments only (filters non-word-like)
|
||||
- `splitSentences(input, locale)` — sentence-segments, trimmed, empty entries dropped
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
countGraphemes,
|
||||
splitSentences,
|
||||
splitWords,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
|
||||
countGraphemes("hello", "en-GB"); // 5
|
||||
countGraphemes("👨👩👧", "en-GB"); // 1 (single grapheme)
|
||||
|
||||
splitWords("hello, world!", "en-GB"); // ["hello", "world"]
|
||||
splitSentences("Hi. How are you?", "en-GB"); // ["Hi.", "How are you?"]
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import {
|
||||
countGraphemes,
|
||||
splitSentences,
|
||||
splitWords,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
|
||||
// Tweet-style character limit that respects compound emoji.
|
||||
const MAX_CHARS = 280;
|
||||
function tweetFits(text: string, locale: string) {
|
||||
return countGraphemes(text, locale) <= MAX_CHARS;
|
||||
}
|
||||
|
||||
tweetFits("👨👩👧 hello world", "en-GB"); // true (8 graphemes)
|
||||
|
||||
// Word-frequency map for full-text indexing.
|
||||
function wordFrequencies(input: string, locale: string) {
|
||||
const out: Record<string, number> = {};
|
||||
for (const word of splitWords(input, locale)) {
|
||||
const k = word.toLowerCase();
|
||||
out[k] = (out[k] ?? 0) + 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Sentence-aware preview that always ends at a sentence boundary.
|
||||
function preview(input: string, locale: string, maxSentences: number) {
|
||||
return splitSentences(input, locale).slice(0, maxSentences).join(" ");
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `countGraphemes(input, locale): number`
|
||||
|
||||
Returns the grapheme count. Uses `granularity: "grapheme"` so
|
||||
compound emoji and combining-character sequences count as one.
|
||||
|
||||
### `splitWords(input, locale): string[]`
|
||||
|
||||
Returns word-segments only. Internally calls `formatSegments` with
|
||||
`granularity: "word"` and filters segments where `isWordLike === true`.
|
||||
Punctuation, whitespace, and other non-word-like segments are
|
||||
dropped.
|
||||
|
||||
### `splitSentences(input, locale): string[]`
|
||||
|
||||
Returns sentence-segments. Each entry is trimmed of leading/trailing
|
||||
whitespace; empty entries are dropped. Punctuation that ends a
|
||||
sentence (`.`/`?`/`!`) is included with the preceding sentence per
|
||||
ICU sentence-boundary rules.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Each helper is a thin wrapper around `segmentText` / `formatSegments`.
|
||||
The underlying per-(locale, granularity) cache means calling
|
||||
`splitWords` and `splitSentences` for the same locale uses two
|
||||
cached `Intl.Segmenter` instances (one per granularity) — minimal
|
||||
per-call cost.
|
||||
- `splitSentences` trim-and-filter chain is deliberate: ICU's
|
||||
sentence-boundary rules emit each sentence WITH its trailing
|
||||
whitespace; trimming gives the consumer the expected "sentence
|
||||
text" without trailing space, and filtering drops the trailing
|
||||
empty segment that `Intl.Segmenter` sometimes emits for terminator-
|
||||
only input.
|
||||
- The helpers are pure functions with no shared state.
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import { clearSegmenterCache } from "../formatter/formatter.js";
|
||||
import { countGraphemes, splitSentences, splitWords } from "./helpers.js";
|
||||
|
||||
describe("ergonomic segmenter helpers", () => {
|
||||
it("countGraphemes treats compound emoji as one", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
expect(countGraphemes("👨👩👧", "en-GB")).toBe(1);
|
||||
});
|
||||
|
||||
it("countGraphemes counts ASCII characters", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
expect(countGraphemes("hello", "en-GB")).toBe(5);
|
||||
});
|
||||
|
||||
it("countGraphemes returns 0 for empty string", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
expect(countGraphemes("", "en-GB")).toBe(0);
|
||||
});
|
||||
|
||||
it("splitWords filters non-word-like segments", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
expect(splitWords("hello, world!", "en-GB")).toEqual([
|
||||
"hello",
|
||||
"world",
|
||||
]);
|
||||
});
|
||||
|
||||
it("splitWords returns empty array for whitespace-only input", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
expect(splitWords(" ", "en-GB")).toEqual([]);
|
||||
});
|
||||
|
||||
it("splitSentences returns sentence segments", { timeout: 5000 }, () => {
|
||||
clearSegmenterCache();
|
||||
const out = splitSentences("Hi. How are you?", "en-GB");
|
||||
expect(out).toHaveLength(2);
|
||||
expect(out[0]).toContain("Hi");
|
||||
expect(out[1]).toContain("How");
|
||||
});
|
||||
|
||||
it("splitSentences returns single sentence for input without delimiters", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
const out = splitSentences("hello world", "en-GB");
|
||||
expect(out).toHaveLength(1);
|
||||
expect(out[0]).toBe("hello world");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { formatSegments, segmentText } from "../formatter/formatter.js";
|
||||
|
||||
/**
|
||||
* Count the graphemes in a string (emoji-aware). A compound emoji
|
||||
* (e.g. the family-of-three ZWJ sequence `"👨👩👧"`) counts as a single
|
||||
* grapheme.
|
||||
*/
|
||||
export function countGraphemes(input: string, locale: string): number {
|
||||
return segmentText(input, locale, { granularity: "grapheme" }).length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a string into word-segments, dropping non-word-like segments
|
||||
* (whitespace, punctuation). Useful for word counts, tokenisers, and
|
||||
* search-term extraction.
|
||||
*/
|
||||
export function splitWords(input: string, locale: string): string[] {
|
||||
return formatSegments(input, locale, { granularity: "word" })
|
||||
.filter((seg) => seg.isWordLike === true)
|
||||
.map((seg) => seg.segment);
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a string into sentence-segments per ICU sentence-boundary
|
||||
* rules. Returns the full sentence text including trailing
|
||||
* punctuation; surrounding whitespace is trimmed and empty entries
|
||||
* are dropped.
|
||||
*/
|
||||
export function splitSentences(input: string, locale: string): string[] {
|
||||
return segmentText(input, locale, { granularity: "sentence" })
|
||||
.map((s) => s.trim())
|
||||
.filter((s) => s.length > 0);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
export {
|
||||
countGraphemes,
|
||||
splitSentences,
|
||||
splitWords,
|
||||
} from "./helpers.js";
|
||||
@@ -0,0 +1,89 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import * as root from "./index.js";
|
||||
|
||||
describe("@suaveplan/i18n-formatter-segmenter root barrel", () => {
|
||||
it("re-exports SegmenterFormatter", { timeout: 5000 }, () => {
|
||||
expect(root.SegmenterFormatter).toBeDefined();
|
||||
});
|
||||
|
||||
it("re-exports segmentText + formatSegments", { timeout: 5000 }, () => {
|
||||
expect(typeof root.segmentText).toBe("function");
|
||||
expect(typeof root.formatSegments).toBe("function");
|
||||
});
|
||||
|
||||
it("re-exports clearSegmenterCache + schema", { timeout: 5000 }, () => {
|
||||
expect(typeof root.clearSegmenterCache).toBe("function");
|
||||
expect(root.SegmenterFormatterOptionsSchema).toBeDefined();
|
||||
});
|
||||
|
||||
it("re-exports three ergonomic helpers", { timeout: 5000 }, () => {
|
||||
expect(typeof root.countGraphemes).toBe("function");
|
||||
expect(typeof root.splitWords).toBe("function");
|
||||
expect(typeof root.splitSentences).toBe("function");
|
||||
});
|
||||
|
||||
it("re-exports observability surface", { timeout: 5000 }, () => {
|
||||
expect(typeof root.recordSegmenterFormat).toBe("function");
|
||||
expect(typeof root.recordSegmenterCacheHit).toBe("function");
|
||||
expect(typeof root.getSegmenterMetricsState).toBe("function");
|
||||
expect(typeof root.resetSegmenterMetricsState).toBe("function");
|
||||
});
|
||||
|
||||
it("every documented public export resolves", { timeout: 5000 }, () => {
|
||||
const expected = [
|
||||
"SegmenterFormatter",
|
||||
"segmentText",
|
||||
"formatSegments",
|
||||
"clearSegmenterCache",
|
||||
"SegmenterFormatterOptionsSchema",
|
||||
"countGraphemes",
|
||||
"splitWords",
|
||||
"splitSentences",
|
||||
"recordSegmenterFormat",
|
||||
"recordSegmenterCacheHit",
|
||||
"getSegmenterMetricsState",
|
||||
"resetSegmenterMetricsState",
|
||||
];
|
||||
const exported = Object.keys(root);
|
||||
for (const name of expected) {
|
||||
expect(exported).toContain(name);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("@suaveplan/i18n-formatter-segmenter subpath imports", () => {
|
||||
it("every documented subpath export resolves", {
|
||||
timeout: 5000,
|
||||
}, async () => {
|
||||
const formatter = await import("./formatter/formatter.js");
|
||||
expect(formatter.SegmenterFormatter).toBeDefined();
|
||||
const helpers = await import("./helpers/helpers.js");
|
||||
expect(typeof helpers.countGraphemes).toBe("function");
|
||||
});
|
||||
});
|
||||
|
||||
const PACKAGE_JSON_PATH = fileURLToPath(
|
||||
new URL("../package.json", import.meta.url)
|
||||
);
|
||||
|
||||
describe("package.json invariants", () => {
|
||||
it("package.json declares sideEffects false", { timeout: 5000 }, () => {
|
||||
const raw = readFileSync(PACKAGE_JSON_PATH, "utf8");
|
||||
const pkg = JSON.parse(raw) as { sideEffects?: boolean };
|
||||
expect(pkg.sideEffects).toBe(false);
|
||||
});
|
||||
|
||||
it("package.json declares the three documented subpath exports", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
const raw = readFileSync(PACKAGE_JSON_PATH, "utf8");
|
||||
const pkg = JSON.parse(raw) as { exports?: Record<string, unknown> };
|
||||
const exports = pkg.exports ?? {};
|
||||
const required = [".", "./formatter", "./helpers"];
|
||||
for (const subpath of required) {
|
||||
expect(exports).toHaveProperty(subpath);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
// Root barrel for @suaveplan/i18n-formatter-segmenter. Explicit named
|
||||
// re-exports only — no `export *` per CLAUDE.md §10.
|
||||
|
||||
export {
|
||||
clearSegmenterCache,
|
||||
formatSegments,
|
||||
SegmenterFormatter,
|
||||
segmentText,
|
||||
} from "./formatter/index.js";
|
||||
export { SegmenterFormatterOptionsSchema } from "./formatter.schemas.js";
|
||||
export type {
|
||||
SegmentData,
|
||||
SegmenterFormatterCacheKey,
|
||||
SegmenterFormatterOptions,
|
||||
SegmenterGranularity,
|
||||
} from "./formatter.types.js";
|
||||
export {
|
||||
countGraphemes,
|
||||
splitSentences,
|
||||
splitWords,
|
||||
} from "./helpers/index.js";
|
||||
export {
|
||||
getSegmenterMetricsState,
|
||||
recordSegmenterCacheHit,
|
||||
recordSegmenterFormat,
|
||||
resetSegmenterMetricsState,
|
||||
type SegmenterMetricSample,
|
||||
type SegmenterMetricsState,
|
||||
} from "./observability/index.js";
|
||||
@@ -0,0 +1,10 @@
|
||||
export type {
|
||||
SegmenterMetricSample,
|
||||
SegmenterMetricsState,
|
||||
} from "./observability.js";
|
||||
export {
|
||||
getSegmenterMetricsState,
|
||||
recordSegmenterCacheHit,
|
||||
recordSegmenterFormat,
|
||||
resetSegmenterMetricsState,
|
||||
} from "./observability.js";
|
||||
@@ -0,0 +1,98 @@
|
||||
# Segmenter observability
|
||||
|
||||
<!-- begin:auto-docs -->
|
||||
_(no submodules)_
|
||||
<!-- end:auto-docs -->
|
||||
|
||||
## Purpose
|
||||
|
||||
The Segmenter formatter exports two OpenTelemetry instruments and a
|
||||
small in-memory state ring so spec-conformance tests can assert exact
|
||||
attribute payloads without standing up a full SDK exporter. The
|
||||
instrument names match the canonical Wave-3 formatter contract —
|
||||
every sibling formatter emits the same two metric names so dashboards
|
||||
and alerts compose cleanly across the family.
|
||||
|
||||
## Features
|
||||
|
||||
- Histogram `i18n.format.segmenter.duration_ms` (unit `ms`) — per-segment wall-clock time
|
||||
- Counter `i18n.format.cache_hit{formatter="segmenter"}` — increments on cache reuse only
|
||||
- In-memory state ring (`getSegmenterMetricsState()`) for spec-conformance
|
||||
- `resetSegmenterMetricsState()` test-only helper
|
||||
|
||||
## Usage: Basic Example
|
||||
|
||||
```ts
|
||||
import { metrics } from "@opentelemetry/api";
|
||||
import { MeterProvider } from "@opentelemetry/sdk-metrics";
|
||||
|
||||
const provider = new MeterProvider({ readers: [/* reader */] });
|
||||
metrics.setGlobalMeterProvider(provider);
|
||||
|
||||
import { segmentText } from "@suaveplan/i18n-formatter-segmenter";
|
||||
segmentText("hello", "en-GB", { granularity: "word" });
|
||||
// emits via your reader
|
||||
```
|
||||
|
||||
## Usage: Advanced Example
|
||||
|
||||
```ts
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
clearSegmenterCache,
|
||||
segmentText,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
import {
|
||||
getSegmenterMetricsState,
|
||||
resetSegmenterMetricsState,
|
||||
} from "@suaveplan/i18n-formatter-segmenter";
|
||||
|
||||
it("emits histogram + cache-hit counter", () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("hello", "en-GB", { granularity: "word" });
|
||||
segmentText("hello", "en-GB", { granularity: "word" });
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.durationSamples).toHaveLength(2);
|
||||
expect(state.cacheHitSamples).toHaveLength(1);
|
||||
expect(state.durationSamples[0]?.attributes).toMatchObject({
|
||||
locale: "en-GB",
|
||||
granularity: "word",
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### `recordSegmenterFormat(locale, granularity, durationMs): void`
|
||||
|
||||
Records the duration histogram sample. Tags: `locale`, `granularity`.
|
||||
Called internally by `segmentText` / `formatSegments`.
|
||||
|
||||
### `recordSegmenterCacheHit(): void`
|
||||
|
||||
Increments the cache-hit counter with the constant tag
|
||||
`{ formatter: "segmenter" }`.
|
||||
|
||||
### `getSegmenterMetricsState(): SegmenterMetricsState`
|
||||
|
||||
Returns the in-memory ring of `durationSamples` and
|
||||
`cacheHitSamples`. Used by spec-conformance tests.
|
||||
|
||||
### `resetSegmenterMetricsState(): void`
|
||||
|
||||
Clears both ring buffers. Test-only helper.
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- The OTel meter is named `suaveplan.i18n.formatter.segmenter`. Every
|
||||
formatter in the family has its own meter so OTel resource
|
||||
attributes can be tied per-package if a consumer chooses.
|
||||
- `granularity` defaults to `"grapheme"` in the histogram tag when
|
||||
the caller omits it; `locale` is always required so it's never
|
||||
defaulted.
|
||||
- Sample dashboard query (PromQL/Grafana over OTLP):
|
||||
`histogram_quantile(0.99, sum(rate(i18n_format_segmenter_duration_ms_bucket[5m])) by (le, locale, granularity))`
|
||||
- The in-memory ring is intentionally unbounded — tests are short
|
||||
and call `resetSegmenterMetricsState()` between cases. A
|
||||
long-running test loop should call reset to avoid memory growth.
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from "@suaveplan/testing/runner";
|
||||
import {
|
||||
clearSegmenterCache,
|
||||
formatSegments,
|
||||
segmentText,
|
||||
} from "../formatter/formatter.js";
|
||||
import {
|
||||
getSegmenterMetricsState,
|
||||
resetSegmenterMetricsState,
|
||||
} from "./observability.js";
|
||||
|
||||
describe("Segmenter observability", () => {
|
||||
it("emits i18n.format.segmenter.duration_ms histogram on every call", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("hello", "en-GB", { granularity: "word" });
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.durationSamples).toHaveLength(1);
|
||||
expect(state.durationSamples[0]?.attributes).toMatchObject({
|
||||
locale: "en-GB",
|
||||
granularity: "word",
|
||||
});
|
||||
});
|
||||
|
||||
it("cache-hit counter increments only on second call with same options", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
segmentText("a", "en-GB");
|
||||
expect(getSegmenterMetricsState().cacheHitSamples).toHaveLength(0);
|
||||
segmentText("a", "en-GB");
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.cacheHitSamples).toHaveLength(1);
|
||||
expect(state.cacheHitSamples[0]?.attributes.formatter).toBe(
|
||||
"segmenter"
|
||||
);
|
||||
});
|
||||
|
||||
it("formatSegments also emits histogram + cache-hit", {
|
||||
timeout: 5000,
|
||||
}, () => {
|
||||
clearSegmenterCache();
|
||||
resetSegmenterMetricsState();
|
||||
formatSegments("hi", "en-GB", { granularity: "word" });
|
||||
formatSegments("ok", "en-GB", { granularity: "word" });
|
||||
const state = getSegmenterMetricsState();
|
||||
expect(state.durationSamples).toHaveLength(2);
|
||||
expect(state.cacheHitSamples).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { type Counter, type Histogram, metrics } from "@opentelemetry/api";
|
||||
|
||||
/**
|
||||
* OTel instruments for the Segmenter formatter.
|
||||
*
|
||||
* Histogram records per-segment wall-clock duration; the counter
|
||||
* increments on every cache reuse so consumers can observe cache
|
||||
* effectiveness.
|
||||
*
|
||||
* The module retains a small in-memory state ring so spec-conformance
|
||||
* tests can assert exact attribute payloads without standing up a
|
||||
* full SDK exporter.
|
||||
*/
|
||||
|
||||
export interface SegmenterMetricSample {
|
||||
readonly attributes: Record<string, string | number>;
|
||||
}
|
||||
|
||||
export interface SegmenterMetricsState {
|
||||
readonly durationSamples: SegmenterMetricSample[];
|
||||
readonly cacheHitSamples: SegmenterMetricSample[];
|
||||
}
|
||||
|
||||
let cached:
|
||||
| {
|
||||
readonly durationMs: Histogram;
|
||||
readonly cacheHit: Counter;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
const state: SegmenterMetricsState = {
|
||||
durationSamples: [],
|
||||
cacheHitSamples: [],
|
||||
};
|
||||
|
||||
function ensureInstruments() {
|
||||
if (cached) return cached;
|
||||
const meter = metrics.getMeter("suaveplan.i18n.formatter.segmenter");
|
||||
cached = {
|
||||
durationMs: meter.createHistogram("i18n.format.segmenter.duration_ms", {
|
||||
description:
|
||||
"Duration in ms of SegmenterFormatter.segment / formatSegments; labels: locale, granularity.",
|
||||
unit: "ms",
|
||||
}),
|
||||
cacheHit: meter.createCounter("i18n.format.cache_hit", {
|
||||
description:
|
||||
"Format-time cache reuse counter; labels: formatter (segmenter).",
|
||||
}),
|
||||
};
|
||||
return cached;
|
||||
}
|
||||
|
||||
export function recordSegmenterFormat(
|
||||
locale: string,
|
||||
granularity: string,
|
||||
durationMs: number
|
||||
): void {
|
||||
const ins = ensureInstruments();
|
||||
const attributes = { locale, granularity };
|
||||
ins.durationMs.record(durationMs, attributes);
|
||||
state.durationSamples.push({ attributes });
|
||||
}
|
||||
|
||||
export function recordSegmenterCacheHit(): void {
|
||||
const ins = ensureInstruments();
|
||||
const attributes = { formatter: "segmenter" };
|
||||
ins.cacheHit.add(1, attributes);
|
||||
state.cacheHitSamples.push({ attributes });
|
||||
}
|
||||
|
||||
export function getSegmenterMetricsState(): SegmenterMetricsState {
|
||||
return state;
|
||||
}
|
||||
|
||||
export function resetSegmenterMetricsState(): void {
|
||||
state.durationSamples.length = 0;
|
||||
state.cacheHitSamples.length = 0;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "../../../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"lib": ["ES2022"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["src/**/*.test.ts", "dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { defineConfig } from "vite";
|
||||
import dts from "vite-plugin-dts";
|
||||
|
||||
export default defineConfig({
|
||||
root: __dirname,
|
||||
cacheDir: "../../../node_modules/.vite/packages/i18n-formatter-segmenter",
|
||||
plugins: [
|
||||
dts({
|
||||
entryRoot: "src",
|
||||
tsconfigPath: "./tsconfig.lib.json",
|
||||
insertTypesEntry: true,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
outDir: "./dist",
|
||||
emptyOutDir: true,
|
||||
reportCompressedSize: true,
|
||||
sourcemap: true,
|
||||
lib: { entry: "src/index.ts", formats: ["es"] },
|
||||
rollupOptions: {
|
||||
treeshake: true,
|
||||
output: {
|
||||
preserveModules: true,
|
||||
preserveModulesRoot: "src",
|
||||
entryFileNames: "[name].js",
|
||||
},
|
||||
external: [
|
||||
"@suaveplan/error",
|
||||
"@suaveplan/i18n-shared",
|
||||
"@suaveplan/types",
|
||||
"@opentelemetry/api",
|
||||
"tslib",
|
||||
"zod",
|
||||
/^node:/,
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: false,
|
||||
environment: "node",
|
||||
pool: "forks",
|
||||
include: ["src/**/*.test.ts"],
|
||||
exclude: ["dist/**", "node_modules/**"],
|
||||
coverage: {
|
||||
provider: "istanbul",
|
||||
reporter: ["text", "html", "lcov"],
|
||||
include: ["src/**/*.ts"],
|
||||
exclude: [
|
||||
"src/**/*.test.ts",
|
||||
"src/**/*.types.ts",
|
||||
"src/**/*.schemas.ts",
|
||||
"src/**/index.ts",
|
||||
],
|
||||
thresholds: {
|
||||
statements: 100,
|
||||
branches: 100,
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
},
|
||||
testTimeout: 5000,
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user