Initial Commit

packages/api-stack/api/api-plugin-search/src/analyzer/analyzer.test.ts

@@ -0,0 +1,163 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { SEARCH_CODES, SearchError } from "../error-codes/error-codes.js";
+import {
+    createKeywordAnalyzer,
+    createSimpleAnalyzer,
+    createStandardAnalyzer,
+    createWhitespaceAnalyzer,
+    getAnalyzer,
+    listAnalyzers,
+    removeStopWords,
+    stem,
+    tokenize,
+} from "./analyzer.js";
+
+describe("tokenize", () => {
+    it("lowercases and strips punctuation", { timeout: 5000 }, () => {
+        expect(tokenize("Hello, World!")).toEqual(["hello", "world"]);
+    });
+
+    it("drops empty tokens", { timeout: 5000 }, () => {
+        expect(tokenize("   a    b   ")).toEqual(["a", "b"]);
+    });
+});
+
+describe("removeStopWords", () => {
+    it("filters the default English stop-word set", { timeout: 5000 }, () => {
+        expect(removeStopWords(["the", "cat", "is", "fast"])).toEqual([
+            "cat",
+            "fast",
+        ]);
+    });
+
+    it("accepts a custom stop-word set", { timeout: 5000 }, () => {
+        expect(removeStopWords(["foo", "bar"], new Set(["foo"]))).toEqual([
+            "bar",
+        ]);
+    });
+});
+
+describe("stem", () => {
+    it("returns short words unchanged", { timeout: 5000 }, () => {
+        expect(stem("at")).toBe("at");
+        expect(stem("cat")).toBe("cat");
+    });
+
+    it("strips common suffixes", { timeout: 5000 }, () => {
+        expect(stem("running")).toBe("runn");
+        expect(stem("happiness")).toBe("happi");
+        expect(stem("rationalization")).toBe("rationalize");
+    });
+
+    it("returns the original word when the stem would be too short", {
+        timeout: 5000,
+    }, () => {
+        // "ling" suffix on "ling" itself → "" replacement → too short, returned as-is
+        expect(stem("ling")).toBe("ling");
+    });
+
+    it("falls through when no suffix rule matches", { timeout: 5000 }, () => {
+        expect(stem("xyzfoo")).toBe("xyzfoo");
+    });
+});
+
+describe("standard analyzer", () => {
+    it("standard analyzer tokenises, removes stop words, and stems", {
+        timeout: 5000,
+    }, () => {
+        const a = createStandardAnalyzer();
+        const tokens = a.analyze("Running the cats quickly");
+        expect(tokens).toContain("cat");
+        expect(tokens).toContain("quick");
+        expect(tokens).not.toContain("the");
+    });
+
+    it("respects opt-out flags", { timeout: 5000 }, () => {
+        const a = createStandardAnalyzer({
+            removeStops: false,
+            stemTokens: false,
+        });
+        const tokens = a.analyze("the cats running");
+        expect(tokens).toEqual(["the", "cats", "running"]);
+    });
+
+    it("enforces min/max token length", { timeout: 5000 }, () => {
+        const a = createStandardAnalyzer({
+            removeStops: false,
+            stemTokens: false,
+            minTokenLength: 3,
+            maxTokenLength: 5,
+        });
+        expect(a.analyze("a bb ccc dddd eeeee ffffff")).toEqual([
+            "ccc",
+            "dddd",
+            "eeeee",
+        ]);
+    });
+});
+
+describe("simple analyzer", () => {
+    it("tokenises with no stemming or stop-word removal", {
+        timeout: 5000,
+    }, () => {
+        const a = createSimpleAnalyzer();
+        expect(a.analyze("The Cats Running")).toEqual([
+            "the",
+            "cats",
+            "running",
+        ]);
+    });
+});
+
+describe("whitespace analyzer", () => {
+    it("splits on whitespace and preserves case", { timeout: 5000 }, () => {
+        const a = createWhitespaceAnalyzer();
+        expect(a.analyze("Hello, World!")).toEqual(["Hello,", "World!"]);
+    });
+});
+
+describe("keyword analyzer", () => {
+    it("keyword analyzer returns the trimmed input as a single token", {
+        timeout: 5000,
+    }, () => {
+        expect(createKeywordAnalyzer().analyze("  Hello World  ")).toEqual([
+            "Hello World",
+        ]);
+    });
+
+    it("returns empty array for empty input", { timeout: 5000 }, () => {
+        expect(createKeywordAnalyzer().analyze("   ")).toEqual([]);
+    });
+});
+
+describe("registry", () => {
+    it("lists the four built-in analyzer names", { timeout: 5000 }, () => {
+        expect(listAnalyzers()).toEqual([
+            "standard",
+            "simple",
+            "whitespace",
+            "keyword",
+        ]);
+    });
+
+    it("getAnalyzer throws SEARCH.UNKNOWN_ANALYZER for an unregistered name", {
+        timeout: 5000,
+    }, () => {
+        try {
+            getAnalyzer("does-not-exist");
+            expect.fail("expected SearchError");
+        } catch (err) {
+            expect(err).toBeInstanceOf(SearchError);
+            expect((err as SearchError).code).toBe(
+                SEARCH_CODES.UNKNOWN_ANALYZER
+            );
+        }
+    });
+
+    it("resolves each built-in analyzer", { timeout: 5000 }, () => {
+        for (const name of listAnalyzers()) {
+            const a = getAnalyzer(name);
+            expect(a.name).toBe(name);
+        }
+    });
+});