suaveplan/packages/api-stack/api/api-plugin-search/src/analyzer/analyzer.test.ts

import { describe, expect, it } from "@suaveplan/testing/runner";
import { SEARCH_CODES, SearchError } from "../error-codes/error-codes.js";
import {
    createKeywordAnalyzer,
    createSimpleAnalyzer,
    createStandardAnalyzer,
    createWhitespaceAnalyzer,
    getAnalyzer,
    listAnalyzers,
    removeStopWords,
    stem,
    tokenize,
} from "./analyzer.js";

describe("tokenize", () => {
    it("lowercases and strips punctuation", { timeout: 5000 }, () => {
        expect(tokenize("Hello, World!")).toEqual(["hello", "world"]);
    });

    it("drops empty tokens", { timeout: 5000 }, () => {
        expect(tokenize("   a    b   ")).toEqual(["a", "b"]);
    });
});

describe("removeStopWords", () => {
    it("filters the default English stop-word set", { timeout: 5000 }, () => {
        expect(removeStopWords(["the", "cat", "is", "fast"])).toEqual([
            "cat",
            "fast",
        ]);
    });

    it("accepts a custom stop-word set", { timeout: 5000 }, () => {
        expect(removeStopWords(["foo", "bar"], new Set(["foo"]))).toEqual([
            "bar",
        ]);
    });
});

describe("stem", () => {
    it("returns short words unchanged", { timeout: 5000 }, () => {
        expect(stem("at")).toBe("at");
        expect(stem("cat")).toBe("cat");
    });

    it("strips common suffixes", { timeout: 5000 }, () => {
        expect(stem("running")).toBe("runn");
        expect(stem("happiness")).toBe("happi");
        expect(stem("rationalization")).toBe("rationalize");
    });

    it("returns the original word when the stem would be too short", {
        timeout: 5000,
    }, () => {
        // "ling" suffix on "ling" itself → "" replacement → too short, returned as-is
        expect(stem("ling")).toBe("ling");
    });

    it("falls through when no suffix rule matches", { timeout: 5000 }, () => {
        expect(stem("xyzfoo")).toBe("xyzfoo");
    });
});

describe("standard analyzer", () => {
    it("standard analyzer tokenises, removes stop words, and stems", {
        timeout: 5000,
    }, () => {
        const a = createStandardAnalyzer();
        const tokens = a.analyze("Running the cats quickly");
        expect(tokens).toContain("cat");
        expect(tokens).toContain("quick");
        expect(tokens).not.toContain("the");
    });

    it("respects opt-out flags", { timeout: 5000 }, () => {
        const a = createStandardAnalyzer({
            removeStops: false,
            stemTokens: false,
        });
        const tokens = a.analyze("the cats running");
        expect(tokens).toEqual(["the", "cats", "running"]);
    });

    it("enforces min/max token length", { timeout: 5000 }, () => {
        const a = createStandardAnalyzer({
            removeStops: false,
            stemTokens: false,
            minTokenLength: 3,
            maxTokenLength: 5,
        });
        expect(a.analyze("a bb ccc dddd eeeee ffffff")).toEqual([
            "ccc",
            "dddd",
            "eeeee",
        ]);
    });
});

describe("simple analyzer", () => {
    it("tokenises with no stemming or stop-word removal", {
        timeout: 5000,
    }, () => {
        const a = createSimpleAnalyzer();
        expect(a.analyze("The Cats Running")).toEqual([
            "the",
            "cats",
            "running",
        ]);
    });
});

describe("whitespace analyzer", () => {
    it("splits on whitespace and preserves case", { timeout: 5000 }, () => {
        const a = createWhitespaceAnalyzer();
        expect(a.analyze("Hello, World!")).toEqual(["Hello,", "World!"]);
    });
});

describe("keyword analyzer", () => {
    it("keyword analyzer returns the trimmed input as a single token", {
        timeout: 5000,
    }, () => {
        expect(createKeywordAnalyzer().analyze("  Hello World  ")).toEqual([
            "Hello World",
        ]);
    });

    it("returns empty array for empty input", { timeout: 5000 }, () => {
        expect(createKeywordAnalyzer().analyze("   ")).toEqual([]);
    });
});

describe("registry", () => {
    it("lists the four built-in analyzer names", { timeout: 5000 }, () => {
        expect(listAnalyzers()).toEqual([
            "standard",
            "simple",
            "whitespace",
            "keyword",
        ]);
    });

    it("getAnalyzer throws SEARCH.UNKNOWN_ANALYZER for an unregistered name", {
        timeout: 5000,
    }, () => {
        try {
            getAnalyzer("does-not-exist");
            expect.fail("expected SearchError");
        } catch (err) {
            expect(err).toBeInstanceOf(SearchError);
            expect((err as SearchError).code).toBe(
                SEARCH_CODES.UNKNOWN_ANALYZER
            );
        }
    });

    it("resolves each built-in analyzer", { timeout: 5000 }, () => {
        for (const name of listAnalyzers()) {
            const a = getAnalyzer(name);
            expect(a.name).toBe(name);
        }
    });
});