Initial Commit

packages/foundation/data/mime/src/charset/charset.test.ts

@@ -0,0 +1,132 @@
+import { describe, expect, it } from "@suaveplan/testing/runner";
+import { sniffBom } from "./sniff-bom.js";
+import { sniffHtmlMeta } from "./sniff-html-meta.js";
+import { sniffXmlDecl } from "./sniff-xml-decl.js";
+
+describe("sniffBom", () => {
+    it("detects UTF-8 BOM (EF BB BF)", { timeout: 5000 }, () => {
+        const bytes = new Uint8Array([0xef, 0xbb, 0xbf, 0x68, 0x65]);
+        const result = sniffBom(bytes);
+        expect(result?.charset).toBe("utf-8");
+        expect(result?.bomLength).toBe(3);
+    });
+
+    it("detects UTF-16 LE BOM (FF FE)", { timeout: 5000 }, () => {
+        const bytes = new Uint8Array([0xff, 0xfe, 0x68, 0x00]);
+        const result = sniffBom(bytes);
+        expect(result?.charset).toBe("utf-16le");
+        expect(result?.bomLength).toBe(2);
+    });
+
+    it("detects UTF-16 BE BOM (FE FF)", { timeout: 5000 }, () => {
+        const bytes = new Uint8Array([0xfe, 0xff, 0x00, 0x68]);
+        const result = sniffBom(bytes);
+        expect(result?.charset).toBe("utf-16be");
+        expect(result?.bomLength).toBe(2);
+    });
+
+    it("detects UTF-32 LE BOM (FF FE 00 00)", { timeout: 5000 }, () => {
+        const bytes = new Uint8Array([0xff, 0xfe, 0x00, 0x00, 0x68]);
+        const result = sniffBom(bytes);
+        expect(result?.charset).toBe("utf-32le");
+        expect(result?.bomLength).toBe(4);
+    });
+
+    it("detects UTF-32 BE BOM (00 00 FE FF)", { timeout: 5000 }, () => {
+        const bytes = new Uint8Array([0x00, 0x00, 0xfe, 0xff, 0x68]);
+        const result = sniffBom(bytes);
+        expect(result?.charset).toBe("utf-32be");
+        expect(result?.bomLength).toBe(4);
+    });
+
+    it("returns null for no BOM", { timeout: 5000 }, () => {
+        const bytes = new Uint8Array([0x68, 0x65, 0x6c, 0x6c, 0x6f]);
+        expect(sniffBom(bytes)).toBeNull();
+    });
+
+    it("returns null for empty buffer", { timeout: 5000 }, () => {
+        expect(sniffBom(new Uint8Array(0))).toBeNull();
+    });
+
+    it("accepts ArrayBuffer", { timeout: 5000 }, () => {
+        const buf = new Uint8Array([0xef, 0xbb, 0xbf, 0x41]).buffer;
+        expect(sniffBom(buf)?.charset).toBe("utf-8");
+    });
+
+    it("UTF-32 LE takes priority over UTF-16 LE when first 4 bytes are FF FE 00 00", {
+        timeout: 5000,
+    }, () => {
+        const bytes = new Uint8Array([0xff, 0xfe, 0x00, 0x00]);
+        expect(sniffBom(bytes)?.charset).toBe("utf-32le");
+    });
+});
+
+describe("sniffHtmlMeta", () => {
+    it("detects charset from <meta charset>", { timeout: 5000 }, () => {
+        const html = '<html><head><meta charset="UTF-8"></head></html>';
+        expect(sniffHtmlMeta(html)).toBe("utf-8");
+    });
+
+    it("detects charset from <meta charset> without quotes", {
+        timeout: 5000,
+    }, () => {
+        const html = "<meta charset=UTF-8>";
+        expect(sniffHtmlMeta(html)).toBe("utf-8");
+    });
+
+    it("detects charset from http-equiv content-type", {
+        timeout: 5000,
+    }, () => {
+        const html =
+            '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">';
+        expect(sniffHtmlMeta(html)).toBe("iso-8859-1");
+    });
+
+    it("returns null when no charset found", { timeout: 5000 }, () => {
+        expect(
+            sniffHtmlMeta("<html><head><title>Hello</title></head></html>")
+        ).toBeNull();
+    });
+
+    it("only scans first 1024 characters", { timeout: 5000 }, () => {
+        const prefix = "x".repeat(1100);
+        const html = `${prefix}<meta charset="UTF-8">`;
+        expect(sniffHtmlMeta(html)).toBeNull();
+    });
+
+    it("returns lowercase charset", { timeout: 5000 }, () => {
+        expect(sniffHtmlMeta('<meta charset="ISO-8859-1">')).toBe("iso-8859-1");
+    });
+});
+
+describe("sniffXmlDecl", () => {
+    it("detects encoding from XML declaration", { timeout: 5000 }, () => {
+        const xml = '<?xml version="1.0" encoding="UTF-8"?><root/>';
+        expect(sniffXmlDecl(xml)).toBe("utf-8");
+    });
+
+    it("detects encoding with single quotes", { timeout: 5000 }, () => {
+        const xml = "<?xml version='1.0' encoding='ISO-8859-1'?><root/>";
+        expect(sniffXmlDecl(xml)).toBe("iso-8859-1");
+    });
+
+    it("returns null when no encoding attribute", { timeout: 5000 }, () => {
+        const xml = '<?xml version="1.0"?><root/>';
+        expect(sniffXmlDecl(xml)).toBeNull();
+    });
+
+    it("returns null for non-XML content", { timeout: 5000 }, () => {
+        expect(sniffXmlDecl("<html>not xml</html>")).toBeNull();
+    });
+
+    it("only scans first 256 characters", { timeout: 5000 }, () => {
+        const prefix = "x".repeat(300);
+        const xml = `${prefix}<?xml version="1.0" encoding="UTF-8"?>`;
+        expect(sniffXmlDecl(xml)).toBeNull();
+    });
+
+    it("returns lowercase encoding", { timeout: 5000 }, () => {
+        const xml = '<?xml version="1.0" encoding="UTF-16"?>';
+        expect(sniffXmlDecl(xml)).toBe("utf-16");
+    });
+});

packages/foundation/data/mime/src/charset/charset.types.ts

@@ -0,0 +1,16 @@
+/**
+ * Charset detection type definitions.
+ *
+ * @module charset/charset.types
+ */
+
+/**
+ * Result of a BOM detection scan.
+ *
+ * `charset` is the IANA charset name in lowercase (e.g. `"utf-8"`, `"utf-16le"`).
+ * `bomLength` is the number of bytes consumed by the BOM prefix.
+ */
+export interface BomDetection {
+    readonly charset: string;
+    readonly bomLength: number;
+}

packages/foundation/data/mime/src/charset/index.ts

@@ -0,0 +1,4 @@
+export type { BomDetection } from "./charset.types.js";
+export { sniffBom } from "./sniff-bom.js";
+export { sniffHtmlMeta } from "./sniff-html-meta.js";
+export { sniffXmlDecl } from "./sniff-xml-decl.js";

packages/foundation/data/mime/src/charset/sniff-bom.md

@@ -0,0 +1,59 @@
+# sniff-bom
+
+## Purpose
+
+Detects character encoding from a Byte Order Mark (BOM) at the start of a binary buffer. BOM detection is the highest-confidence method of charset identification — when present, a BOM unambiguously identifies the encoding without relying on declared metadata or heuristics.
+
+## Features
+
+- Detects UTF-8 BOM (`EF BB BF`)
+- Detects UTF-16 BE BOM (`FE FF`)
+- Detects UTF-16 LE BOM (`FF FE`)
+- Detects UTF-32 BE BOM (`00 00 FE FF`)
+- Detects UTF-32 LE BOM (`FF FE 00 00`)
+- Accepts any `ArrayBuffer`, `SharedArrayBuffer`, or `ArrayBufferView` (`Uint8Array`, `DataView`, etc.)
+- Returns the IANA charset name in lowercase, or `null` when no BOM is present
+- Zero allocation path for no-BOM case
+
+## Usage: Basic Example
+
+```typescript
+import { sniffBom } from "@suaveplan/mime/charset/sniff-bom";
+
+const bytes = new Uint8Array([0xef, 0xbb, 0xbf, 0x3c, 0x68, 0x74, 0x6d, 0x6c]);
+const charset = sniffBom(bytes);
+console.log(charset); // "utf-8"
+```
+
+## Usage: Advanced Example
+
+```typescript
+import { sniffBom } from "@suaveplan/mime/charset/sniff-bom";
+import { sniffHtmlMeta } from "@suaveplan/mime/charset/sniff-html-meta";
+
+async function detectEncoding(buffer: Uint8Array): Promise<string> {
+    // BOM takes precedence over meta-tag declarations
+    const bomCharset = sniffBom(buffer);
+    if (bomCharset) return bomCharset;
+
+    // Fall back to meta-tag detection
+    const text = new TextDecoder("ascii", { fatal: false }).decode(buffer.slice(0, 1024));
+    return sniffHtmlMeta(text) ?? "utf-8";
+}
+```
+
+## API Reference
+
+### `sniffBom(buffer: ArrayBufferView | ArrayBuffer): string | null`
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `buffer` | `ArrayBufferView \| ArrayBuffer` | The byte buffer to inspect (any view or raw buffer). |
+
+**Returns:** The IANA charset name in lowercase, or `null` when no BOM is detected.
+
+**Throws:** Never — all buffer accesses are bounds-checked.
+
+## Implementation Notes
+
+UTF-32 BOMs (both BE and LE) must be tested before UTF-16 LE because the UTF-32 LE BOM (`FF FE 00 00`) starts with the same two bytes as the UTF-16 LE BOM (`FF FE`). The implementation checks the 4-byte sequences first to avoid misidentification. The function uses direct `Uint8Array` byte comparisons for maximum performance — no string allocations or regex matching.

packages/foundation/data/mime/src/charset/sniff-bom.ts

@@ -0,0 +1,57 @@
+/**
+ * Detect a Unicode BOM at the start of a byte sequence.
+ *
+ * Returns the IANA charset name (lower-case) plus the BOM length, so callers
+ * can skip past it. Returns `null` when no BOM is present.
+ *
+ * Recognised BOMs (in priority order):
+ * - UTF-32 LE: `FF FE 00 00` (checked before UTF-16 LE)
+ * - UTF-32 BE: `00 00 FE FF`
+ * - UTF-8:     `EF BB BF`
+ * - UTF-16 LE: `FF FE`
+ * - UTF-16 BE: `FE FF`
+ *
+ * @module charset/sniff-bom
+ */
+
+import type { BomDetection } from "./charset.types.js";
+
+/**
+ * Detect a Unicode BOM at the start of `bytes`.
+ *
+ * @param bytes - Byte buffer to inspect.
+ * @returns `BomDetection` when a BOM is found, `null` otherwise.
+ */
+export function sniffBom(bytes: Uint8Array | ArrayBuffer): BomDetection | null {
+    const b = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+
+    // UTF-32 LE must be checked before UTF-16 LE (FF FE prefix overlap)
+    if (
+        b.length >= 4 &&
+        b[0] === 0xff &&
+        b[1] === 0xfe &&
+        b[2] === 0 &&
+        b[3] === 0
+    ) {
+        return { charset: "utf-32le", bomLength: 4 };
+    }
+    if (
+        b.length >= 4 &&
+        b[0] === 0 &&
+        b[1] === 0 &&
+        b[2] === 0xfe &&
+        b[3] === 0xff
+    ) {
+        return { charset: "utf-32be", bomLength: 4 };
+    }
+    if (b.length >= 3 && b[0] === 0xef && b[1] === 0xbb && b[2] === 0xbf) {
+        return { charset: "utf-8", bomLength: 3 };
+    }
+    if (b.length >= 2 && b[0] === 0xff && b[1] === 0xfe) {
+        return { charset: "utf-16le", bomLength: 2 };
+    }
+    if (b.length >= 2 && b[0] === 0xfe && b[1] === 0xff) {
+        return { charset: "utf-16be", bomLength: 2 };
+    }
+    return null;
+}

packages/foundation/data/mime/src/charset/sniff-html-meta.md

@@ -0,0 +1,65 @@
+# sniff-html-meta
+
+## Purpose
+
+Extracts a declared character encoding from an HTML `<meta>` tag by scanning the first 1,024 characters of an HTML document. Handles both the modern HTML5 `<meta charset="...">` form and the legacy HTML4 `<meta http-equiv="Content-Type" content="...; charset=...">` form using a single regex that matches `charset=` in any attribute context.
+
+## Features
+
+- Scans up to 1,024 bytes (configurable via the slice parameter)
+- Matches HTML5 `<meta charset="...">` tags
+- Matches HTML4 `<meta http-equiv="Content-Type" content="...; charset=...">` tags
+- Case-insensitive matching for both tag names and attribute values
+- Returns the charset name in lowercase for canonical comparison
+- Returns `null` when no charset declaration is found
+- Zero allocation on the no-match path
+
+## Usage: Basic Example
+
+```typescript
+import { sniffHtmlMeta } from "@suaveplan/mime/charset/sniff-html-meta";
+
+const html = '<html><head><meta charset="UTF-8"><title>Hello</title>';
+const charset = sniffHtmlMeta(html);
+console.log(charset); // "utf-8"
+```
+
+## Usage: Advanced Example
+
+```typescript
+import { sniffHtmlMeta } from "@suaveplan/mime/charset/sniff-html-meta";
+import { sniffBom } from "@suaveplan/mime/charset/sniff-bom";
+
+function detectHtmlEncoding(bytes: Uint8Array): string {
+    // BOM has highest priority
+    const bom = sniffBom(bytes);
+    if (bom) return bom;
+
+    // Decode as ASCII/Latin-1 to scan for meta tags
+    const head = new TextDecoder("latin1").decode(bytes.slice(0, 1024));
+    const meta = sniffHtmlMeta(head);
+
+    // HTML5 spec default is utf-8 when no declaration is found
+    return meta ?? "utf-8";
+}
+
+// Legacy HTML4 http-equiv form
+const legacyHtml = '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">';
+console.log(sniffHtmlMeta(legacyHtml)); // "iso-8859-1"
+```
+
+## API Reference
+
+### `sniffHtmlMeta(source: string): string | null`
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `source` | `string` | HTML content (or the first chunk thereof). |
+
+**Returns:** The lowercase IANA charset name, or `null` when no charset declaration is found in the first 1,024 characters.
+
+**Throws:** Never.
+
+## Implementation Notes
+
+A single regex (`/<meta[^>]+charset\s*=\s*["']?([A-Za-z0-9_\-:]+)/i`) covers both the HTML5 and HTML4 forms because `charset=` appears literally in the `content` attribute of the `http-equiv` tag. A separate legacy regex for the full `http-equiv` pattern is unnecessary — the simpler pattern matches earlier and always wins. The 1,024-character scan window follows the HTML5 parsing spec for prescan byte-sequence limits.

packages/foundation/data/mime/src/charset/sniff-html-meta.ts

@@ -0,0 +1,32 @@
+/**
+ * Extract a declared charset from an HTML `<meta>` tag.
+ *
+ * Scans up to 1 024 bytes of HTML for:
+ * - `<meta charset="...">` (HTML5)
+ * - `<meta http-equiv="content-type" content="...; charset=...">` (legacy HTML4)
+ *
+ * Both patterns are matched case-insensitively. Returns the charset name in
+ * lowercase, or `null` when no charset declaration is found.
+ *
+ * @module charset/sniff-html-meta
+ */
+
+const META_CHARSET = /<meta[^>]+charset\s*=\s*["']?([A-Za-z0-9_\-:]+)/i;
+
+/**
+ * Sniff the charset from the first 1 024 characters of an HTML document.
+ *
+ * Matches both HTML5 `<meta charset="...">` and legacy
+ * `<meta http-equiv="content-type" content="...; charset=...">` because the
+ * single META_CHARSET pattern `charset\s*=` catches `charset=` in any
+ * attribute context, including the `content` attribute of an `http-equiv` tag.
+ *
+ * @param source - String of HTML content (or first chunk thereof).
+ * @returns Lowercase IANA charset name, or `null` when not found.
+ */
+export function sniffHtmlMeta(source: string): string | null {
+    const window = source.slice(0, 1024);
+    const m = META_CHARSET.exec(window);
+    if (m?.[1]) return m[1].toLowerCase();
+    return null;
+}

packages/foundation/data/mime/src/charset/sniff-xml-decl.md

@@ -0,0 +1,64 @@
+# sniff-xml-decl
+
+## Purpose
+
+Extracts the declared encoding from an XML or SVG document's processing instruction (`<?xml version="1.0" encoding="UTF-8"?>`). XML documents commonly declare their encoding in the prolog, and this declaration must be trusted before any BOM or HTTP header when present, per the XML specification (RFC 7303).
+
+## Features
+
+- Matches the `encoding` attribute in `<?xml ... encoding="..."?>` declarations
+- Supports both double-quoted and single-quoted attribute values
+- Case-insensitive matching — `ENCODING` and `encoding` both match
+- Scans only the first 256 characters (the XML prolog is always at the start)
+- Returns the encoding name in lowercase for canonical comparison
+- Returns `null` when no XML declaration or encoding attribute is found
+- Handles whitespace variations between attributes
+
+## Usage: Basic Example
+
+```typescript
+import { sniffXmlDecl } from "@suaveplan/mime/charset/sniff-xml-decl";
+
+const xml = '<?xml version="1.0" encoding="UTF-8"?><root/>';
+const encoding = sniffXmlDecl(xml);
+console.log(encoding); // "utf-8"
+```
+
+## Usage: Advanced Example
+
+```typescript
+import { sniffXmlDecl } from "@suaveplan/mime/charset/sniff-xml-decl";
+import { sniffBom } from "@suaveplan/mime/charset/sniff-bom";
+
+function detectXmlEncoding(bytes: Uint8Array): string {
+    // BOM overrides XML declaration per RFC 7303 §4.3
+    const bom = sniffBom(bytes);
+    if (bom) return bom;
+
+    // Use latin-1 to safely read the ASCII-range prolog
+    const prolog = new TextDecoder("latin1").decode(bytes.slice(0, 256));
+    const declared = sniffXmlDecl(prolog);
+
+    return declared ?? "utf-8"; // XML default encoding
+}
+
+// Single-quoted attribute
+const svgWithSingleQuotes = "<?xml version='1.0' encoding='iso-8859-1'?><svg/>";
+console.log(sniffXmlDecl(svgWithSingleQuotes)); // "iso-8859-1"
+```
+
+## API Reference
+
+### `sniffXmlDecl(source: string): string | null`
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `source` | `string` | XML/SVG content (or the first chunk thereof). |
+
+**Returns:** The lowercase IANA encoding name, or `null` when no encoding attribute is found.
+
+**Throws:** Never.
+
+## Implementation Notes
+
+The regex `/<\?xml[^?]*encoding\s*=\s*["']([^"']+)["']/i` intentionally uses `[^?]*` (non-greedy via exclusion) rather than `.*?` to avoid backtracking across large documents when no `<?xml` declaration is present. The 256-character scan window is sufficient because the XML specification mandates that the encoding declaration, if present, must appear in the document prolog before any other content.

packages/foundation/data/mime/src/charset/sniff-xml-decl.ts

@@ -0,0 +1,24 @@
+/**
+ * Extract the charset from an XML declaration: `<?xml ... encoding="..."?>`.
+ *
+ * Scans the first 256 characters for an XML declaration and extracts the
+ * `encoding` attribute value. Returns the charset in lowercase, or `null`
+ * when no encoding declaration is found.
+ *
+ * @module charset/sniff-xml-decl
+ */
+
+const XML_ENCODING = /<\?xml[^?]*encoding\s*=\s*["']([A-Za-z0-9_\-:]+)/i;
+
+/**
+ * Sniff the encoding from an XML declaration in the first 256 characters.
+ *
+ * @param source - String of XML content (or first chunk thereof).
+ * @returns Lowercase IANA charset name, or `null` when not found.
+ */
+export function sniffXmlDecl(source: string): string | null {
+    const window = source.slice(0, 256);
+    const m = XML_ENCODING.exec(window);
+    if (m?.[1]) return m[1].toLowerCase();
+    return null;
+}