From b10952e9c6fc165a1a71b52798ddb6fc8482c1c4 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Wed, 22 Jul 2026 05:26:43 -0700 Subject: [PATCH] =?UTF-8?q?feat(state):=20cjk=5Funicode61=20FTS5=20tokeniz?= =?UTF-8?q?er=20=E2=80=94=20unicode61=20+=20CJK=20bigrams=20(native=20exte?= =?UTF-8?q?nsion)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit unicode61 indexes a CJK run as ONE token, so 2-char Korean terms (일본, 구글, 우리, ...) can never match it and the trigram tokenizer needs >=3 chars per term — any query containing a 1-2 char CJK token falls through to a LIKE full-table scan (measured 3-6.4s CPU per query on a 6.8GB production state.db; the #1 base cost behind a 12.4s session_search average on CJK workloads). This ships a ~250-line loadable FTS5 tokenizer (no deps) that wraps unicode61: maximal CJK runs inside its tokens are re-emitted as overlapping character bigrams (Lucene CJKAnalyzer semantics), everything else passes through unchanged. FTS5 phrase semantics turn consecutive sub-tokens into exact substring matching down to 2-char terms at index speed. Build: native/fts5_cjk/build.sh -> ~/.hermes/lib/libfts5_cjk.so (override: HERMES_FTS5_CJK_SO). Salvaged from PR #65544; the schema integration lands separately on the v23 external-content layout. --- native/fts5_cjk/README.md | 12 ++ native/fts5_cjk/build.sh | 9 ++ native/fts5_cjk/fts5_cjk.c | 252 +++++++++++++++++++++++++++++++++++++ 3 files changed, 273 insertions(+) create mode 100644 native/fts5_cjk/README.md create mode 100755 native/fts5_cjk/build.sh create mode 100644 native/fts5_cjk/fts5_cjk.c diff --git a/native/fts5_cjk/README.md b/native/fts5_cjk/README.md new file mode 100644 index 0000000000000..7ae17aa7f54fe --- /dev/null +++ b/native/fts5_cjk/README.md @@ -0,0 +1,12 @@ +# fts5_cjk — cjk_unicode61 FTS5 tokenizer + +unicode61 + CJK character bigrams (Lucene CJKAnalyzer semantics). Fixes +2-char Korean/CJK terms falling through to LIKE full-table scans. + +Build & install to ~/.hermes/lib/: + + ./build.sh + +Then run `scripts/fts_v2_migrate.py` to create + backfill messages_fts_v2, +and set `HERMES_FTS_V2_READ=1` in ~/.hermes/.env to cut reads over. +Override the .so location with `HERMES_FTS5_CJK_SO`. diff --git a/native/fts5_cjk/build.sh b/native/fts5_cjk/build.sh new file mode 100755 index 0000000000000..4af00e0dff425 --- /dev/null +++ b/native/fts5_cjk/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Build libfts5_cjk.so and install to ~/.hermes/lib/ (or $1). +set -euo pipefail +cd "$(dirname "$0")" +gcc -shared -fPIC -O2 -Wall -Wextra fts5_cjk.c -o libfts5_cjk.so +dest="${1:-$HOME/.hermes/lib}" +mkdir -p "$dest" +install -m 0644 libfts5_cjk.so "$dest/libfts5_cjk.so" +echo "installed: $dest/libfts5_cjk.so" diff --git a/native/fts5_cjk/fts5_cjk.c b/native/fts5_cjk/fts5_cjk.c new file mode 100644 index 0000000000000..3dc95fda6c20a --- /dev/null +++ b/native/fts5_cjk/fts5_cjk.c @@ -0,0 +1,252 @@ +/* +** fts5_cjk.c — "cjk_unicode61" FTS5 tokenizer: unicode61 + CJK bigrams. +** +** Why: SQLite's unicode61 tokenizer treats a CJK run as ONE token +** ("웅기가말했다" indexes as a single 6-char token), so a 2-char Korean +** query can never match inside it. The stock trigram tokenizer fixes +** substring search but needs >=3 chars per query term — 2-char Korean +** words (일본, 구글, 우리, ...) fall through to a full-table LIKE scan, +** measured at 3-6s per query on a 6.8GB messages table and the #1 driver +** of hermes session_search latency. +** +** What: wrap unicode61. Every token it emits is re-examined; maximal CJK +** runs inside the token are re-emitted as overlapping character BIGRAMS +** (Lucene CJKAnalyzer semantics), non-CJK segments pass through unchanged. +** A lone CJK char (run length 1) is emitted as a unigram. Because FTS5 +** turns consecutive tokens emitted from one query term into a phrase, +** a query word like 캘린더 → [캘린][린더] gets exact substring semantics +** with index-speed lookups, down to 2-char terms. +** +** Build: gcc -shared -fPIC -O2 fts5_cjk.c -o libfts5_cjk.so +** Load: conn.load_extension(path) # default entrypoint sqlite3_ftscjk_init +** Use: CREATE VIRTUAL TABLE t USING fts5(c, tokenize='cjk_unicode61'); +** Extra args pass through to unicode61: +** tokenize='cjk_unicode61 remove_diacritics 2' +*/ +#include +SQLITE_EXTENSION_INIT1 + +#include +#include + +/* ── CJK classification ──────────────────────────────────────────────── */ + +static int cjk_is_cjk(unsigned int cp) { + return (cp >= 0xAC00 && cp <= 0xD7A3) /* Hangul syllables */ + || (cp >= 0x1100 && cp <= 0x11FF) /* Hangul Jamo */ + || (cp >= 0x3130 && cp <= 0x318F) /* Hangul compat Jamo */ + || (cp >= 0xA960 && cp <= 0xA97F) /* Hangul Jamo ext-A */ + || (cp >= 0xD7B0 && cp <= 0xD7FF) /* Hangul Jamo ext-B */ + || (cp >= 0x4E00 && cp <= 0x9FFF) /* CJK unified ideographs */ + || (cp >= 0x3400 && cp <= 0x4DBF) /* CJK ext A */ + || (cp >= 0xF900 && cp <= 0xFAFF) /* CJK compat ideographs */ + || (cp >= 0x20000 && cp <= 0x2FA1F) /* CJK ext B..F, compat sup*/ + || (cp >= 0x3040 && cp <= 0x309F) /* Hiragana */ + || (cp >= 0x30A0 && cp <= 0x30FF) /* Katakana */ + || (cp >= 0x31F0 && cp <= 0x31FF); /* Katakana phonetic ext */ +} + +/* Decode one UTF-8 codepoint at p (n bytes available). Returns byte len +** consumed (>=1); stores codepoint in *pCp. Invalid bytes decode as +** themselves so segmentation still terminates. */ +static int cjk_utf8_decode(const unsigned char *p, int n, unsigned int *pCp) { + unsigned int c = p[0]; + if (c < 0x80) { *pCp = c; return 1; } + if ((c & 0xE0) == 0xC0 && n >= 2) { + *pCp = ((c & 0x1F) << 6) | (p[1] & 0x3F); + return 2; + } + if ((c & 0xF0) == 0xE0 && n >= 3) { + *pCp = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F); + return 3; + } + if ((c & 0xF8) == 0xF0 && n >= 4) { + *pCp = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) | + ((p[2] & 0x3F) << 6) | (p[3] & 0x3F); + return 4; + } + *pCp = c; + return 1; +} + +/* ── tokenizer plumbing ──────────────────────────────────────────────── */ + +typedef struct CjkTokenizer CjkTokenizer; +struct CjkTokenizer { + fts5_tokenizer inner; /* unicode61 methods */ + Fts5Tokenizer *pInner; /* unicode61 instance */ +}; + +typedef struct CjkCallbackCtx CjkCallbackCtx; +struct CjkCallbackCtx { + void *pOuterCtx; + int (*xOuterToken)(void*, int, const char*, int, int, int); +}; + +/* Re-emit one unicode61 token, splitting CJK runs into bigrams. +** +** Offsets: unicode61 reports [iStart,iEnd) into the ORIGINAL text. For +** CJK bytes unicode61's folding is the identity, and ASCII case folding +** preserves byte length, so mapping sub-token offsets by byte position +** within the token is exact for CJK and correct-length for ASCII. For +** rare length-changing folds (accented latin) the highlight offsets can +** drift by a few bytes inside that token; matching is unaffected. Every +** emitted offset is clamped to [iStart,iEnd). +*/ +static int cjk_emit(CjkCallbackCtx *p, int tflags, + const char *pToken, int nToken, int iStart, int iEnd) { + const unsigned char *z = (const unsigned char*)pToken; + int i = 0; + int rc = SQLITE_OK; + + /* Fast path: no CJK anywhere → pass through untouched. */ + int hasCjk = 0; + while (i < nToken) { + unsigned int cp; + i += cjk_utf8_decode(z + i, nToken - i, &cp); + if (cjk_is_cjk(cp)) { hasCjk = 1; break; } + } + if (!hasCjk) { + return p->xOuterToken(p->pOuterCtx, tflags, pToken, nToken, iStart, iEnd); + } + +#define CJK_CLAMP_END(v) ((iStart + (v)) > iEnd ? iEnd : (iStart + (v))) + i = 0; + while (i < nToken && rc == SQLITE_OK) { + unsigned int cp; + int segStart = i; + int len = cjk_utf8_decode(z + i, nToken - i, &cp); + if (!cjk_is_cjk(cp)) { + /* non-CJK segment: extend to the next CJK char (or end) */ + i += len; + while (i < nToken) { + int l2 = cjk_utf8_decode(z + i, nToken - i, &cp); + if (cjk_is_cjk(cp)) break; + i += l2; + } + rc = p->xOuterToken(p->pOuterCtx, tflags, + pToken + segStart, i - segStart, + CJK_CLAMP_END(segStart), CJK_CLAMP_END(i)); + } else { + /* CJK run: collect char byte-boundaries, emit bigrams. */ + int bounds[3]; /* rolling window: start, mid, end */ + bounds[0] = segStart; + bounds[1] = segStart + len; + i += len; + int nChars = 1; + while (i < nToken) { + int l2 = cjk_utf8_decode(z + i, nToken - i, &cp); + if (!cjk_is_cjk(cp)) break; + i += l2; + nChars++; + if (nChars >= 2) { + bounds[2] = i; + if (nChars == 2) { + /* first bigram spans bounds[0]..bounds[2] */ + } + rc = p->xOuterToken(p->pOuterCtx, tflags, + pToken + bounds[0], bounds[2] - bounds[0], + CJK_CLAMP_END(bounds[0]), CJK_CLAMP_END(bounds[2])); + if (rc != SQLITE_OK) break; + bounds[0] = bounds[1]; + bounds[1] = bounds[2]; + } + } + if (rc == SQLITE_OK && nChars == 1) { + /* lone CJK char: emit as unigram */ + rc = p->xOuterToken(p->pOuterCtx, tflags, + pToken + segStart, bounds[1] - segStart, + CJK_CLAMP_END(segStart), CJK_CLAMP_END(bounds[1])); + } + } + } +#undef CJK_CLAMP_END + return rc; +} + +static int cjkInnerCallback(void *pCtx, int tflags, + const char *pToken, int nToken, + int iStart, int iEnd) { + return cjk_emit((CjkCallbackCtx*)pCtx, tflags, pToken, nToken, iStart, iEnd); +} + +static int cjkCreate(void *pApiCtx, const char **azArg, int nArg, + Fts5Tokenizer **ppOut) { + fts5_api *pApi = (fts5_api*)pApiCtx; + CjkTokenizer *p; + void *pInnerCtx = 0; + int rc; + + p = (CjkTokenizer*)sqlite3_malloc(sizeof(CjkTokenizer)); + if (!p) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + + rc = pApi->xFindTokenizer(pApi, "unicode61", &pInnerCtx, &p->inner); + if (rc == SQLITE_OK) { + rc = p->inner.xCreate(pInnerCtx, azArg, nArg, &p->pInner); + } + if (rc != SQLITE_OK) { + sqlite3_free(p); + return rc; + } + *ppOut = (Fts5Tokenizer*)p; + return SQLITE_OK; +} + +static void cjkDelete(Fts5Tokenizer *pTok) { + CjkTokenizer *p = (CjkTokenizer*)pTok; + if (p) { + if (p->pInner) p->inner.xDelete(p->pInner); + sqlite3_free(p); + } +} + +static int cjkTokenize(Fts5Tokenizer *pTok, void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int)) { + CjkTokenizer *p = (CjkTokenizer*)pTok; + CjkCallbackCtx cb; + cb.pOuterCtx = pCtx; + cb.xOuterToken = xToken; + return p->inner.xTokenize(p->pInner, &cb, flags, pText, nText, + cjkInnerCallback); +} + +/* ── registration ────────────────────────────────────────────────────── */ + +static fts5_api *cjkFts5Api(sqlite3 *db) { + fts5_api *pRet = 0; + sqlite3_stmt *pStmt = 0; + if (sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0) == SQLITE_OK) { + sqlite3_bind_pointer(pStmt, 1, (void*)&pRet, "fts5_api_ptr", 0); + sqlite3_step(pStmt); + } + sqlite3_finalize(pStmt); + return pRet; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ftscjk_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { + fts5_api *pFts; + static fts5_tokenizer tok = { cjkCreate, cjkDelete, cjkTokenize }; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + pFts = cjkFts5Api(db); + if (!pFts) { + if (pzErrMsg) *pzErrMsg = sqlite3_mprintf("fts5_cjk: FTS5 unavailable"); + return SQLITE_ERROR; + } + return pFts->xCreateTokenizer(pFts, "cjk_unicode61", (void*)pFts, &tok, 0); +} + +/* Alias for callers that spell out the underscored basename. */ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts5_cjk_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { + return sqlite3_ftscjk_init(db, pzErrMsg, pApi); +}