diff --git a/.env.template b/.env.template
index 2f737dba..e5f00bba 100644
--- a/.env.template
+++ b/.env.template
@@ -22,6 +22,7 @@ PERFORMANCE_LOG_FORMAT=compact # compact|rich
 # EMBEDDING_MODEL_CONFIG__TRANSPORT=openai
 # EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small
 # EMBEDDING_MODEL_CONFIG__MAX_BATCH_SIZE=10
+# EMBEDDING_MODEL_CONFIG__TOKENIZER=
 # EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=
 # EMBEDDING_MODEL_CONFIG__OVERRIDES__API_KEY_ENV=
 
diff --git a/config.toml.example b/config.toml.example
index 60d9092a..366ce277 100644
--- a/config.toml.example
+++ b/config.toml.example
@@ -79,6 +79,7 @@ model = "text-embedding-3-small"
 # max_batch_size = 10
 # Optional client HTTP timeout in seconds (OpenAI + Gemini).
 # timeout = 90.0
+# tokenizer = "tiktoken:cl100k_base" # or "hf:BAAI/bge-m3" / "file:/path/to/tokenizer.json"
 
 # Optional module-level endpoint overrides
 # [embedding.model_config.overrides]
diff --git a/docs/v3/contributing/changing-embeddings.mdx b/docs/v3/contributing/changing-embeddings.mdx
index b93e30ec..d46e7d26 100644
--- a/docs/v3/contributing/changing-embeddings.mdx
+++ b/docs/v3/contributing/changing-embeddings.mdx
@@ -40,6 +40,9 @@ Concretely, for either a dim change or a model change:
    export EMBEDDING_MODEL_CONFIG__TRANSPORT=openai
    export EMBEDDING_MODEL_CONFIG__MODEL=nomic-embed-text
    export EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=http://your-ollama:11434/v1
+   # Non-OpenAI models need a matching tokenizer; otherwise chunk-size decisions
+   # use Honcho's tiktoken fallback and can be rejected by the provider (see #827).
+   # export EMBEDDING_MODEL_CONFIG__TOKENIZER=hf:nomic-ai/nomic-embed-text
    alembic upgrade head
    uv run python scripts/configure_embeddings.py --dry-run
    uv run python scripts/configure_embeddings.py --yes
diff --git a/docs/v3/contributing/configuration.mdx b/docs/v3/contributing/configuration.mdx
index 3c210e71..5f3db366 100644
--- a/docs/v3/contributing/configuration.mdx
+++ b/docs/v3/contributing/configuration.mdx
@@ -268,6 +268,7 @@ EMBEDDING_MODEL_CONFIG__TRANSPORT=openai  # openai, gemini
 EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small
 EMBEDDING_MODEL_CONFIG__MAX_BATCH_SIZE=10  # optional per-request input cap
 EMBEDDING_MODEL_CONFIG__TIMEOUT=90.0       # optional client HTTP timeout (seconds)
+# EMBEDDING_MODEL_CONFIG__TOKENIZER=  # unset = tiktoken auto-detect (default)
 
 # Optional endpoint overrides
 EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=http://localhost:8000/v1
@@ -287,6 +288,15 @@ milliseconds on `http_options.timeout`, and keeps its existing 10-minute
 default when unset. The value is validated at config load the same way as
 LLM `provider_params.timeout` (positive, finite number of seconds).
 
+`EMBEDDING_MODEL_CONFIG__TOKENIZER` is optional. Leave it unset for Honcho's
+current tiktoken auto-detection behavior. For embedding models with a
+different tokenizer (OpenAI-compatible or Gemini), set `tiktoken:<encoding>`,
+`hf:<repo-id>`, or `file:/absolute/path/to/tokenizer.json`. Hugging Face and
+file tokenizers require installing the optional `honcho[tokenizers]` extra.
+When the tokenizer does not match the model's real tokenizer, chunk-size
+decisions can undercount tokens and the provider may reject oversized inputs
+(see issue #827).
+
 Forwarding `dimensions=` to OpenAI-compatible providers is controlled by `EMBEDDING_MODEL_CONFIG__DIMENSIONS_MODE`:
 
 - `auto` (default): forwards `dimensions=` when **the operator has explicitly set `EMBEDDING_VECTOR_DIMENSIONS`** — provenance, not value — and the configured model is not on the known-rejecting list (currently `text-embedding-ada-002`). Explicit `EMBEDDING_VECTOR_DIMENSIONS=1536` *does* trigger the forward; this is how `text-embedding-3-large` truncation to 1536 is expressed. Deployments that leave the setting unset get their existing behavior (`dimensions=` is not forwarded).
diff --git a/pyproject.toml b/pyproject.toml
index 1a6bbec8..6788b705 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,6 +43,9 @@ lancedb = [
     "lancedb>=0.25.3; sys_platform != \"darwin\" or platform_machine != \"x86_64\"",
     "pyarrow>=19.0.0",
 ]
+tokenizers = [
+    "tokenizers>=0.22.0",
+]
 [dependency-groups]
 dev = [
     "pytest>=8.2.2",
diff --git a/src/config.py b/src/config.py
index 092cb6ee..b971f812 100644
--- a/src/config.py
+++ b/src/config.py
@@ -396,6 +396,7 @@ class ConfiguredEmbeddingModelSettings(BaseModel):
     max_batch_size: Annotated[int, Field(gt=0)] | None = None
     # Client HTTP timeout in seconds. OpenAI receives seconds; Gemini converts to ms.
     timeout: float | None = None
+    tokenizer: str | None = None
 
     @field_validator("timeout", mode="before")
     @classmethod
@@ -442,6 +443,7 @@ class EmbeddingModelConfig(BaseModel):
     max_batch_size: Annotated[int, Field(gt=0)] | None = None
     # Client HTTP timeout in seconds. OpenAI receives seconds; Gemini converts to ms.
     timeout: float | None = None
+    tokenizer: str | None = None
 
     @field_validator("timeout", mode="before")
     @classmethod
@@ -575,6 +577,7 @@ def resolve_embedding_model_config(
         base_url=configured.overrides.base_url,
         max_batch_size=configured.max_batch_size,
         timeout=configured.timeout,
+        tokenizer=configured.tokenizer,
     )
 
 
diff --git a/src/embedding_client.py b/src/embedding_client.py
index 3cee4484..183f33c7 100644
--- a/src/embedding_client.py
+++ b/src/embedding_client.py
@@ -6,7 +6,8 @@ import threading
 import time
 from collections import defaultdict
 from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypeVar, cast
+from importlib import import_module
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeVar, cast
 
 import tiktoken
 from nanoid import generate as generate_nanoid
@@ -17,6 +18,7 @@ from .config import (
     resolve_embedding_model_config,
     settings,
 )
+from .exceptions import ValidationException
 
 if TYPE_CHECKING:
     from google import genai
@@ -168,6 +170,118 @@ class BatchItem(NamedTuple):
     token_count: int
 
 
+class TokenizerLike(Protocol):
+    """Minimal encode/decode surface the embedding pipeline needs from a tokenizer."""
+
+    def encode(self, text: str) -> list[int]: ...
+
+    def decode(self, tokens: list[int]) -> str: ...
+
+
+class _HuggingFaceTokenizer:
+    """Adapter making a `tokenizers.Tokenizer` satisfy TokenizerLike."""
+
+    def __init__(self, tokenizer: Any) -> None:
+        self._tokenizer: Any = tokenizer
+        # Special tokens ([CLS]/[SEP]) the provider adds per input also count
+        # toward its token limit, so they must come out of the chunk budget.
+        self.special_tokens_overhead: int = len(
+            tokenizer.encode("", add_special_tokens=True).ids
+        )
+
+    def encode(self, text: str) -> list[int]:
+        return list(self._tokenizer.encode(text, add_special_tokens=False).ids)
+
+    def decode(self, tokens: list[int]) -> str:
+        return str(self._tokenizer.decode(tokens, skip_special_tokens=True))
+
+
+def _load_huggingface_tokenizer(spec: str) -> TokenizerLike:
+    if spec.startswith("hf:"):
+        model_name = spec.removeprefix("hf:")
+        if not model_name:
+            raise ValidationException(
+                "Embedding tokenizer spec 'hf:' requires a model name"
+            )
+        tokenizer_cls = _import_tokenizers_cls()
+        return _HuggingFaceTokenizer(tokenizer_cls.from_pretrained(model_name))
+
+    path = spec.removeprefix("file:")
+    if not path:
+        raise ValidationException(
+            "Embedding tokenizer spec 'file:' requires a tokenizer path"
+        )
+    tokenizer_cls = _import_tokenizers_cls()
+    return _HuggingFaceTokenizer(tokenizer_cls.from_file(path))
+
+
+def _import_tokenizers_cls() -> Any:
+    """Import the `tokenizers.Tokenizer` class, raising a clear ValidationException
+    when the optional `tokenizers` package is not installed."""
+    try:
+        return import_module("tokenizers").Tokenizer
+    except ImportError as exc:
+        raise ValidationException(
+            "The 'tokenizers' package is required for hf: and file: embedding "
+            + "tokenizers. Install it with the honcho[tokenizers] extra."
+        ) from exc
+
+
+def _default_tokenizer_for_model(model: str) -> TokenizerLike:
+    try:
+        return tiktoken.encoding_for_model(model)
+    except KeyError:
+        # The provider's real tokenizer may count differently (e.g. bge-m3's
+        # XLM-RoBERTa tokenizer vs cl100k_base), which can push chunks the
+        # provider then rejects. Set EMBEDDING_MODEL_CONFIG__TOKENIZER to match
+        # the configured model's tokenizer.
+        logger.warning(
+            "No tiktoken encoding for embedding model %r; falling back to "
+            + "cl100k_base for token counting. If the provider uses a different "
+            + "tokenizer, chunk-size decisions may be wrong. Configure "
+            + "EMBEDDING_MODEL_CONFIG__TOKENIZER to override.",
+            model,
+        )
+        return tiktoken.get_encoding("cl100k_base")
+
+
+def _resolve_tokenizer(model: str, spec: str | None) -> TokenizerLike:
+    """Resolve the tokenizer used for embedding token counting and chunking.
+
+    Args:
+        model: Configured embedding model name (used when no spec is given)
+        spec: Optional explicit tokenizer spec: "tiktoken:<encoding>",
+            "hf:<repo-id>", or "file:/path/to/tokenizer.json"
+
+    Returns:
+        A TokenizerLike for chunk-size decisions.
+    """
+    if spec is None or spec.strip() == "":
+        return _default_tokenizer_for_model(model)
+
+    spec = spec.strip()
+
+    if spec.startswith("tiktoken:"):
+        encoding_name = spec.removeprefix("tiktoken:")
+        if not encoding_name:
+            raise ValidationException(
+                "Embedding tokenizer spec 'tiktoken:' requires an encoding name"
+            )
+        try:
+            return tiktoken.get_encoding(encoding_name)
+        except ValueError as exc:
+            raise ValidationException(
+                f"Embedding tokenizer spec uses unknown tiktoken encoding: {encoding_name}"
+            ) from exc
+
+    if spec.startswith(("hf:", "file:")):
+        return _load_huggingface_tokenizer(spec)
+
+    raise ValidationException(
+        "Embedding tokenizer must be unset or start with one of: tiktoken:, hf:, file:"
+    )
+
+
 class _EmbeddingClient:
     """
     Embedding client supporting OpenAI and Gemini with chunking and batching support.
@@ -227,10 +341,19 @@ class _EmbeddingClient:
             self.max_embedding_tokens = max_input_tokens
             self.max_batch_size = config.max_batch_size or 2048
 
-        try:
-            self.encoding: tiktoken.Encoding = tiktoken.encoding_for_model(self.model)
-        except KeyError:
-            self.encoding = tiktoken.get_encoding("cl100k_base")
+        self.encoding: TokenizerLike = _resolve_tokenizer(self.model, config.tokenizer)
+        # Providers count the special tokens they add per input (e.g. [CLS]/[SEP]
+        # for BERT-family models) toward the limit; reserve them from the budget.
+        self.max_embedding_tokens -= getattr(
+            self.encoding, "special_tokens_overhead", 0
+        )
+        if self.max_embedding_tokens <= 0:
+            msg = (
+                f"Effective embedding token budget is {self.max_embedding_tokens} after"
+                " reserving special-token overhead. Reduce"
+                " EMBEDDING_MODEL_CONFIG__TOKENIZER overhead or raise EMBEDDING_MAX_INPUT_TOKENS."
+            )
+            raise ValidationException(msg)
         self.max_embedding_tokens_per_request: int = max_tokens_per_request
 
     @property
@@ -594,7 +717,7 @@ def _chunk_text_with_tokens(
     text: str,
     encoded_tokens: list[int],
     max_tokens: int,
-    encoding: tiktoken.Encoding,
+    encoding: TokenizerLike,
 ) -> list[tuple[str, int]]:
     """
     Split text into chunks that fit within token limits, with 20% overlap.
@@ -603,7 +726,7 @@ def _chunk_text_with_tokens(
         text: Original text to chunk
         encoded_tokens: Pre-encoded tokens for the text
         max_tokens: Maximum tokens per chunk
-        encoding: Tiktoken encoding model
+        encoding: Tokenizer used to decode token slices back into text
 
     Returns:
         List of (chunk_text, token_count) tuples
@@ -684,6 +807,7 @@ class EmbeddingClient:
             runtime_config.api_key,
             runtime_config.base_url,
             runtime_config.max_batch_size,
+            runtime_config.tokenizer,
             settings.EMBEDDING.VECTOR_DIMENSIONS,
             settings.EMBEDDING.MAX_INPUT_TOKENS,
             settings.EMBEDDING.MAX_TOKENS_PER_REQUEST,
@@ -735,20 +859,31 @@ class EmbeddingClient:
         return self._get_client().vector_dimensions
 
     @property
-    def encoding(self) -> tiktoken.Encoding:
-        """Get the tiktoken encoding.
+    def encoding(self) -> TokenizerLike:
+        """Get the configured embedding tokenizer.
 
-        Resolved without constructing the underlying client: tiktoken needs no
-        API key, and token-counting callers (e.g. the document dedup tie-break)
-        must work in environments with no embedding credentials, such as CI for
-        pull requests from forks.
+        Resolved without constructing the underlying client when possible:
+        token counting (e.g. the document dedup tie-break) must work in
+        environments with no embedding credentials, such as CI for pull
+        requests from forks. Only HF/file tokenizers require client
+        construction (to load the tokenizer once and cache it on the
+        instance), so for those specs we fall back to the constructed
+        client's encoding.
         """
-        if self._instance is not None:
+        if (
+            self._instance is not None
+            and self._instance_signature == self._get_settings_signature()
+        ):
             return self._instance.encoding
-        try:
-            return tiktoken.encoding_for_model(self._resolve_runtime_config().model)
-        except KeyError:
-            return tiktoken.get_encoding("cl100k_base")
+        runtime_config = self._resolve_runtime_config()
+        spec = runtime_config.tokenizer
+        if spec and spec.strip() and spec.strip().startswith(("hf:", "file:")):
+            # HF/file tokenizers are loaded once inside _EmbeddingClient and
+            # cached on the instance; resolving them here would either
+            # duplicate the load or bypass the special_tokens_overhead
+            # accounting. Delegate to the constructed client.
+            return self._get_client().encoding
+        return _resolve_tokenizer(runtime_config.model, spec)
 
 
 # Shared singleton embedding client instance
diff --git a/tests/llm/test_embedding_client.py b/tests/llm/test_embedding_client.py
index 9df80be1..d45a0ee2 100644
--- a/tests/llm/test_embedding_client.py
+++ b/tests/llm/test_embedding_client.py
@@ -14,7 +14,9 @@ from src.config import (
 from src.embedding_client import (
     BatchItem,
     _EmbeddingClient,  # pyright: ignore[reportPrivateUsage]
+    _resolve_tokenizer,  # pyright: ignore[reportPrivateUsage]
 )
+from src.exceptions import ValidationException
 
 
 def gemini_call_texts(contents: Any) -> list[str]:
@@ -1047,3 +1049,236 @@ async def test_gemini_process_batch_wraps_contents_as_content_part(
     assert all(isinstance(c, genai_types.Content) for c in contents)
     assert contents[0].parts[0].text == "hello"
     assert contents[1].parts[0].text == "world"
+
+
+class FakeEncoding:
+    """Deterministic tokenizer double: every text maps to `token_count` tokens."""
+
+    def __init__(self, token_count: int) -> None:
+        self.token_count: int = token_count
+
+    def encode(self, _text: str) -> list[int]:
+        return list(range(self.token_count))
+
+    def decode(self, tokens: list[int]) -> str:
+        return " ".join(str(token) for token in tokens)
+
+
+def test_resolve_tokenizer_returns_model_encoding_for_known_models() -> None:
+    tokenizer = _resolve_tokenizer("text-embedding-3-small", None)
+    assert tokenizer.encode("hello") is not None
+
+
+def test_resolve_tokenizer_warns_and_falls_back_for_unknown_models(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    import logging
+
+    with caplog.at_level(logging.WARNING, logger="src.embedding_client"):
+        tokenizer = _resolve_tokenizer("baai/bge-m3", None)
+
+    assert tokenizer.encode("hello") is not None
+    assert any(
+        "falling back to cl100k_base" in record.message for record in caplog.records
+    )
+
+
+def test_resolve_tokenizer_blank_spec_uses_default() -> None:
+    default = _resolve_tokenizer("text-embedding-3-small", None)
+    blank = _resolve_tokenizer("text-embedding-3-small", "   ")
+    assert type(blank) is type(default)
+
+
+def test_resolve_tokenizer_tiktoken_spec_uses_named_encoding() -> None:
+    import tiktoken
+
+    tokenizer = _resolve_tokenizer("text-embedding-3-small", "tiktoken:o200k_base")
+    assert tokenizer is tiktoken.get_encoding("o200k_base")
+
+
+def test_resolve_tokenizer_rejects_unknown_tiktoken_encoding() -> None:
+    with pytest.raises(ValidationException, match="unknown tiktoken encoding"):
+        _resolve_tokenizer("text-embedding-3-small", "tiktoken:not-a-real-encoding")
+
+
+def test_resolve_tokenizer_rejects_empty_tiktoken_spec() -> None:
+    with pytest.raises(ValidationException, match="requires an encoding name"):
+        _resolve_tokenizer("text-embedding-3-small", "tiktoken:")
+
+
+def test_resolve_tokenizer_rejects_unknown_prefix() -> None:
+    with pytest.raises(ValidationException, match="tiktoken:, hf:, file:"):
+        _resolve_tokenizer("text-embedding-3-small", "sentencepiece:foo")
+
+
+def test_resolve_tokenizer_rejects_empty_hf_and_file_specs() -> None:
+    with pytest.raises(ValidationException, match="requires a model name"):
+        _resolve_tokenizer("text-embedding-3-small", "hf:")
+    with pytest.raises(ValidationException, match="requires a tokenizer path"):
+        _resolve_tokenizer("text-embedding-3-small", "file:")
+
+
+def test_resolve_tokenizer_hf_requires_tokenizers_package(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def fake_import_module(name: str) -> Any:
+        raise ImportError(name)
+
+    monkeypatch.setattr("src.embedding_client.import_module", fake_import_module)
+
+    with pytest.raises(ValidationException, match=r"honcho\[tokenizers\]"):
+        _resolve_tokenizer("text-embedding-3-small", "hf:BAAI/bge-m3")
+
+
+def test_resolve_tokenizer_hf_from_pretrained(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    calls: list[str] = []
+
+    class FakeTokenizerCls:
+        @staticmethod
+        def from_pretrained(model_name: str) -> Any:
+            calls.append(model_name)
+
+            def _encode(_text: str, add_special_tokens: bool = True) -> Any:
+                _ = add_special_tokens
+                return SimpleNamespace(ids=[1, 2, 3])
+
+            def _decode(_ids: list[int], skip_special_tokens: bool = True) -> str:
+                _ = skip_special_tokens
+                return "decoded"
+
+            return SimpleNamespace(encode=_encode, decode=_decode)
+
+    def _fake_import_module(_name: str) -> Any:
+        return SimpleNamespace(Tokenizer=FakeTokenizerCls)
+
+    monkeypatch.setattr("src.embedding_client.import_module", _fake_import_module)
+
+    tokenizer = _resolve_tokenizer("text-embedding-3-small", "hf:BAAI/bge-m3")
+
+    assert calls == ["BAAI/bge-m3"]
+    assert tokenizer.encode("anything") == [1, 2, 3]
+    assert tokenizer.decode([1, 2, 3]) == "decoded"
+
+
+def test_resolve_tokenizer_loads_from_file(tmp_path: Any) -> None:
+    tokenizers = pytest.importorskip("tokenizers")
+
+    tokenizer = tokenizers.Tokenizer(
+        tokenizers.models.WordLevel({"hello": 0, "world": 1})
+    )
+    tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()
+    path = tmp_path / "tokenizer.json"
+    tokenizer.save(str(path))
+
+    loaded = _resolve_tokenizer("text-embedding-3-small", f"file:{path}")
+
+    assert loaded.encode("hello world") == [0, 1]
+    assert loaded.decode([0, 1]) == "hello world"
+
+
+def test_embedding_client_uses_configured_tokenizer_for_chunking(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Chunk-size decisions must follow the configured tokenizer, not the
+    tiktoken fallback — the core of issue #827."""
+    fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 4)
+
+    class FakeOpenAIClient:
+        def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
+            self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings
+
+    monkeypatch.setattr("openai.AsyncOpenAI", FakeOpenAIClient)
+
+    def _fake_get_encoding(name: str) -> FakeEncoding | None:
+        return FakeEncoding(token_count=100) if name == "fake-enc" else None
+
+    monkeypatch.setattr(
+        "src.embedding_client.tiktoken.get_encoding", _fake_get_encoding
+    )
+
+    client = _EmbeddingClient(
+        EmbeddingModelConfig(
+            transport="openai",
+            model="text-embedding-3-small",
+            api_key="test-key",
+            tokenizer="tiktoken:fake-enc",
+        ),
+        vector_dimensions=4,
+        max_input_tokens=10,
+        max_tokens_per_request=1000,
+        send_dimensions=False,
+    )
+
+    # cl100k_base would count "hi" as 1 token and not chunk; the configured
+    # tokenizer counts 100 tokens, so it must be split into <=10-token chunks.
+    chunks = client.prepare_chunks({"msg": "hi"})["msg"]
+
+    assert len(chunks) > 1
+    for chunk in chunks:
+        assert len(chunk.split()) <= 10
+
+
+def test_hf_tokenizer_special_tokens_reserved_from_budget(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """BERT-family providers count [CLS]/[SEP] toward the input limit, so the
+    client's effective budget must shrink by the special-token overhead."""
+
+    def _encode(text: str, add_special_tokens: bool = True) -> Any:
+        ids = [10 + i for i, _ in enumerate(text.split())]
+        return SimpleNamespace(ids=([0, *ids, 1] if add_special_tokens else ids))
+
+    def _decode(ids: list[int], **_kwargs: Any) -> str:
+        return " ".join("w" for _ in ids)
+
+    class FakeTokenizerCls:
+        @staticmethod
+        def from_file(_path: str) -> Any:
+            return SimpleNamespace(encode=_encode, decode=_decode)
+
+    def _fake_import_module(_name: str) -> Any:
+        return SimpleNamespace(Tokenizer=FakeTokenizerCls)
+
+    monkeypatch.setattr("src.embedding_client.import_module", _fake_import_module)
+
+    fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 4)
+
+    class FakeOpenAIClient:
+        def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
+            self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings
+
+    monkeypatch.setattr("openai.AsyncOpenAI", FakeOpenAIClient)
+
+    client = _EmbeddingClient(
+        EmbeddingModelConfig(
+            transport="openai",
+            model="baai/bge-m3",
+            api_key="test-key",
+            tokenizer="file:/tmp/whatever.json",
+        ),
+        vector_dimensions=4,
+        max_input_tokens=100,
+        max_tokens_per_request=1000,
+        send_dimensions=False,
+    )
+
+    # 2 specials ([CLS]/[SEP]) reserved: budget is 98, not 100.
+    assert client.max_embedding_tokens == 98
+
+
+def test_settings_signature_tracks_tokenizer(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from src.config import settings
+    from src.embedding_client import EmbeddingClient
+
+    wrapper = EmbeddingClient()
+    before = wrapper._get_settings_signature()  # pyright: ignore[reportPrivateUsage]
+    monkeypatch.setattr(
+        settings.EMBEDDING.MODEL_CONFIG, "tokenizer", "tiktoken:o200k_base"
+    )
+    after = wrapper._get_settings_signature()  # pyright: ignore[reportPrivateUsage]
+
+    assert before != after
diff --git a/uv.lock b/uv.lock
index 82f85e04..daa707cf 100644
--- a/uv.lock
+++ b/uv.lock
@@ -8,7 +8,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-08-07T23:49:08.393963Z"
+exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
 exclude-newer-span = "P5D"
 
 [manifest]
@@ -466,14 +466,14 @@ wheels = [
 
 [[package]]
 name = "click"
-version = "8.3.3"
+version = "8.4.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "colorama", marker = "sys_platform == 'win32'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" },
 ]
 
 [[package]]
@@ -905,6 +905,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" },
 ]
 
+[[package]]
+name = "fsspec"
+version = "2026.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" },
+]
+
 [[package]]
 name = "google-auth"
 version = "2.52.0"
@@ -1022,6 +1031,38 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
 ]
 
+[[package]]
+name = "hf-xet"
+version = "1.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/64/ee/dd9ba7beae1005e54131b7d45263cc74c8a066d47d354e6d58ae9445a388/hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", size = 4069485, upload-time = "2026-06-08T23:02:13.193Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/bc/9cae6cfeb4e03070874e73e5c97c66eb90369d3206b6a2b1ef5f96520888/hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", size = 3838493, upload-time = "2026-06-08T23:02:15.282Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/b4/d5c01e0eb6d9f2ca2dacd84d0d1b71e6cfbb2ef3208c968528e010e9b3d7/hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", size = 4505658, upload-time = "2026-06-08T23:02:17.196Z" },
+    { url = "https://files.pythonhosted.org/packages/76/c5/29a7598c0c6383c523dc22186d577f4e04267a626cd95ae60f67c00bfe66/hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", size = 4292822, upload-time = "2026-06-08T23:02:18.608Z" },
+    { url = "https://files.pythonhosted.org/packages/04/9a/dceaf6ca69390126b86ea825fb354b93d01163199070b7bd849225de9468/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", size = 4491255, upload-time = "2026-06-08T23:02:20.124Z" },
+    { url = "https://files.pythonhosted.org/packages/48/a7/e5a7afaacf6c1791fdbeeac42951fb81c3d2bc482992b115dedcc86d963e/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", size = 4711062, upload-time = "2026-06-08T23:02:21.863Z" },
+    { url = "https://files.pythonhosted.org/packages/53/49/2802f8433c9742ce281bddc1e65c02c32268ca3098d66828b05e12e45ee2/hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", size = 4017205, upload-time = "2026-06-08T23:02:23.495Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/5a/50c71195b9fb883659f596e7252faf4c18c58e753a9013bdbf9bac5d2250/hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", size = 3845426, upload-time = "2026-06-08T23:02:25.124Z" },
+    { url = "https://files.pythonhosted.org/packages/05/24/5e0c28f80371c17d49fed004597d9d132cb75c1f6f53db2cb95f459d2312/hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", size = 4069676, upload-time = "2026-06-08T23:02:26.759Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/17/261ba565b6a4d960fb478f61fdf919c0be5824645aaf1c319eca660c1611/hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", size = 3838509, upload-time = "2026-06-08T23:02:28.573Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/44/7ffdc2e184b0d41fc0f683ba3936ef669ab63cf242cf36ef50e57d683668/hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", size = 4505881, upload-time = "2026-06-08T23:02:30.257Z" },
+    { url = "https://files.pythonhosted.org/packages/63/b6/788060d5aa4d5e671f1a31bf69624c314eb2d8babab3aa562f9e5d53444e/hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", size = 4292995, upload-time = "2026-06-08T23:02:31.993Z" },
+    { url = "https://files.pythonhosted.org/packages/22/93/c5540cbd6b55529b7dc42f6734e88cebee21aefbea34128b66229df56c57/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", size = 4491570, upload-time = "2026-06-08T23:02:33.86Z" },
+    { url = "https://files.pythonhosted.org/packages/03/f3/9d8ceab30f44f36c1679b1b8683054c71a0dadc787dbf07421891742d3ca/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", size = 4711565, upload-time = "2026-06-08T23:02:35.454Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/54/27ed9a5e2cc583b4df82f75a03a4df8dbf55f5a9fa1f47f1fadfb20dbeac/hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", size = 4017343, upload-time = "2026-06-08T23:02:37.14Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/12/ecb2fc8d45e767580e3a37faa97cb895608b614965567efb4f18cff67e27/hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", size = 3845716, upload-time = "2026-06-08T23:02:39.073Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" },
+    { url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" },
+    { url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" },
+    { url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" },
+]
+
 [[package]]
 name = "honcho"
 version = "3.0.12"
@@ -1063,6 +1104,9 @@ lancedb = [
     { name = "lancedb", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
     { name = "pyarrow" },
 ]
+tokenizers = [
+    { name = "tokenizers" },
+]
 
 [package.dev-dependencies]
 dev = [
@@ -1113,10 +1157,11 @@ requires-dist = [
     { name = "sqlalchemy", specifier = ">=2.0.30" },
     { name = "tenacity", specifier = ">=9.1.2" },
     { name = "tiktoken", specifier = ">=0.9.0" },
+    { name = "tokenizers", marker = "extra == 'tokenizers'", specifier = ">=0.22.0" },
     { name = "turbopuffer", specifier = ">=1.8.1" },
     { name = "typing-extensions", specifier = ">=4.11.0" },
 ]
-provides-extras = ["lancedb"]
+provides-extras = ["lancedb", "tokenizers"]
 
 [package.metadata.requires-dev]
 dev = [
@@ -1262,6 +1307,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
 ]
 
+[[package]]
+name = "huggingface-hub"
+version = "1.23.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "click" },
+    { name = "filelock" },
+    { name = "fsspec" },
+    { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
+    { name = "httpx" },
+    { name = "packaging" },
+    { name = "pyyaml" },
+    { name = "tqdm" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1a/8f/999e4dda11c6187c78f090eac00895a47e11a0049308f07579bcb7aa3aa2/huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88", size = 919163, upload-time = "2026-07-09T14:49:32.315Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f1/ce/13b2ba57838b8db1e6bd033c1b21ce0b9f6153b87d4e4939f77074e41eb0/huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2", size = 770336, upload-time = "2026-07-09T14:49:30.597Z" },
+]
+
 [[package]]
 name = "identify"
 version = "2.6.19"
@@ -3538,6 +3603,33 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" },
 ]
 
+[[package]]
+name = "tokenizers"
+version = "0.23.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "huggingface-hub" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" },
+    { url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" },
+    { url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" },
+    { url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" },
+    { url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" },
+    { url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" },
+    { url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" },
+    { url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" },
+]
+
 [[package]]
 name = "tomli"
 version = "2.4.1"