Merge 2c3ac50e71 into 699c99368d
This commit is contained in:
Vansh Sharma
GitHub
commit
ceacb323d7
|
|
@ -25,6 +25,7 @@ PERFORMANCE_LOG_FORMAT=compact # compact|rich
|
|||
# EMBEDDING_MODEL_CONFIG__TRANSPORT=openai
|
||||
# EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small
|
||||
# EMBEDDING_MODEL_CONFIG__MAX_BATCH_SIZE=10
|
||||
# EMBEDDING_MODEL_CONFIG__TOKENIZER=
|
||||
# EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=
|
||||
# EMBEDDING_MODEL_CONFIG__OVERRIDES__API_KEY_ENV=
|
||||
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ model = "text-embedding-3-small"
|
|||
# max_batch_size = 10
|
||||
# Optional client HTTP timeout in seconds (OpenAI + Gemini).
|
||||
# timeout = 90.0
|
||||
# tokenizer = "tiktoken:cl100k_base" # or "hf:BAAI/bge-m3" / "file:/path/to/tokenizer.json"
|
||||
|
||||
# Optional module-level endpoint overrides
|
||||
# [embedding.model_config.overrides]
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ Concretely, for either a dim change or a model change:
|
|||
export EMBEDDING_MODEL_CONFIG__TRANSPORT=openai
|
||||
export EMBEDDING_MODEL_CONFIG__MODEL=nomic-embed-text
|
||||
export EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=http://your-ollama:11434/v1
|
||||
# Non-OpenAI models need a matching tokenizer; otherwise chunk-size decisions
|
||||
# use Honcho's tiktoken fallback and can be rejected by the provider (see #827).
|
||||
# export EMBEDDING_MODEL_CONFIG__TOKENIZER=hf:nomic-ai/nomic-embed-text
|
||||
alembic upgrade head
|
||||
uv run python scripts/configure_embeddings.py --dry-run
|
||||
uv run python scripts/configure_embeddings.py --yes
|
||||
|
|
|
|||
|
|
@ -268,6 +268,7 @@ EMBEDDING_MODEL_CONFIG__TRANSPORT=openai # openai, gemini
|
|||
EMBEDDING_MODEL_CONFIG__MODEL=text-embedding-3-small
|
||||
EMBEDDING_MODEL_CONFIG__MAX_BATCH_SIZE=10 # optional per-request input cap
|
||||
EMBEDDING_MODEL_CONFIG__TIMEOUT=90.0 # optional client HTTP timeout (seconds)
|
||||
# EMBEDDING_MODEL_CONFIG__TOKENIZER= # unset = tiktoken auto-detect (default)
|
||||
|
||||
# Optional endpoint overrides
|
||||
EMBEDDING_MODEL_CONFIG__OVERRIDES__BASE_URL=http://localhost:8000/v1
|
||||
|
|
@ -287,6 +288,15 @@ milliseconds on `http_options.timeout`, and keeps its existing 10-minute
|
|||
default when unset. The value is validated at config load the same way as
|
||||
LLM `provider_params.timeout` (positive, finite number of seconds).
|
||||
|
||||
`EMBEDDING_MODEL_CONFIG__TOKENIZER` is optional. Leave it unset for Honcho's
|
||||
current tiktoken auto-detection behavior. For embedding models with a
|
||||
different tokenizer (OpenAI-compatible or Gemini), set `tiktoken:<encoding>`,
|
||||
`hf:<repo-id>`, or `file:/absolute/path/to/tokenizer.json`. Hugging Face and
|
||||
file tokenizers require installing the optional `honcho[tokenizers]` extra.
|
||||
When the tokenizer does not match the model's real tokenizer, chunk-size
|
||||
decisions can undercount tokens and the provider may reject oversized inputs
|
||||
(see issue #827).
|
||||
|
||||
Forwarding `dimensions=` to OpenAI-compatible providers is controlled by `EMBEDDING_MODEL_CONFIG__DIMENSIONS_MODE`:
|
||||
|
||||
- `auto` (default): forwards `dimensions=` when **the operator has explicitly set `EMBEDDING_VECTOR_DIMENSIONS`** — provenance, not value — and the configured model is not on the known-rejecting list (currently `text-embedding-ada-002`). Explicit `EMBEDDING_VECTOR_DIMENSIONS=1536` *does* trigger the forward; this is how `text-embedding-3-large` truncation to 1536 is expressed. Deployments that leave the setting unset get their existing behavior (`dimensions=` is not forwarded).
|
||||
|
|
|
|||
|
|
@ -43,6 +43,9 @@ lancedb = [
|
|||
"lancedb>=0.25.3; sys_platform != \"darwin\" or platform_machine != \"x86_64\"",
|
||||
"pyarrow>=19.0.0",
|
||||
]
|
||||
tokenizers = [
|
||||
"tokenizers>=0.22.0",
|
||||
]
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.2.2",
|
||||
|
|
|
|||
|
|
@ -396,6 +396,7 @@ class ConfiguredEmbeddingModelSettings(BaseModel):
|
|||
max_batch_size: Annotated[int, Field(gt=0)] | None = None
|
||||
# Client HTTP timeout in seconds. OpenAI receives seconds; Gemini converts to ms.
|
||||
timeout: float | None = None
|
||||
tokenizer: str | None = None
|
||||
|
||||
@field_validator("timeout", mode="before")
|
||||
@classmethod
|
||||
|
|
@ -442,6 +443,7 @@ class EmbeddingModelConfig(BaseModel):
|
|||
max_batch_size: Annotated[int, Field(gt=0)] | None = None
|
||||
# Client HTTP timeout in seconds. OpenAI receives seconds; Gemini converts to ms.
|
||||
timeout: float | None = None
|
||||
tokenizer: str | None = None
|
||||
|
||||
@field_validator("timeout", mode="before")
|
||||
@classmethod
|
||||
|
|
@ -575,6 +577,7 @@ def resolve_embedding_model_config(
|
|||
base_url=configured.overrides.base_url,
|
||||
max_batch_size=configured.max_batch_size,
|
||||
timeout=configured.timeout,
|
||||
tokenizer=configured.tokenizer,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,13 @@ async def run_deriver():
|
|||
# gate the API runs in its lifespan. Inside the try block so the
|
||||
# telemetry buffer is still flushed if validation raises.
|
||||
await validate_embedding_schema(engine)
|
||||
# Eagerly build the embedding client so an hf: tokenizer download
|
||||
# happens here, not on the first reconciler call under the lock.
|
||||
# Run it in a worker thread so the blocking download doesn't stall
|
||||
# the event loop.
|
||||
from src.embedding_client import embedding_client
|
||||
|
||||
await asyncio.to_thread(embedding_client.warmup)
|
||||
await main()
|
||||
finally:
|
||||
# Shutdown telemetry (flush CloudEvents buffer)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ import threading
|
|||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, TypeVar, cast
|
||||
from importlib import import_module
|
||||
from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Protocol, TypeVar, cast
|
||||
|
||||
import tiktoken
|
||||
from nanoid import generate as generate_nanoid
|
||||
|
|
@ -17,6 +18,7 @@ from .config import (
|
|||
resolve_embedding_model_config,
|
||||
settings,
|
||||
)
|
||||
from .exceptions import ValidationException
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from google import genai
|
||||
|
|
@ -179,6 +181,118 @@ class BatchItem(NamedTuple):
|
|||
token_count: int
|
||||
|
||||
|
||||
class TokenizerLike(Protocol):
|
||||
"""Minimal encode/decode surface the embedding pipeline needs from a tokenizer."""
|
||||
|
||||
def encode(self, text: str) -> list[int]: ...
|
||||
|
||||
def decode(self, tokens: list[int]) -> str: ...
|
||||
|
||||
|
||||
class _HuggingFaceTokenizer:
|
||||
"""Adapter making a `tokenizers.Tokenizer` satisfy TokenizerLike."""
|
||||
|
||||
def __init__(self, tokenizer: Any) -> None:
|
||||
self._tokenizer: Any = tokenizer
|
||||
# Special tokens ([CLS]/[SEP]) the provider adds per input also count
|
||||
# toward its token limit, so they must come out of the chunk budget.
|
||||
self.special_tokens_overhead: int = len(
|
||||
tokenizer.encode("", add_special_tokens=True).ids
|
||||
)
|
||||
|
||||
def encode(self, text: str) -> list[int]:
|
||||
return list(self._tokenizer.encode(text, add_special_tokens=False).ids)
|
||||
|
||||
def decode(self, tokens: list[int]) -> str:
|
||||
return str(self._tokenizer.decode(tokens, skip_special_tokens=True))
|
||||
|
||||
|
||||
def _load_huggingface_tokenizer(spec: str) -> TokenizerLike:
|
||||
if spec.startswith("hf:"):
|
||||
model_name = spec.removeprefix("hf:")
|
||||
if not model_name:
|
||||
raise ValidationException(
|
||||
"Embedding tokenizer spec 'hf:' requires a model name"
|
||||
)
|
||||
tokenizer_cls = _import_tokenizers_cls()
|
||||
return _HuggingFaceTokenizer(tokenizer_cls.from_pretrained(model_name))
|
||||
|
||||
path = spec.removeprefix("file:")
|
||||
if not path:
|
||||
raise ValidationException(
|
||||
"Embedding tokenizer spec 'file:' requires a tokenizer path"
|
||||
)
|
||||
tokenizer_cls = _import_tokenizers_cls()
|
||||
return _HuggingFaceTokenizer(tokenizer_cls.from_file(path))
|
||||
|
||||
|
||||
def _import_tokenizers_cls() -> Any:
|
||||
"""Import the `tokenizers.Tokenizer` class, raising a clear ValidationException
|
||||
when the optional `tokenizers` package is not installed."""
|
||||
try:
|
||||
return import_module("tokenizers").Tokenizer
|
||||
except ImportError as exc:
|
||||
raise ValidationException(
|
||||
"The 'tokenizers' package is required for hf: and file: embedding "
|
||||
+ "tokenizers. Install it with the honcho[tokenizers] extra."
|
||||
) from exc
|
||||
|
||||
|
||||
def _default_tokenizer_for_model(model: str) -> TokenizerLike:
|
||||
try:
|
||||
return tiktoken.encoding_for_model(model)
|
||||
except KeyError:
|
||||
# The provider's real tokenizer may count differently (e.g. bge-m3's
|
||||
# XLM-RoBERTa tokenizer vs cl100k_base), which can push chunks the
|
||||
# provider then rejects. Set EMBEDDING_MODEL_CONFIG__TOKENIZER to match
|
||||
# the configured model's tokenizer.
|
||||
logger.warning(
|
||||
"No tiktoken encoding for embedding model %r; falling back to "
|
||||
+ "cl100k_base for token counting. If the provider uses a different "
|
||||
+ "tokenizer, chunk-size decisions may be wrong. Configure "
|
||||
+ "EMBEDDING_MODEL_CONFIG__TOKENIZER to override.",
|
||||
model,
|
||||
)
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
|
||||
def _resolve_tokenizer(model: str, spec: str | None) -> TokenizerLike:
|
||||
"""Resolve the tokenizer used for embedding token counting and chunking.
|
||||
|
||||
Args:
|
||||
model: Configured embedding model name (used when no spec is given)
|
||||
spec: Optional explicit tokenizer spec: "tiktoken:<encoding>",
|
||||
"hf:<repo-id>", or "file:/path/to/tokenizer.json"
|
||||
|
||||
Returns:
|
||||
A TokenizerLike for chunk-size decisions.
|
||||
"""
|
||||
if spec is None or spec.strip() == "":
|
||||
return _default_tokenizer_for_model(model)
|
||||
|
||||
spec = spec.strip()
|
||||
|
||||
if spec.startswith("tiktoken:"):
|
||||
encoding_name = spec.removeprefix("tiktoken:")
|
||||
if not encoding_name:
|
||||
raise ValidationException(
|
||||
"Embedding tokenizer spec 'tiktoken:' requires an encoding name"
|
||||
)
|
||||
try:
|
||||
return tiktoken.get_encoding(encoding_name)
|
||||
except ValueError as exc:
|
||||
raise ValidationException(
|
||||
f"Embedding tokenizer spec uses unknown tiktoken encoding: {encoding_name}"
|
||||
) from exc
|
||||
|
||||
if spec.startswith(("hf:", "file:")):
|
||||
return _load_huggingface_tokenizer(spec)
|
||||
|
||||
raise ValidationException(
|
||||
"Embedding tokenizer must be unset or start with one of: tiktoken:, hf:, file:"
|
||||
)
|
||||
|
||||
|
||||
class _EmbeddingClient:
|
||||
"""
|
||||
Embedding client supporting OpenAI and Gemini with chunking and batching support.
|
||||
|
|
@ -238,10 +352,19 @@ class _EmbeddingClient:
|
|||
self.max_embedding_tokens = max_input_tokens
|
||||
self.max_batch_size = config.max_batch_size or 2048
|
||||
|
||||
try:
|
||||
self.encoding: tiktoken.Encoding = tiktoken.encoding_for_model(self.model)
|
||||
except KeyError:
|
||||
self.encoding = tiktoken.get_encoding("cl100k_base")
|
||||
self.encoding: TokenizerLike = _resolve_tokenizer(self.model, config.tokenizer)
|
||||
# Providers count the special tokens they add per input (e.g. [CLS]/[SEP]
|
||||
# for BERT-family models) toward the limit; reserve them from the budget.
|
||||
self.max_embedding_tokens -= getattr(
|
||||
self.encoding, "special_tokens_overhead", 0
|
||||
)
|
||||
if self.max_embedding_tokens <= 0:
|
||||
msg = (
|
||||
f"Effective embedding token budget is {self.max_embedding_tokens} after"
|
||||
" reserving special-token overhead. Reduce"
|
||||
" EMBEDDING_MODEL_CONFIG__TOKENIZER overhead or raise EMBEDDING_MAX_INPUT_TOKENS."
|
||||
)
|
||||
raise ValidationException(msg)
|
||||
self.max_embedding_tokens_per_request: int = max_tokens_per_request
|
||||
|
||||
@property
|
||||
|
|
@ -644,7 +767,7 @@ def _chunk_text_with_tokens(
|
|||
text: str,
|
||||
encoded_tokens: list[int],
|
||||
max_tokens: int,
|
||||
encoding: tiktoken.Encoding,
|
||||
encoding: TokenizerLike,
|
||||
) -> list[tuple[str, int]]:
|
||||
"""
|
||||
Split text into chunks that fit within token limits, with 20% overlap.
|
||||
|
|
@ -653,7 +776,7 @@ def _chunk_text_with_tokens(
|
|||
text: Original text to chunk
|
||||
encoded_tokens: Pre-encoded tokens for the text
|
||||
max_tokens: Maximum tokens per chunk
|
||||
encoding: Tiktoken encoding model
|
||||
encoding: Tokenizer used to decode token slices back into text
|
||||
|
||||
Returns:
|
||||
List of (chunk_text, token_count) tuples
|
||||
|
|
@ -726,6 +849,30 @@ class EmbeddingClient:
|
|||
def _resolve_runtime_config(self) -> EmbeddingModelConfig:
|
||||
return resolve_embedding_model_config(settings.EMBEDDING.MODEL_CONFIG)
|
||||
|
||||
def warmup(self) -> None:
|
||||
"""Eagerly construct the underlying client at process startup.
|
||||
|
||||
``hf:`` tokenizer specs fetch from the Hugging Face Hub; doing that on
|
||||
the first request would stall the event loop under the singleton lock.
|
||||
Calling this from the API and deriver lifespans moves any tokenizer
|
||||
download off the request path and fails the process fast on a bad
|
||||
tokenizer spec. Any other construction error (e.g. a missing API key)
|
||||
is logged and deferred to first use, so deployments that never embed
|
||||
keep starting up exactly as they did before warmup existed.
|
||||
"""
|
||||
try:
|
||||
self._get_client()
|
||||
except ValidationException:
|
||||
# An explicitly configured tokenizer spec that cannot load is a
|
||||
# misconfiguration of this feature; keep failing fast.
|
||||
raise
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Embedding client warmup failed; embedding calls will fail "
|
||||
+ "until the embedding configuration is fixed",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _get_settings_signature(self) -> tuple[object, ...]:
|
||||
runtime_config = self._resolve_runtime_config()
|
||||
return (
|
||||
|
|
@ -734,6 +881,7 @@ class EmbeddingClient:
|
|||
runtime_config.api_key,
|
||||
runtime_config.base_url,
|
||||
runtime_config.max_batch_size,
|
||||
runtime_config.tokenizer,
|
||||
settings.EMBEDDING.VECTOR_DIMENSIONS,
|
||||
settings.EMBEDDING.MAX_INPUT_TOKENS,
|
||||
settings.EMBEDDING.MAX_TOKENS_PER_REQUEST,
|
||||
|
|
@ -792,20 +940,31 @@ class EmbeddingClient:
|
|||
return self._get_client().vector_dimensions
|
||||
|
||||
@property
|
||||
def encoding(self) -> tiktoken.Encoding:
|
||||
"""Get the tiktoken encoding.
|
||||
def encoding(self) -> TokenizerLike:
|
||||
"""Get the configured embedding tokenizer.
|
||||
|
||||
Resolved without constructing the underlying client: tiktoken needs no
|
||||
API key, and token-counting callers (e.g. the document dedup tie-break)
|
||||
must work in environments with no embedding credentials, such as CI for
|
||||
pull requests from forks.
|
||||
Resolved without constructing the underlying client when possible:
|
||||
token counting (e.g. the document dedup tie-break) must work in
|
||||
environments with no embedding credentials, such as CI for pull
|
||||
requests from forks. Only HF/file tokenizers require client
|
||||
construction (to load the tokenizer once and cache it on the
|
||||
instance), so for those specs we fall back to the constructed
|
||||
client's encoding.
|
||||
"""
|
||||
if self._instance is not None:
|
||||
if (
|
||||
self._instance is not None
|
||||
and self._instance_signature == self._get_settings_signature()
|
||||
):
|
||||
return self._instance.encoding
|
||||
try:
|
||||
return tiktoken.encoding_for_model(self._resolve_runtime_config().model)
|
||||
except KeyError:
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
runtime_config = self._resolve_runtime_config()
|
||||
spec = runtime_config.tokenizer
|
||||
if spec and spec.strip() and spec.strip().startswith(("hf:", "file:")):
|
||||
# HF/file tokenizers are loaded once inside _EmbeddingClient and
|
||||
# cached on the instance; resolving them here would either
|
||||
# duplicate the load or bypass the special_tokens_overhead
|
||||
# accounting. Delegate to the constructed client.
|
||||
return self._get_client().encoding
|
||||
return _resolve_tokenizer(runtime_config.model, spec)
|
||||
|
||||
|
||||
# Shared singleton embedding client instance
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
|
|
@ -24,6 +25,7 @@ from src.db import (
|
|||
register_db_query_instrumentation,
|
||||
request_context,
|
||||
)
|
||||
from src.embedding_client import embedding_client
|
||||
from src.exceptions import HonchoException
|
||||
from src.routers import (
|
||||
conclusions,
|
||||
|
|
@ -130,6 +132,12 @@ async def lifespan(_: FastAPI):
|
|||
# writing wrong-dim vectors.
|
||||
await validate_embedding_schema(engine)
|
||||
|
||||
# Eagerly build the embedding client so an hf: tokenizer download happens
|
||||
# here, not on the first request under the singleton lock. Run it in a
|
||||
# worker thread: the download is blocking network I/O and would otherwise
|
||||
# stall the event loop (signal handling, telemetry) during startup.
|
||||
await asyncio.to_thread(embedding_client.warmup)
|
||||
|
||||
try:
|
||||
await init_cache()
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ from src.embedding_client import (
|
|||
EmbeddingClient,
|
||||
EmbeddingTokenLimitError,
|
||||
_EmbeddingClient, # pyright: ignore[reportPrivateUsage]
|
||||
_resolve_tokenizer, # pyright: ignore[reportPrivateUsage]
|
||||
)
|
||||
from src.exceptions import ValidationException
|
||||
|
||||
|
||||
def gemini_call_texts(contents: Any) -> list[str]:
|
||||
|
|
@ -1176,6 +1178,292 @@ async def test_gemini_process_batch_wraps_contents_as_content_part(
|
|||
assert contents[1].parts[0].text == "world"
|
||||
|
||||
|
||||
class FakeEncoding:
|
||||
"""Deterministic tokenizer double: every text maps to `token_count` tokens."""
|
||||
|
||||
def __init__(self, token_count: int) -> None:
|
||||
self.token_count: int = token_count
|
||||
|
||||
def encode(self, _text: str) -> list[int]:
|
||||
return list(range(self.token_count))
|
||||
|
||||
def decode(self, tokens: list[int]) -> str:
|
||||
return " ".join(str(token) for token in tokens)
|
||||
|
||||
|
||||
def test_resolve_tokenizer_returns_model_encoding_for_known_models() -> None:
|
||||
tokenizer = _resolve_tokenizer("text-embedding-3-small", None)
|
||||
assert tokenizer.encode("hello") is not None
|
||||
|
||||
|
||||
def test_resolve_tokenizer_warns_and_falls_back_for_unknown_models(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
import logging
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="src.embedding_client"):
|
||||
tokenizer = _resolve_tokenizer("baai/bge-m3", None)
|
||||
|
||||
assert tokenizer.encode("hello") is not None
|
||||
assert any(
|
||||
"falling back to cl100k_base" in record.message for record in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_tokenizer_blank_spec_uses_default() -> None:
|
||||
default = _resolve_tokenizer("text-embedding-3-small", None)
|
||||
blank = _resolve_tokenizer("text-embedding-3-small", " ")
|
||||
assert type(blank) is type(default)
|
||||
|
||||
|
||||
def test_resolve_tokenizer_tiktoken_spec_uses_named_encoding() -> None:
|
||||
import tiktoken
|
||||
|
||||
tokenizer = _resolve_tokenizer("text-embedding-3-small", "tiktoken:o200k_base")
|
||||
assert tokenizer is tiktoken.get_encoding("o200k_base")
|
||||
|
||||
|
||||
def test_resolve_tokenizer_rejects_unknown_tiktoken_encoding() -> None:
|
||||
with pytest.raises(ValidationException, match="unknown tiktoken encoding"):
|
||||
_resolve_tokenizer("text-embedding-3-small", "tiktoken:not-a-real-encoding")
|
||||
|
||||
|
||||
def test_resolve_tokenizer_rejects_empty_tiktoken_spec() -> None:
|
||||
with pytest.raises(ValidationException, match="requires an encoding name"):
|
||||
_resolve_tokenizer("text-embedding-3-small", "tiktoken:")
|
||||
|
||||
|
||||
def test_resolve_tokenizer_rejects_unknown_prefix() -> None:
|
||||
with pytest.raises(ValidationException, match="tiktoken:, hf:, file:"):
|
||||
_resolve_tokenizer("text-embedding-3-small", "sentencepiece:foo")
|
||||
|
||||
|
||||
def test_resolve_tokenizer_rejects_empty_hf_and_file_specs() -> None:
|
||||
with pytest.raises(ValidationException, match="requires a model name"):
|
||||
_resolve_tokenizer("text-embedding-3-small", "hf:")
|
||||
with pytest.raises(ValidationException, match="requires a tokenizer path"):
|
||||
_resolve_tokenizer("text-embedding-3-small", "file:")
|
||||
|
||||
|
||||
def test_resolve_tokenizer_hf_requires_tokenizers_package(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def fake_import_module(name: str) -> Any:
|
||||
raise ImportError(name)
|
||||
|
||||
monkeypatch.setattr("src.embedding_client.import_module", fake_import_module)
|
||||
|
||||
with pytest.raises(ValidationException, match=r"honcho\[tokenizers\]"):
|
||||
_resolve_tokenizer("text-embedding-3-small", "hf:BAAI/bge-m3")
|
||||
|
||||
|
||||
def test_resolve_tokenizer_hf_from_pretrained(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeTokenizerCls:
|
||||
@staticmethod
|
||||
def from_pretrained(model_name: str) -> Any:
|
||||
calls.append(model_name)
|
||||
|
||||
def _encode(_text: str, add_special_tokens: bool = True) -> Any:
|
||||
_ = add_special_tokens
|
||||
return SimpleNamespace(ids=[1, 2, 3])
|
||||
|
||||
def _decode(_ids: list[int], skip_special_tokens: bool = True) -> str:
|
||||
_ = skip_special_tokens
|
||||
return "decoded"
|
||||
|
||||
return SimpleNamespace(encode=_encode, decode=_decode)
|
||||
|
||||
def _fake_import_module(_name: str) -> Any:
|
||||
return SimpleNamespace(Tokenizer=FakeTokenizerCls)
|
||||
|
||||
monkeypatch.setattr("src.embedding_client.import_module", _fake_import_module)
|
||||
|
||||
tokenizer = _resolve_tokenizer("text-embedding-3-small", "hf:BAAI/bge-m3")
|
||||
|
||||
assert calls == ["BAAI/bge-m3"]
|
||||
assert tokenizer.encode("anything") == [1, 2, 3]
|
||||
assert tokenizer.decode([1, 2, 3]) == "decoded"
|
||||
|
||||
|
||||
def test_resolve_tokenizer_loads_from_file(tmp_path: Any) -> None:
|
||||
tokenizers = pytest.importorskip("tokenizers")
|
||||
|
||||
tokenizer = tokenizers.Tokenizer(
|
||||
tokenizers.models.WordLevel({"hello": 0, "world": 1})
|
||||
)
|
||||
tokenizer.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()
|
||||
path = tmp_path / "tokenizer.json"
|
||||
tokenizer.save(str(path))
|
||||
|
||||
loaded = _resolve_tokenizer("text-embedding-3-small", f"file:{path}")
|
||||
|
||||
assert loaded.encode("hello world") == [0, 1]
|
||||
assert loaded.decode([0, 1]) == "hello world"
|
||||
|
||||
|
||||
def test_embedding_client_uses_configured_tokenizer_for_chunking(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Chunk-size decisions must follow the configured tokenizer, not the
|
||||
tiktoken fallback — the core of issue #827."""
|
||||
fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 4)
|
||||
|
||||
class FakeOpenAIClient:
|
||||
def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
|
||||
self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings
|
||||
|
||||
monkeypatch.setattr("openai.AsyncOpenAI", FakeOpenAIClient)
|
||||
|
||||
def _fake_get_encoding(name: str) -> FakeEncoding | None:
|
||||
return FakeEncoding(token_count=100) if name == "fake-enc" else None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"src.embedding_client.tiktoken.get_encoding", _fake_get_encoding
|
||||
)
|
||||
|
||||
client = _EmbeddingClient(
|
||||
EmbeddingModelConfig(
|
||||
transport="openai",
|
||||
model="text-embedding-3-small",
|
||||
api_key="test-key",
|
||||
tokenizer="tiktoken:fake-enc",
|
||||
),
|
||||
vector_dimensions=4,
|
||||
max_input_tokens=10,
|
||||
max_tokens_per_request=1000,
|
||||
send_dimensions=False,
|
||||
)
|
||||
|
||||
# cl100k_base would count "hi" as 1 token and not chunk; the configured
|
||||
# tokenizer counts 100 tokens, so it must be split into <=10-token chunks.
|
||||
chunks = client.prepare_chunks({"msg": "hi"})["msg"]
|
||||
|
||||
assert len(chunks) > 1
|
||||
for chunk in chunks:
|
||||
assert len(chunk.split()) <= 10
|
||||
|
||||
|
||||
def test_hf_tokenizer_special_tokens_reserved_from_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""BERT-family providers count [CLS]/[SEP] toward the input limit, so the
|
||||
client's effective budget must shrink by the special-token overhead."""
|
||||
|
||||
def _encode(text: str, add_special_tokens: bool = True) -> Any:
|
||||
ids = [10 + i for i, _ in enumerate(text.split())]
|
||||
return SimpleNamespace(ids=([0, *ids, 1] if add_special_tokens else ids))
|
||||
|
||||
def _decode(ids: list[int], **_kwargs: Any) -> str:
|
||||
return " ".join("w" for _ in ids)
|
||||
|
||||
class FakeTokenizerCls:
|
||||
@staticmethod
|
||||
def from_file(_path: str) -> Any:
|
||||
return SimpleNamespace(encode=_encode, decode=_decode)
|
||||
|
||||
def _fake_import_module(_name: str) -> Any:
|
||||
return SimpleNamespace(Tokenizer=FakeTokenizerCls)
|
||||
|
||||
monkeypatch.setattr("src.embedding_client.import_module", _fake_import_module)
|
||||
|
||||
fake_embeddings = FakeOpenAIEmbeddingsAPI([0.1] * 4)
|
||||
|
||||
class FakeOpenAIClient:
|
||||
def __init__(self, *, api_key: str | None, base_url: str | None) -> None:
|
||||
self.embeddings: FakeOpenAIEmbeddingsAPI = fake_embeddings
|
||||
|
||||
monkeypatch.setattr("openai.AsyncOpenAI", FakeOpenAIClient)
|
||||
|
||||
client = _EmbeddingClient(
|
||||
EmbeddingModelConfig(
|
||||
transport="openai",
|
||||
model="baai/bge-m3",
|
||||
api_key="test-key",
|
||||
tokenizer="file:/tmp/whatever.json",
|
||||
),
|
||||
vector_dimensions=4,
|
||||
max_input_tokens=100,
|
||||
max_tokens_per_request=1000,
|
||||
send_dimensions=False,
|
||||
)
|
||||
|
||||
# 2 specials ([CLS]/[SEP]) reserved: budget is 98, not 100.
|
||||
assert client.max_embedding_tokens == 98
|
||||
|
||||
|
||||
def test_settings_signature_tracks_tokenizer(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from src.config import settings
|
||||
from src.embedding_client import EmbeddingClient
|
||||
|
||||
wrapper = EmbeddingClient()
|
||||
before = wrapper._get_settings_signature() # pyright: ignore[reportPrivateUsage]
|
||||
monkeypatch.setattr(
|
||||
settings.EMBEDDING.MODEL_CONFIG, "tokenizer", "tiktoken:o200k_base"
|
||||
)
|
||||
after = wrapper._get_settings_signature() # pyright: ignore[reportPrivateUsage]
|
||||
|
||||
assert before != after
|
||||
|
||||
|
||||
def test_warmup_defers_missing_api_key_instead_of_failing_startup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
"""A keyless embedding config must not crash startup: warmup logs and
|
||||
defers the failure to first use, matching pre-warmup lazy behavior."""
|
||||
from src.embedding_client import EmbeddingClient
|
||||
|
||||
wrapper = EmbeddingClient()
|
||||
monkeypatch.setattr(EmbeddingClient, "_instance", None)
|
||||
monkeypatch.setattr(EmbeddingClient, "_instance_signature", None)
|
||||
monkeypatch.setattr(
|
||||
wrapper,
|
||||
"_resolve_runtime_config",
|
||||
lambda: EmbeddingModelConfig(
|
||||
transport="openai",
|
||||
model="text-embedding-3-small",
|
||||
api_key=None,
|
||||
),
|
||||
)
|
||||
|
||||
with caplog.at_level("WARNING", logger="src.embedding_client"):
|
||||
wrapper.warmup()
|
||||
|
||||
assert EmbeddingClient._instance is None # pyright: ignore[reportPrivateUsage]
|
||||
assert "warmup failed" in caplog.text
|
||||
|
||||
|
||||
def test_warmup_still_fails_fast_on_bad_tokenizer_spec(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""An explicitly configured tokenizer spec that cannot load is a
|
||||
misconfiguration of the tokenizer feature and must keep failing startup."""
|
||||
from src.embedding_client import EmbeddingClient
|
||||
|
||||
wrapper = EmbeddingClient()
|
||||
monkeypatch.setattr(EmbeddingClient, "_instance", None)
|
||||
monkeypatch.setattr(EmbeddingClient, "_instance_signature", None)
|
||||
monkeypatch.setattr(
|
||||
wrapper,
|
||||
"_resolve_runtime_config",
|
||||
lambda: EmbeddingModelConfig(
|
||||
transport="openai",
|
||||
model="text-embedding-3-small",
|
||||
api_key="test-key",
|
||||
tokenizer="tiktoken:",
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationException, match="requires an encoding name"):
|
||||
wrapper.warmup()
|
||||
|
||||
|
||||
# --- Token-limit classification (issue #568) -------------------------------
|
||||
#
|
||||
# Only genuine "content too long" conditions may raise
|
||||
|
|
|
|||
154
uv.lock
154
uv.lock
|
|
@ -359,14 +359,14 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.3.3"
|
||||
version = "8.4.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061, upload-time = "2026-04-22T15:11:27.506Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502, upload-time = "2026-04-22T15:11:25.044Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -725,6 +725,15 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fsspec"
|
||||
version = "2026.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/10/a1/ae4e3e5003468d6391d2c77b6fa1cd73bd5d13511d81c642d7b28ac90ed4/fsspec-2026.6.0.tar.gz", hash = "sha256:f5bac145310fe30e16e1471bd6840b2d990d609e872251d7e674241822abf01a", size = 313646, upload-time = "2026-06-16T01:57:28.105Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-auth"
|
||||
version = "2.52.0"
|
||||
|
|
@ -822,6 +831,38 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hf-xet"
|
||||
version = "1.5.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/64/ee/dd9ba7beae1005e54131b7d45263cc74c8a066d47d354e6d58ae9445a388/hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", size = 4069485, upload-time = "2026-06-08T23:02:13.193Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/bc/9cae6cfeb4e03070874e73e5c97c66eb90369d3206b6a2b1ef5f96520888/hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", size = 3838493, upload-time = "2026-06-08T23:02:15.282Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ba/b4/d5c01e0eb6d9f2ca2dacd84d0d1b71e6cfbb2ef3208c968528e010e9b3d7/hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", size = 4505658, upload-time = "2026-06-08T23:02:17.196Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/c5/29a7598c0c6383c523dc22186d577f4e04267a626cd95ae60f67c00bfe66/hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", size = 4292822, upload-time = "2026-06-08T23:02:18.608Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/9a/dceaf6ca69390126b86ea825fb354b93d01163199070b7bd849225de9468/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", size = 4491255, upload-time = "2026-06-08T23:02:20.124Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/a7/e5a7afaacf6c1791fdbeeac42951fb81c3d2bc482992b115dedcc86d963e/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", size = 4711062, upload-time = "2026-06-08T23:02:21.863Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/53/49/2802f8433c9742ce281bddc1e65c02c32268ca3098d66828b05e12e45ee2/hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", size = 4017205, upload-time = "2026-06-08T23:02:23.495Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/5a/50c71195b9fb883659f596e7252faf4c18c58e753a9013bdbf9bac5d2250/hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", size = 3845426, upload-time = "2026-06-08T23:02:25.124Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/24/5e0c28f80371c17d49fed004597d9d132cb75c1f6f53db2cb95f459d2312/hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", size = 4069676, upload-time = "2026-06-08T23:02:26.759Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/17/261ba565b6a4d960fb478f61fdf919c0be5824645aaf1c319eca660c1611/hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", size = 3838509, upload-time = "2026-06-08T23:02:28.573Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4e/44/7ffdc2e184b0d41fc0f683ba3936ef669ab63cf242cf36ef50e57d683668/hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", size = 4505881, upload-time = "2026-06-08T23:02:30.257Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/63/b6/788060d5aa4d5e671f1a31bf69624c314eb2d8babab3aa562f9e5d53444e/hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", size = 4292995, upload-time = "2026-06-08T23:02:31.993Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/93/c5540cbd6b55529b7dc42f6734e88cebee21aefbea34128b66229df56c57/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", size = 4491570, upload-time = "2026-06-08T23:02:33.86Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/f3/9d8ceab30f44f36c1679b1b8683054c71a0dadc787dbf07421891742d3ca/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", size = 4711565, upload-time = "2026-06-08T23:02:35.454Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/54/27ed9a5e2cc583b4df82f75a03a4df8dbf55f5a9fa1f47f1fadfb20dbeac/hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", size = 4017343, upload-time = "2026-06-08T23:02:37.14Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ae/12/ecb2fc8d45e767580e3a37faa97cb895608b614965567efb4f18cff67e27/hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", size = 3845716, upload-time = "2026-06-08T23:02:39.073Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "honcho"
|
||||
version = "3.1.1"
|
||||
|
|
@ -863,6 +904,9 @@ lancedb = [
|
|||
{ name = "lancedb", marker = "platform_machine != 'x86_64' or sys_platform != 'darwin'" },
|
||||
{ name = "pyarrow" },
|
||||
]
|
||||
tokenizers = [
|
||||
{ name = "tokenizers" },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
dev = [
|
||||
|
|
@ -913,10 +957,11 @@ requires-dist = [
|
|||
{ name = "sqlalchemy", specifier = ">=2.0.30" },
|
||||
{ name = "tenacity", specifier = ">=9.1.2" },
|
||||
{ name = "tiktoken", specifier = ">=0.9.0" },
|
||||
{ name = "tokenizers", marker = "extra == 'tokenizers'", specifier = ">=0.22.0" },
|
||||
{ name = "turbopuffer", specifier = ">=1.8.1" },
|
||||
{ name = "typing-extensions", specifier = ">=4.11.0" },
|
||||
]
|
||||
provides-extras = ["lancedb"]
|
||||
provides-extras = ["lancedb", "tokenizers"]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
|
|
@ -1047,6 +1092,26 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "huggingface-hub"
|
||||
version = "1.23.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "click" },
|
||||
{ name = "filelock" },
|
||||
{ name = "fsspec" },
|
||||
{ name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" },
|
||||
{ name = "httpx" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "tqdm" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1a/8f/999e4dda11c6187c78f090eac00895a47e11a0049308f07579bcb7aa3aa2/huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88", size = 919163, upload-time = "2026-07-09T14:49:32.315Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/ce/13b2ba57838b8db1e6bd033c1b21ce0b9f6153b87d4e4939f77074e41eb0/huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2", size = 770336, upload-time = "2026-07-09T14:49:30.597Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "identify"
|
||||
version = "2.6.19"
|
||||
|
|
@ -2844,6 +2909,87 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokenizers"
|
||||
version = "0.23.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "huggingface-hub" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c1/60/21f715d9faba5f5407ff759472ade058ec4a507ad62bcea47cb847239a73/tokenizers-0.23.1.tar.gz", hash = "sha256:1feeeadf865a7915adc25445dea30e9933e593c31bb96c277cee36de227c8bfa", size = 365748, upload-time = "2026-04-27T14:43:25.606Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/87/39/b87a87d5bb9470610b80a2d31df42fcffeaf35118b8b97952b2aff598cc7/tokenizers-0.23.1-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e03d6ffcbe0d56ee9c1ccd070e70a13fa750727c0277e138152acbc0252c2224", size = 3146732, upload-time = "2026-04-27T14:43:15.427Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/6a/068ed9f6e444c9d7e9d55ce134181325700f3d7f30410721bdc8f848d727/tokenizers-0.23.1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e0948bbb1ac1d7cdfc9fb6d62c596e3b7550036ad60ecd654a66ad273326324e", size = 3054954, upload-time = "2026-04-27T14:43:13.745Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6c/36/e006edf031154cba92b8416057d92c3abe3635e4c4b0aa0b5b9bb39dde70/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bf13402aff9bc533c89cb849ec3b412dc3fbeacc9744840e423d7bf3f7dc0e3", size = 3374081, upload-time = "2026-04-27T14:43:01.241Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a2/ef/7735d226f9c7f874a6bee5e3f27fb25ecabdf207d37b8cf45286d0795893/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f836ca703b89ae07919a309f9651f7a88fd5a33d5f718ba5ad0870ec0256bad6", size = 3247641, upload-time = "2026-04-27T14:43:03.856Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/d9/24827036f6e21297bfffda0768e58eb6096a4f411e932964a01707857931/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae848657742035523fdf261773630cb819a26995fcd3d9ecae0c1daf6e5a4959", size = 3585624, upload-time = "2026-04-27T14:43:10.664Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/9a/22f3582b3a4f49358293a5206e25317621ee4526bfe9cdaa0f07a12e770e/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53b09e85775d5187941e7bab30e941b4134ab4a7dd8c68e783d231fb7ca27c51", size = 3844062, upload-time = "2026-04-27T14:43:05.643Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7e/65/b8f8814eef95800f20721384136d9a1d22241d50b2874357cb70542c392f/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea5a0ce170074329faaa8ea3f6400ecde604b6678192688533af80980daae71a", size = 3460098, upload-time = "2026-04-27T14:43:08.854Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/d5/1353e5f677ec27c2494fb6a6725e82d56c985f53e90ec511369e7e4f02c6/tokenizers-0.23.1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5075b405006415ea148a992d093699c66eb01952bf59f4d5727089a98bda45a4", size = 3346235, upload-time = "2026-04-27T14:43:12.377Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/89/39b6b8fc073fb6d413d0147aa333dc7eff7be65639ac9d19930a0b21bf33/tokenizers-0.23.1-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:56f3a77de629917652f876294dc9fe6bad4a0c43bc229dc72e59bb23a0f4729a", size = 3426398, upload-time = "2026-04-27T14:43:07.264Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/80/127c854da64827e5b79264ce524993a90dddcb320e5cd42412c5c02f9e8a/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9d10a6d957ef01896dc274e890eee27d41bd0e74ef31e60616f0fc311345184e", size = 9823279, upload-time = "2026-04-27T14:43:17.222Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/fe/ba/44c2502feb1a058f096ddfb4e0996ef3225a01a388e1a9b094e91689fe93/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:1974288a609c343774f1b897c8b482c791ab17b75ab5c8c2b1737565c1d82288", size = 9644986, upload-time = "2026-04-27T14:43:19.45Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/c1/464019a9fb059870bfe4eebb4ba12208f3042035e258bf5e782906bd3847/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:120468fb4c24faf0543c835a4fabafa4deb3f20a035c9b6e83d0b553a97615d4", size = 9976181, upload-time = "2026-04-27T14:43:21.463Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/79/94/3ac1432bda31626071e9b6a12709b97ae05131c804b94c8f3ac622c5da32/tokenizers-0.23.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e3d8f40ea6268047de7046906326abed5134f27d4e8447b23763afe5808c8a96", size = 10113853, upload-time = "2026-04-27T14:43:23.617Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/dd/631b21433c771b1382535326f0eca80b9c9cee2e64961dd993bc9ac4669e/tokenizers-0.23.1-cp310-abi3-win32.whl", hash = "sha256:93120a930b919416da7cd10a2f606ac9919cc69cacae7980fa2140e277660948", size = 2536263, upload-time = "2026-04-27T14:43:29.888Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/c9/2553f72aaf65a2797d4229e37fa7fbe38ffbf3e32912d31bdd78b3323e59/tokenizers-0.23.1-cp310-abi3-win_amd64.whl", hash = "sha256:e7bfaf995c1bdbbd21d13539decb6650967013759318627d85daeb7881af16b7", size = 2798223, upload-time = "2026-04-27T14:43:28.51Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/2b/2be299bab55fc595e3d38567edb1a87f86e594842968fa9515a07bdcf422/tokenizers-0.23.1-cp310-abi3-win_arm64.whl", hash = "sha256:a26197957d8e4425dfba746315f3c425ea00cfa8367c5fbc4ec73447893dcea9", size = 2664127, upload-time = "2026-04-27T14:43:26.949Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tomli"
|
||||
version = "2.4.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tqdm"
|
||||
version = "4.67.3"
|
||||
|
|
|
|||
Loading…
Reference in New Issue