From 8c0709eec6df66756fb4faa4da7bd664845f2206 Mon Sep 17 00:00:00 2001
From: Vansh-Sharma27 <connect.vansh.sharma@gmail.com>
Date: Sun, 19 Jul 2026 14:47:58 +0000
Subject: [PATCH] fix(embedding): pre-warm embedding client at startup

Addresses coderabbit review: Tokenizer.from_pretrained for the hf: spec
fetches from the Hugging Face Hub and ran synchronously under the
singleton lock on the first embed/batch_embed call, stalling the event
loop and blocking concurrent callers behind the download.

Add EmbeddingClient.warmup() and call it from the API lifespan and the
deriver startup, right after validate_embedding_schema. This moves any
tokenizer download off the request path and fails the process fast on
a bad hf: spec instead of stalling the first request. tiktoken: and
file: paths are unaffected (no network); unset tokenizer still
auto-detects lazily but never hits the network.
---
 src/deriver/__main__.py |  5 +++++
 src/embedding_client.py | 10 ++++++++++
 src/main.py             |  5 +++++
 3 files changed, 20 insertions(+)

diff --git a/src/deriver/__main__.py b/src/deriver/__main__.py
index c56ed6a0..4a904092 100644
--- a/src/deriver/__main__.py
+++ b/src/deriver/__main__.py
@@ -71,6 +71,11 @@ async def run_deriver():
         # gate the API runs in its lifespan. Inside the try block so the
         # telemetry buffer is still flushed if validation raises.
         await validate_embedding_schema(engine)
+        # Eagerly build the embedding client so an hf: tokenizer download
+        # happens here, not on the first reconciler call under the lock.
+        from src.embedding_client import embedding_client
+
+        embedding_client.warmup()
         await main()
     finally:
         # Shutdown telemetry (flush CloudEvents buffer)
diff --git a/src/embedding_client.py b/src/embedding_client.py
index 183f33c7..d7f188a5 100644
--- a/src/embedding_client.py
+++ b/src/embedding_client.py
@@ -799,6 +799,16 @@ class EmbeddingClient:
     def _resolve_runtime_config(self) -> EmbeddingModelConfig:
         return resolve_embedding_model_config(settings.EMBEDDING.MODEL_CONFIG)
 
+    def warmup(self) -> None:
+        """Eagerly construct the underlying client at process startup.
+
+        ``hf:`` tokenizer specs fetch from the Hugging Face Hub; doing that on
+        the first request would stall the event loop under the singleton lock.
+        Calling this from the API and deriver lifespans moves any tokenizer
+        download off the request path and fails the process fast on a bad spec.
+        """
+        self._get_client()
+
     def _get_settings_signature(self) -> tuple[object, ...]:
         runtime_config = self._resolve_runtime_config()
         return (
diff --git a/src/main.py b/src/main.py
index 75eac455..4fa65819 100644
--- a/src/main.py
+++ b/src/main.py
@@ -18,6 +18,7 @@ from src._version import HONCHO_VERSION
 from src.cache.client import close_cache, init_cache
 from src.config import settings
 from src.db import engine, register_db_query_instrumentation, request_context
+from src.embedding_client import embedding_client
 from src.exceptions import HonchoException
 from src.routers import (
     conclusions,
@@ -116,6 +117,10 @@ async def lifespan(_: FastAPI):
     # writing wrong-dim vectors.
     await validate_embedding_schema(engine)
 
+    # Eagerly build the embedding client so an hf: tokenizer download happens
+    # here, not on the first request under the singleton lock.
+    embedding_client.warmup()
+
     try:
         await init_cache()
     except Exception as e: