diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbaa9b9..3aac016 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,10 +106,10 @@ soup_cli/ registry/ - Model Registry (hashing, store, diff) (v0.26.0) cans/ - Shareable .can artifact format (v0.26.0) data/traces/ - Trace-to-Preference harvester (v0.26.0) - utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload + utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (93 files, 2677 tests) +tests/ - Test suite (94 files, 2801 tests) examples/ - Real-world config examples and datasets ``` @@ -223,6 +223,14 @@ pytest tests/ --cov=soup_cli --cov-report=html | test_data_augment.py | Data augmentation: rephrase/translate/style strategies, CLI, security (v0.25.0) | | test_training_intelligence.py | Forgetting detection + checkpoint intelligence + SQLite (v0.25.0) | | test_autopilot.py | Autopilot: analyzers, decision engine, CLI (v0.25.0) | +| test_registry.py | Model Registry: hashing, store CRUD, artifacts, lineage DAG, diff, CLI, history (v0.26.0) | +| test_eval_gate.py | Eval-Gated Training: config, suite loading, baseline, callback, CLI (v0.26.0) | +| test_trace_to_pref.py | Trace-to-Preference: LangChain/OpenAI/Soup-serve parsers, pair builder, CLI (v0.26.0) | +| test_quant_check.py | Quant-Lobotomy: classify_delta, resolve_model_ref, render formats, CLI (v0.26.0) | +| test_cans.py | Soup Cans: manifest schema, pack/unpack, tar traversal, fork security, CLI (v0.26.0) | +| test_multi_gpu.py | Multi-GPU Mastery: topology, --gpus, accelerate launcher, ZeRO++, FSDP2+compile, pipeline (v0.27.0) | +| test_training_speed.py | Training Speed & Memory: CCE, FP8, grad-ckpt tiers, kernel picker, cross-doc attn, activation offload (v0.28.0) | +| test_hf_integration.py | HF Hub Deep Integration: token/endpoint/repo_id, auto-push callback, model card v2, collections, data push, HF Spaces, private-IP SSRF (v0.29.0) | ## Making Changes diff --git a/README.md b/README.md index 9c17781..73726e1 100644 --- a/README.md +++ b/README.md @@ -40,12 +40,13 @@ soup train Latest highlights only. Full history: [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases). -- **Cut Cross-Entropy (CCE)** — `training.use_cut_ce: true` fuses the LM-head + cross-entropy on large-vocab models. Saves 8-24 GB VRAM on Llama 3.1 (128k vocab) and similar. -- **FP8 training** — `training.quantization_aware: "fp8"` enables float8 matmuls on Hopper+ (H100/H200/B100/B200) via `torchao.float8`. Bool `true` stays on the int8 QAT path. -- **Gradient checkpointing tiers** — `training.gradient_checkpointing: selective | medium | full | auto`. `auto` picks based on detected VRAM (< 24 GB → full, 24-80 GB → medium, > 80 GB → selective). -- **Kernel auto-composition** — `training.kernel_auto_compose: true` enumerates available kernel combos (baseline / Liger / FlashAttn / CCE) and picks the fastest. -- **Cross-document attention masking** — `training.packing_cross_doc_attn_mask: true` with `packing: true` blocks attention from crossing document boundaries in packed sequences. -- **Activation offloading** — `training.activation_offloading: cpu | disk` offloads saved activations to RAM or a scratch file during backward pass for small-VRAM large-batch runs. +- **Auto-push checkpoints** — `soup train --push-as user/my-model` uploads every `save_steps` checkpoint to HuggingFace Hub as a `checkpoint-` branch. Pair with `--hf-resume` to pull the latest and keep going after a crash. +- **HuggingFace Collections** — `soup push --collection user/my-collection-abc123` groups pushed adapters into an existing Collection. +- **Self-hosted Hub** — set `HF_ENDPOINT=https://hf.internal.example.com` and every HF operation in Soup routes there. Localhost HTTP permitted; private/RFC1918 IPs rejected. +- **Publish HF datasets** — `soup data push --input data.jsonl --hf-dataset user/my-data` uploads a local JSONL as an HF dataset repo. +- **Deploy HF Space** — `soup deploy hf-space --model user/my-model --space user/my-space --template gradio-chat` creates a Gradio (or Streamlit) Space wrapping your fine-tuned model in one command. +- **Model card v2** — generated README surfaces the training config (task / base / lr / optimizer) and an optional eval scorecard; markdown-active chars are neutralised for safe HF Hub rendering. +- **License:** Soup is now Apache-2.0 (previously MIT). Downstream redistributors must retain the `NOTICE` file per §4(d). ## Why Soup? @@ -962,8 +963,61 @@ soup push --model ./output --repo your-username/my-model # Make it private soup push --model ./output --repo your-username/my-model --private + +# Group into a Collection +soup push --model ./output --repo your-username/my-model \ + --collection your-username/my-collection-abc123 ``` +## HuggingFace Hub Deep Integration + +Soup treats HF Hub as a first-class artifact backend. One env var, one flag, +no token flags to plumb — all operations respect `huggingface-cli login` +credentials by default. + +```bash +# Self-hosted Hub: set once, every command routes there. +export HF_ENDPOINT=https://hf.internal.example.com + +# Auto-push each save_steps checkpoint to HF as a 'checkpoint-' branch. +soup train -c soup.yaml --push-as your-username/my-model + +# Resume from the latest branch pushed above. +soup train -c soup.yaml --push-as your-username/my-model --hf-resume + +# Upload a local JSONL file as an HF dataset repo. +soup data push --input train.jsonl --hf-dataset your-username/my-dataset + +# Wrap your fine-tuned model in a Gradio chat Space in one command. +soup deploy hf-space \ + --model your-username/my-model \ + --space your-username/my-chat-space \ + --template gradio-chat + +# Or a Streamlit app: +soup deploy hf-space \ + --model your-username/my-model \ + --space your-username/my-chat-space \ + --template streamlit-chat +``` + +**Auto-resume workflow:** if training crashes, the next `soup train ... --push-as +... --hf-resume` call picks up the latest `checkpoint-` branch from your HF +repo and downloads it back to `output_dir`, then resumes — no manual copy / +paste of checkpoint paths. Cwd containment and `local_dir_use_symlinks=False` +prevent filesystem escape from a crafted repo. + +**Auth** follows standard HF conventions: `HF_TOKEN` env var > `HUGGINGFACE_HUB_TOKEN` +> `~/.cache/huggingface/token` (set by `huggingface-cli login`) > `~/.huggingface/token`. +No custom token flags. The deprecated `--token` on `soup push` still works but emits +a warning. + +**Model card v2** is auto-generated on first push: it reads sidecar +`training_config.yaml` / `soup.yaml` to surface `task` / `base` / `lr` / +`optimizer`, and accepts an optional eval scorecard (markdown table). +Markdown-active chars in task names and scores are neutralised for safe +rendering on HF Hub. + ## Merge LoRA Adapter Merge a LoRA adapter with its base model into a standalone model: diff --git a/SECURITY.md b/SECURITY.md index d5fa6cb..81b5360 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,9 +9,9 @@ We provide security updates for the following versions: - **Versions older than 3 minor versions:** No support Example: -- v0.28.0-0.28.x -- Full support (latest) -- v0.27.0-0.27.x -- Bug-fix support only -- v0.26.x and below -- No support +- v0.29.0-0.29.x -- Full support (latest) +- v0.28.0-0.28.x -- Bug-fix support only +- v0.27.x and below -- No support ## Reporting a Vulnerability @@ -137,6 +137,7 @@ No known critical vulnerabilities in current releases. - **v0.26.0 — Soup Cans**: Manifest format version pinned to 1; name alphanumeric+`_-.`; author max 128 chars, no null bytes/newlines; created_at must parse via `datetime.fromisoformat`; description max 4096; DataRef URL HTTPS-only; hf_dataset regex-validated; tar extraction uses `filter="data"` on Python 3.12+, fallback only on `TypeError`/`AttributeError` (not `TarError`); manual symlink/hardlink rejection + `commonpath` check; 100 MB size cap on pack + fork; dunder-key (`__*__`) and null-byte rejection in fork modifications to prevent prototype pollution; inspect/read_config refuse paths outside cwd - **v0.27.0 — Multi-GPU Mastery**: `--gpus` bounds (reject bool, non-digit, zero, negative, values above `MAX_GPU_COUNT=128`); `--gpus auto` on 0-GPU host prints explicit yellow warning (no silent no-op); Rich markup escaped on `--config` path before embedding in the multi-GPU advice Panel; `accelerate launch` argv assembled via `shlex.quote` per element (copy-pasted command safe against crafted paths); `build_accelerate_argv` validates `num_processes >= 1`, `mixed_precision` Literal (`no/fp16/bf16/fp8`), `num_machines` bounded `[1, 256]`; ZeRO++ integer literals (`int(1e9)` not float) so DeepSpeed strict JSON validator accepts; `validate_fsdp2_compile_config` requires FSDP + CUDA + transformers + torch>=2.2/accelerate>=0.27; DeepSpeed-MII stub exits non-zero to prevent silent mis-start; `validate_pipeline_config` enforces `pipeline_stages >= 2` + CUDA + `gpu_count >= stages`; `pipeline_stages` Pydantic bounds `[1, 16]`; `parallelism` Literal `data|pipeline`; NCCL env (`NCCL_P2P_DISABLE`/`NCCL_IB_DISABLE`/`NCCL_NVLS_ENABLE`) applied via `os.environ.setdefault` only — user/launcher overrides are never stomped - **v0.28.0 — Training Speed & Memory**: `quantization_aware: Union[bool, Literal["fp8"]]` rejects arbitrary strings (only `true` / `false` / `"fp8"`); FP8 path requires CUDA + Hopper+ SM capability + transformers backend; `gradient_checkpointing: Union[bool, Literal["selective","medium","full","auto"]]` rejects unknown tier strings and returns only HF-supported keys (no private markers leak into `TrainingArguments.gradient_checkpointing_kwargs`); `activation_offloading` Literal `cpu|disk`, scratch `save_dir` containment-enforced via shared `utils/paths.is_under_cwd` before disk writes, `torch.load(weights_only=True)` prevents arbitrary Python deserialization on reload, TOCTOU closed between `mkstemp` and `torch.save` by holding the fd open, best-effort cleanup on context exit (handles SIGKILL mid-backward); `kernel_picker.pick_best_kernel` raises `ValueError` when all candidates lack a finite `time_ms` (prevents silent promotion of an untimed combo); Cut CE architecture detector matches on last path component only (so `deepseek-ai/...-phi-...` org-prefix does not trigger a Phi patch on a DeepSeek model); `build_cross_doc_mask` numpy-vectorised to avoid O(seq_length²) pure-Python fill at `max_length` bound (1M); `@model_validator` requires `packing=true` when `packing_cross_doc_attn_mask=true` (prevents silent no-op); `SoupConfig._validate_v028_speed_memory_sft_only` rejects `use_cut_ce`/`quantization_aware="fp8"`/`kernel_auto_compose`/`activation_offloading` on non-SFT tasks — prevents legacy int8-QAT wrapper from crashing on the string `"fp8"` and prevents silent no-ops on DPO/GRPO/KTO/etc. (multi-trainer wiring tracked for v0.28.1) +- **v0.29.0 — HF Hub Deep Integration**: `HF_ENDPOINT` SSRF-hardened — scheme allowlist (http/https), null-byte rejection, `0.0.0.0` explicitly rejected, plain-HTTP only permitted for loopback (`localhost`/`127.0.0.1`/`::1`), RFC1918 / link-local / cloud-metadata (169.254.x) IPs rejected via `ipaddress.ip_address`; repo ID regex `[A-Za-z0-9][A-Za-z0-9._-]{0,95}` per component, ≤200 chars total, null-byte / whitespace / `..` / leading-`/` rejection (applied to `push --repo`, `train --push-as`, `data push --hf-dataset`, `deploy hf-space --model/--space`); collection slug `owner/slug-hash` regex-validated, ≤256 chars; HF token resolution single-sourced in `utils/hf.resolve_token` (env > cached login), explicit non-printable tokens rejected, `push --token` flag deprecated with yellow warning; `soup push --model` confined to cwd via `is_under_cwd` (prevents crafted `soup.yaml output:` from uploading system files); auto-push checkpoint `allow_patterns` restricts uploaded files to `*.safetensors`/`*.bin`/`*.pt`/`*.json`/`tokenizer*`/`trainer_state.json`/`training_args.bin`/`README.md` (keeps `.env` and source files out of auto-pushed branches); `prepare_hf_resume` enforces cwd containment and passes `local_dir_use_symlinks=False` (defeats symlink-based FS escape on older `huggingface_hub`); commit messages stripped to first line and capped at 200 chars (prevents multi-line injection into public HF commit history); `_render_eval_scorecard` neutralises `|`/`[`/`]`/`(`/`)`/`!`/newlines/tabs/`<`/`>` in task names and non-numeric scores; `data_lineage` HTML-escaped (defeats XSS on HF Hub README viewer); `render_space_template` validates `model_repo` via `validate_repo_id` before substitution into rendered `app.py` (crafted repo id cannot inject Python code); `HFPushCallback` uses sticky `_repo_failed` flag to short-circuit retries after hard failure (no log spam, no wasted API calls); `add_to_collection` prefers HfHubHTTPError 409 detection over string-match for duplicate handling ## Security Scanning diff --git a/pyproject.toml b/pyproject.toml index 4e8d0b5..832694e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.28.0" +version = "0.29.0" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "Apache-2.0" diff --git a/soup_cli/__init__.py b/soup_cli/__init__.py index dfe9924..bf54c58 100644 --- a/soup_cli/__init__.py +++ b/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.28.0" +__version__ = "0.29.0" diff --git a/soup_cli/commands/data.py b/soup_cli/commands/data.py index 36994bd..08a1104 100644 --- a/soup_cli/commands/data.py +++ b/soup_cli/commands/data.py @@ -1691,3 +1691,96 @@ def list_registry(): console.print(table) + +@app.command(name="push") +def push_dataset_cmd( + input_path: str = typer.Option( + ..., + "--input", + "-i", + help="Path to local JSONL dataset file", + ), + hf_dataset: str = typer.Option( + ..., + "--hf-dataset", + help="HuggingFace dataset repo id (e.g. user/my-dataset)", + ), + private: bool = typer.Option( + False, "--private", help="Make the HF dataset repo private", + ), + commit_message: str = typer.Option( + "Upload dataset with Soup CLI", + "--message", + help="Commit message for the dataset upload", + ), +): + """Upload a local JSONL dataset to HuggingFace Hub as a dataset repo.""" + from soup_cli.utils.hf import ( + get_hf_api, + resolve_endpoint, + resolve_token, + validate_repo_id, + ) + from soup_cli.utils.paths import is_under_cwd + + file_path = Path(input_path) + if not file_path.exists(): + console.print(f"[red]Dataset file not found: {file_path}[/]") + raise typer.Exit(1) + if not file_path.is_file(): + console.print(f"[red]Expected a file, got a directory: {file_path}[/]") + raise typer.Exit(1) + if not is_under_cwd(file_path): + console.print( + "[red]Dataset path must stay under the current working directory.[/]" + ) + raise typer.Exit(1) + + try: + validate_repo_id(hf_dataset) + except ValueError as exc: + console.print(f"[red]Invalid --hf-dataset repo id:[/] {exc}") + raise typer.Exit(1) from exc + + token = resolve_token() + if token is None: + console.print( + "[red]No HuggingFace token found.[/]\n" + "Set HF_TOKEN env var or run: [bold]huggingface-cli login[/]" + ) + raise typer.Exit(1) + + try: + endpoint = resolve_endpoint() + except ValueError as exc: + console.print(f"[red]HF_ENDPOINT invalid:[/] {exc}") + raise typer.Exit(1) from exc + + try: + api = get_hf_api(token=token, endpoint=endpoint) + except ImportError as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(1) from exc + + # Sanitise commit message to a single short line — prevents multi-line + # injection into HF commit history. + safe_commit = (commit_message.splitlines()[0][:200] if commit_message else "") + try: + api.create_repo( + repo_id=hf_dataset, repo_type="dataset", + private=private, exist_ok=True, + ) + api.upload_file( + path_or_fileobj=str(file_path), + path_in_repo=file_path.name, + repo_id=hf_dataset, + repo_type="dataset", + commit_message=safe_commit, + ) + except Exception as exc: + console.print(f"[red]Upload failed:[/] {exc}") + raise typer.Exit(1) from exc + + console.print( + f"[green]Uploaded to[/] https://huggingface.co/datasets/{hf_dataset}" + ) diff --git a/soup_cli/commands/deploy.py b/soup_cli/commands/deploy.py index abedbef..6514a87 100644 --- a/soup_cli/commands/deploy.py +++ b/soup_cli/commands/deploy.py @@ -1,4 +1,6 @@ -"""soup deploy — deploy models to inference runtimes (Ollama).""" +"""soup deploy — deploy models to inference runtimes (Ollama, HF Spaces).""" + +from __future__ import annotations from pathlib import Path from typing import List, Optional @@ -13,6 +15,185 @@ console = Console() app = typer.Typer(no_args_is_help=True) +# --------------------------------------------------------------------------- +# HF Space templates (Part F of v0.29.0) +# --------------------------------------------------------------------------- + +_GRADIO_APP_PY = '''"""Soup CLI-generated Gradio Chat Space.""" + +import os +import gradio as gr +from transformers import AutoModelForCausalLM, AutoTokenizer + +MODEL_ID = "{MODEL_REPO}" + +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) +model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map="auto") + + +def respond(message, history): + messages = [] + for user, assistant in history: + messages.append({{"role": "user", "content": user}}) + messages.append({{"role": "assistant", "content": assistant}}) + messages.append({{"role": "user", "content": message}}) + prompt = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, + ) + inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + outputs = model.generate( + **inputs, max_new_tokens=512, do_sample=True, + temperature=0.7, top_p=0.9, + ) + reply = tokenizer.decode( + outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True, + ) + return reply + + +chat = gr.ChatInterface(respond, title="Soup CLI Fine-tuned Chat") +chat.launch() +''' + +_GRADIO_REQS = """gradio>=4.0.0 +transformers>=4.40.0 +torch>=2.1.0 +accelerate>=0.27.0 +""" + +_GRADIO_README = """--- +title: Soup Chat +emoji: 🍲 +colorFrom: purple +colorTo: cyan +sdk: gradio +sdk_version: 4.0.0 +app_file: app.py +pinned: false +--- + +# Soup Chat + +Generated with [Soup CLI](https://github.com/MakazhanAlpamys/Soup) — model: `{MODEL_REPO}` +""" + + +_STREAMLIT_APP_PY = '''"""Soup CLI-generated Streamlit Chat Space.""" + +import streamlit as st +from transformers import AutoModelForCausalLM, AutoTokenizer + +MODEL_ID = "{MODEL_REPO}" + +st.set_page_config(page_title="Soup Chat", page_icon="🍲") +st.title("Soup Chat") + + +@st.cache_resource +def load_model(): + tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map="auto") + return tokenizer, model + + +tokenizer, model = load_model() + +if "messages" not in st.session_state: + st.session_state["messages"] = [] + +for msg in st.session_state["messages"]: + with st.chat_message(msg["role"]): + st.markdown(msg["content"]) + +prompt = st.chat_input("Ask me anything") +if prompt: + st.session_state["messages"].append({{"role": "user", "content": prompt}}) + with st.chat_message("user"): + st.markdown(prompt) + with st.chat_message("assistant"): + messages = [ + {{"role": m["role"], "content": m["content"]}} + for m in st.session_state["messages"] + ] + chat_prompt = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, + ) + inputs = tokenizer(chat_prompt, return_tensors="pt").to(model.device) + outputs = model.generate( + **inputs, max_new_tokens=512, do_sample=True, + temperature=0.7, top_p=0.9, + ) + reply = tokenizer.decode( + outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True, + ) + st.markdown(reply) + st.session_state["messages"].append( + {{"role": "assistant", "content": reply}} + ) +''' + +_STREAMLIT_REQS = """streamlit>=1.30.0 +transformers>=4.40.0 +torch>=2.1.0 +accelerate>=0.27.0 +""" + +_STREAMLIT_README = """--- +title: Soup Chat +emoji: 🍲 +colorFrom: purple +colorTo: cyan +sdk: streamlit +sdk_version: 1.30.0 +app_file: app.py +pinned: false +--- + +# Soup Chat + +Generated with [Soup CLI](https://github.com/MakazhanAlpamys/Soup) — model: `{MODEL_REPO}` +""" + + +HF_SPACE_TEMPLATES = { + "gradio-chat": { + "sdk": "gradio", + "app.py": _GRADIO_APP_PY, + "requirements.txt": _GRADIO_REQS, + "README.md": _GRADIO_README, + }, + "streamlit-chat": { + "sdk": "streamlit", + "app.py": _STREAMLIT_APP_PY, + "requirements.txt": _STREAMLIT_REQS, + "README.md": _STREAMLIT_README, + }, +} + + +def render_space_template(template: str, model_repo: str) -> dict[str, str]: + """Render the Space template files with ``model_repo`` substituted. + + Raises ``ValueError`` when the template is unknown or the model repo id + fails validation — injection-proof since repo ids are already a + restrictive alphanumeric subset after validation. + """ + from soup_cli.utils.hf import validate_repo_id + + validate_repo_id(model_repo) + if template not in HF_SPACE_TEMPLATES: + raise ValueError( + f"Unknown template: {template!r}. " + f"Available: {', '.join(HF_SPACE_TEMPLATES.keys())}" + ) + spec = HF_SPACE_TEMPLATES[template] + return { + "app.py": spec["app.py"].replace("{MODEL_REPO}", model_repo), + "requirements.txt": spec["requirements.txt"], + "README.md": spec["README.md"].replace("{MODEL_REPO}", model_repo), + } + + @app.command() def ollama( model: Optional[str] = typer.Option( @@ -234,6 +415,129 @@ def ollama( ) +@app.command(name="hf-space") +def hf_space( + model: str = typer.Option( + ..., + "--model", + "-m", + help="HuggingFace model repo id to wrap in the Space (e.g. user/my-model)", + ), + space: str = typer.Option( + ..., + "--space", + "-s", + help="HuggingFace Space repo id to create (e.g. user/my-space)", + ), + template: str = typer.Option( + "gradio-chat", + "--template", + "-t", + help=f"Space template: {', '.join(HF_SPACE_TEMPLATES.keys())}", + ), + private: bool = typer.Option( + False, "--private", help="Create the Space as private", + ), + yes: bool = typer.Option( + False, "--yes", "-y", help="Skip confirmation", + ), +): + """Create a HuggingFace Space wrapping a fine-tuned model. + + Uploads an app.py, requirements.txt, and README.md rendered from the + chosen template. Supports gradio-chat and streamlit-chat. + """ + from soup_cli.utils.hf import ( + get_hf_api, + resolve_endpoint, + resolve_token, + validate_repo_id, + ) + + # --- Validate space repo id up-front; model is validated by + # render_space_template which is the authoritative entry point. --- + try: + validate_repo_id(space) + except ValueError as exc: + console.print(f"[red]Invalid --space repo id:[/] {exc}") + raise typer.Exit(1) from exc + if template not in HF_SPACE_TEMPLATES: + console.print( + f"[red]Unknown template: {template}[/]\n" + f"Available: {', '.join(HF_SPACE_TEMPLATES.keys())}" + ) + raise typer.Exit(1) + + # --- Resolve credentials --- + token = resolve_token() + if token is None: + console.print( + "[red]No HuggingFace token found.[/]\n" + "Set HF_TOKEN env var or run: [bold]huggingface-cli login[/]" + ) + raise typer.Exit(1) + + try: + endpoint = resolve_endpoint() + except ValueError as exc: + console.print(f"[red]HF_ENDPOINT invalid:[/] {exc}") + raise typer.Exit(1) from exc + + # --- Render template files --- + try: + files = render_space_template(template, model_repo=model) + except ValueError as exc: + console.print(f"[red]Template render failed:[/] {exc}") + raise typer.Exit(1) from exc + + console.print( + Panel( + f"Space: [bold]{space}[/]\n" + f"Model: [bold]{model}[/]\n" + f"Template: [bold]{template}[/]\n" + f"Private: [bold]{private}[/]", + title="Deploy HuggingFace Space", + ) + ) + if not yes: + confirm = typer.confirm("Create Space and upload files?", default=True) + if not confirm: + raise typer.Exit(0) + + # --- Create repo + upload --- + try: + api = get_hf_api(token=token, endpoint=endpoint) + except ImportError as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(1) from exc + + try: + api.create_repo( + repo_id=space, repo_type="space", + space_sdk=HF_SPACE_TEMPLATES[template]["sdk"], + private=private, exist_ok=True, + ) + for in_repo_name, content in files.items(): + api.upload_file( + path_or_fileobj=content.encode("utf-8"), + path_in_repo=in_repo_name, + repo_id=space, + repo_type="space", + commit_message=f"Soup CLI: add {in_repo_name}", + ) + except Exception as exc: + console.print(f"[red]Space deploy failed:[/] {exc}") + raise typer.Exit(1) from exc + + console.print( + Panel( + f"Space: [bold blue]https://huggingface.co/spaces/{space}[/]\n" + f"Model: [bold]{model}[/]", + title="[bold green]Space Deployed![/]", + ) + ) + + def _auto_detect_template() -> Optional[str]: """Try to infer chat template from soup.yaml in cwd.""" from soup_cli.utils.ollama import infer_chat_template diff --git a/soup_cli/commands/push.py b/soup_cli/commands/push.py index 7dbd1a9..2c5edc5 100644 --- a/soup_cli/commands/push.py +++ b/soup_cli/commands/push.py @@ -1,7 +1,10 @@ """soup push — upload a trained model to HuggingFace Hub.""" +from __future__ import annotations + +import html import json -import os +import re from pathlib import Path from typing import Optional @@ -46,8 +49,18 @@ def push( "--message", help="Commit message for the upload", ), + collection: Optional[str] = typer.Option( + None, + "--collection", + help=( + "Add the pushed repo to an existing HF Collection " + "(slug: 'owner/title-hash')" + ), + ), ): """Push a trained model to HuggingFace Hub.""" + from soup_cli.utils.paths import is_under_cwd + model_path = Path(model) # --- Validate model directory --- @@ -59,6 +72,23 @@ def push( console.print(f"[red]Expected a directory, got a file: {model_path}[/]") raise typer.Exit(1) + if not is_under_cwd(model_path): + console.print( + "[red]--model path must stay under the current working directory.[/]" + ) + raise typer.Exit(1) + + # Deprecated --token flag: warn once if explicitly provided. + if token is not None: + console.print( + "[yellow]Warning: --token is deprecated. Use HF_TOKEN env var or " + "run 'huggingface-cli login'.[/]" + ) + + # Sanitise commit message: strip to first line, cap length so a crafted + # multi-line message can't pollute HF commit history. + commit_message = commit_message.splitlines()[0][:200] if commit_message else "" + files_in_dir = {f.name for f in model_path.iterdir() if f.is_file()} is_adapter = ADAPTER_FILES.issubset(files_in_dir) or ADAPTER_FILES_ALT.issubset(files_in_dir) @@ -69,11 +99,16 @@ def push( ) raise typer.Exit(1) - # --- Resolve HF token --- - hf_token = token or os.environ.get("HF_TOKEN") - if not hf_token: - hf_token = _get_cached_token() + # --- Resolve HF token (env > cached login, see utils.hf.resolve_token) --- + from soup_cli.utils.hf import resolve_endpoint, resolve_token, validate_repo_id + try: + validate_repo_id(repo) + except ValueError as exc: + console.print(f"[red]Invalid --repo:[/] {exc}") + raise typer.Exit(1) from exc + + hf_token = resolve_token(explicit=token) if not hf_token: console.print( "[red]No HuggingFace token found.[/]\n" @@ -84,6 +119,12 @@ def push( ) raise typer.Exit(1) + try: + hf_endpoint = resolve_endpoint() + except ValueError as exc: + console.print(f"[red]HF_ENDPOINT invalid:[/] {exc}") + raise typer.Exit(1) from exc + # --- Show upload plan --- file_count = sum(1 for _ in model_path.rglob("*") if _.is_file()) total_size = sum(f.stat().st_size for f in model_path.rglob("*") if f.is_file()) @@ -104,11 +145,18 @@ def push( # --- Upload --- console.print("[dim]Uploading to HuggingFace Hub...[/]") + from soup_cli.utils.hf import get_hf_api + try: - from huggingface_hub import HfApi - - api = HfApi(token=hf_token) + api = get_hf_api(token=hf_token, endpoint=hf_endpoint) + except ImportError as exc: + console.print( + "[red]huggingface-hub not installed.[/]\n" + "Run: [bold]pip install huggingface-hub[/]" + ) + raise typer.Exit(1) from exc + try: # Create repo if it doesn't exist api.create_repo(repo_id=repo, private=private, exist_ok=True) @@ -119,26 +167,55 @@ def push( commit_message=commit_message, ) - # Generate and upload model card if not present + # Generate and upload model card if not present (v2 — includes + # training config and optional eval scorecard) readme_path = model_path / "README.md" if not readme_path.exists(): - model_card = _generate_model_card(model_path, repo, is_adapter) + model_card = generate_model_card_v2( + model_path, repo_id=repo, is_adapter=is_adapter, + ) api.upload_file( path_or_fileobj=model_card.encode("utf-8"), path_in_repo="README.md", repo_id=repo, commit_message="Add model card (generated by Soup CLI)", ) - - except ImportError: - console.print( - "[red]huggingface-hub not installed.[/]\n" - "Run: [bold]pip install huggingface-hub[/]" - ) - raise typer.Exit(1) except Exception as exc: console.print(f"[red]Upload failed: {exc}[/]") - raise typer.Exit(1) + raise typer.Exit(1) from exc + + # --- Optional: add to Collection --- + if collection: + from soup_cli.utils.hf import ( + add_to_collection, + validate_collection_slug, + ) + from soup_cli.utils.hf import ( + resolve_endpoint as _resolve_endpoint, + ) + + try: + validate_collection_slug(collection) + except ValueError as exc: + console.print(f"[red]Invalid --collection slug:[/] {exc}") + raise typer.Exit(1) from exc + + try: + endpoint = _resolve_endpoint() + except ValueError as exc: + console.print(f"[red]Collection: {exc}[/]") + raise typer.Exit(1) from exc + try: + add_to_collection( + collection_slug=collection, + repo_id=repo, + token=hf_token, + endpoint=endpoint, + item_type="model", + ) + console.print(f"[green]Added to collection:[/] {collection}") + except Exception as exc: + console.print(f"[yellow]Could not add to collection:[/] {exc}") repo_url = f"https://huggingface.co/{repo}" console.print( @@ -152,20 +229,6 @@ def push( ) -def _get_cached_token() -> Optional[str]: - """Try to read HF token from cached login.""" - token_path = Path.home() / ".huggingface" / "token" - if token_path.exists(): - return token_path.read_text().strip() - - # New location used by huggingface_hub - token_path_new = Path.home() / ".cache" / "huggingface" / "token" - if token_path_new.exists(): - return token_path_new.read_text().strip() - - return None - - def _format_size(size_bytes: int) -> str: """Format bytes into human-readable string.""" for unit in ("B", "KB", "MB", "GB"): @@ -176,60 +239,207 @@ def _format_size(size_bytes: int) -> str: def _generate_model_card(model_path: Path, repo_id: str, is_adapter: bool) -> str: - """Generate a basic model card README.""" - adapter_info = "" + """Generate a basic model card README (legacy, kept for backward compat).""" + return generate_model_card_v2(model_path, repo_id=repo_id, is_adapter=is_adapter) + + +def _load_adapter_config(model_path: Path) -> dict: + """Read ``adapter_config.json`` if present, return {} on any error.""" config_path = model_path / "adapter_config.json" - if is_adapter and config_path.exists(): + if not config_path.exists(): + return {} + try: + with open(config_path, encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + return data + except (json.JSONDecodeError, OSError): + pass + return {} + + +def _load_training_config(model_path: Path) -> dict: + """Read sidecar ``training_config.yaml`` written by Soup training runs.""" + for name in ("training_config.yaml", "soup.yaml"): + path = model_path / name + if not path.exists(): + continue try: - with open(config_path, encoding="utf-8") as f: - config = json.load(f) - base = config.get("base_model_name_or_path", "unknown") - lora_r = config.get("r", "?") - lora_alpha = config.get("lora_alpha", "?") - adapter_info = ( - f"- **Base model:** `{base}`\n" - f"- **LoRA rank:** {lora_r}\n" - f"- **LoRA alpha:** {lora_alpha}\n" - ) - except (json.JSONDecodeError, OSError): - pass + import yaml + except ImportError: + return {} + try: + with open(path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) + if isinstance(data, dict): + return data + except (yaml.YAMLError, OSError): + continue + return {} + + +_UNSAFE_MD_CHARS = re.compile(r"[\|\[\]\(\)!\n\r\t<>]") + + +def _safe_md_cell(value: str) -> str: + """Neutralise Markdown-active chars so ``value`` cannot inject table rows, + links, images, or raw HTML when rendered on HF Hub.""" + return _UNSAFE_MD_CHARS.sub(" ", str(value)).strip() + + +def _render_eval_scorecard(eval_scorecard: Optional[dict]) -> str: + if not eval_scorecard or not isinstance(eval_scorecard, dict): + return "" + lines = ["## Evaluation", "", "| Task | Score |", "| --- | --- |"] + for task, score in eval_scorecard.items(): + try: + numeric = float(score) + formatted = f"{numeric:.3f}" + except (TypeError, ValueError): + formatted = _safe_md_cell(score) + safe_task = _safe_md_cell(task) or "task" + lines.append(f"| {safe_task} | {formatted} |") + lines.append("") + return "\n".join(lines) + + +def _render_training_section(training_cfg: dict) -> str: + if not training_cfg: + return "" + task = training_cfg.get("task") or "sft" + training = training_cfg.get("training", {}) or {} + base = training_cfg.get("base") or "" + lines = ["## Training", "", f"- **Task:** {task}"] + if base: + lines.append(f"- **Base model:** `{base}`") + for key in ("epochs", "lr", "batch_size", "optimizer", "scheduler"): + if key in training: + lines.append(f"- **{key}:** {training[key]}") + recipe = training_cfg.get("recipe") + if recipe: + lines.append(f"- **Recipe:** `{recipe}`") + lines.append("") + return "\n".join(lines) + + +def generate_model_card_v2( + model_path: Path, + repo_id: str, + is_adapter: Optional[bool] = None, + eval_scorecard: Optional[dict] = None, + data_lineage: Optional[str] = None, +) -> str: + """Model card v2 — enriched with eval scorecard, training config, lineage. + + This is the generator invoked by both ``soup push`` (manual upload) and + the auto-push callback. When the training run wrote a sidecar + ``training_config.yaml`` next to the adapter, we surface task / base / + learning rate / optimizer in the card. When the caller passes a + ``eval_scorecard`` dict, it is rendered as a markdown table. + """ + adapter_config = _load_adapter_config(model_path) + detected_adapter = bool(adapter_config) or (model_path / "adapter_config.json").exists() + if is_adapter is None: + is_adapter = detected_adapter + + adapter_info = "" + if is_adapter and adapter_config: + base = adapter_config.get("base_model_name_or_path", "unknown") + lora_r = adapter_config.get("r", "?") + lora_alpha = adapter_config.get("lora_alpha", "?") + adapter_info = ( + f"- **Base model:** `{base}`\n" + f"- **LoRA rank:** {lora_r}\n" + f"- **LoRA alpha:** {lora_alpha}\n" + ) + + training_cfg = _load_training_config(model_path) + training_section = _render_training_section(training_cfg) + eval_section = _render_eval_scorecard(eval_scorecard) + lineage_section = "" + if data_lineage: + # HTML-escape to block script / javascript: / img-onerror injection + # on the HF Hub README viewer. Markdown chars remain visible but + # inert. + lineage_section = ( + f"## Data Lineage\n\n{html.escape(str(data_lineage))}\n" + ) model_name = repo_id.split("/")[-1] if "/" in repo_id else repo_id + tags_block = "\n".join( + [ + "tags:", + " - soup-cli", + " - fine-tuned", + " - lora" if is_adapter else " - full-model", + ] + ) + library = "peft" if is_adapter else "transformers" - return f"""--- -tags: - - soup-cli - - fine-tuned - - lora -library_name: peft ---- + if adapter_info: + details_block = adapter_info + else: + details_block = "This is a fine-tuned language model." -# {model_name} + usage_block = ( + "```python\n" + "from peft import PeftModel\n" + "from transformers import AutoModelForCausalLM, AutoTokenizer\n\n" + 'model = AutoModelForCausalLM.from_pretrained("BASE_MODEL")\n' + f'model = PeftModel.from_pretrained(model, "{repo_id}")\n' + f'tokenizer = AutoTokenizer.from_pretrained("{repo_id}")\n' + "```\n" + if is_adapter + else ( + "```python\n" + "from transformers import AutoModelForCausalLM, AutoTokenizer\n\n" + f'model = AutoModelForCausalLM.from_pretrained("{repo_id}")\n' + f'tokenizer = AutoTokenizer.from_pretrained("{repo_id}")\n' + "```\n" + ) + ) -Fine-tuned model uploaded with [Soup CLI](https://github.com/MakazhanAlpamys/Soup). - -## Model Details - -{adapter_info if adapter_info else "This is a fine-tuned language model."} - -## Usage - -```python -from peft import PeftModel -from transformers import AutoModelForCausalLM, AutoTokenizer - -model = AutoModelForCausalLM.from_pretrained("BASE_MODEL") -model = PeftModel.from_pretrained(model, "{repo_id}") -tokenizer = AutoTokenizer.from_pretrained("{repo_id}") -``` - -Or with Soup CLI: - -```bash -soup chat --model {repo_id} -``` - -## Training - -Trained using [Soup CLI](https://github.com/MakazhanAlpamys/Soup) — fine-tune LLMs in one command. -""" + sections = [ + "---", + tags_block, + f"library_name: {library}", + "---", + "", + f"# {model_name}", + "", + "Fine-tuned model uploaded with [Soup CLI](https://github.com/MakazhanAlpamys/Soup).", + "", + "## Model Details", + "", + details_block, + ] + if training_section: + sections.append(training_section) + if eval_section: + sections.append(eval_section) + if lineage_section: + sections.append(lineage_section) + tail = [ + "## Usage", + "", + usage_block, + "Or with Soup CLI:", + "", + "```bash", + f"soup chat --model {repo_id}", + "```", + "", + ] + if not training_section: + tail.extend( + [ + "## Training", + "", + "Trained using [Soup CLI]" + "(https://github.com/MakazhanAlpamys/Soup) " + "— fine-tune LLMs in one command.", + "", + ] + ) + sections.extend(tail) + return "\n".join(sections) diff --git a/soup_cli/commands/train.py b/soup_cli/commands/train.py index c2e0669..5c629d0 100644 --- a/soup_cli/commands/train.py +++ b/soup_cli/commands/train.py @@ -1,5 +1,7 @@ """soup train — the main training command.""" +from __future__ import annotations + import os from pathlib import Path @@ -77,6 +79,22 @@ def train( "(shortcut for training.eval_gate.enabled=true + suite=)" ), ), + push_as: str = typer.Option( + None, + "--push-as", + help=( + "Auto-push each save_steps checkpoint to HF Hub as " + "'checkpoint-' branch of the given repo (e.g. user/my-model)" + ), + ), + hf_resume: bool = typer.Option( + False, + "--hf-resume", + help=( + "Download the latest checkpoint branch from the --push-as repo " + "and resume from it. Requires --push-as." + ), + ), yes: bool = typer.Option( False, "--yes", @@ -95,6 +113,19 @@ def train( console.print(f"[dim]Loading config from {config_path}...[/]") cfg = load_config(config_path) + # --- --push-as / --hf-resume validation --- + if push_as: + from soup_cli.utils.hf import validate_repo_id + + try: + validate_repo_id(push_as) + except ValueError as exc: + console.print(f"[red]Invalid --push-as repo id:[/] {exc}") + raise typer.Exit(1) from exc + if hf_resume and not push_as: + console.print("[red]--hf-resume requires --push-as [/]") + raise typer.Exit(1) + # --- Eval-gate shortcut: --gate sets training.eval_gate --- if gate: from soup_cli.config.schema import EvalGateConfig @@ -119,6 +150,36 @@ def train( console.print("[red]No checkpoint found to resume from.[/]") raise typer.Exit(1) + # --- HF auto-resume: pull latest checkpoint branch into output dir --- + if hf_resume and push_as and resume_from is None: + from soup_cli.monitoring.hf_push import prepare_hf_resume + from soup_cli.utils.hf import resolve_endpoint, resolve_token + + try: + hf_endpoint = resolve_endpoint() + except ValueError as exc: + console.print(f"[red]--hf-resume: {exc}[/]") + raise typer.Exit(1) from exc + hf_token = resolve_token() + if hf_token is None: + console.print( + "[yellow]--hf-resume: no HF token available; skipping auto-resume[/]" + ) + else: + local_ckpt = prepare_hf_resume( + repo_id=push_as, + output_dir=cfg.output, + token=hf_token, + endpoint=hf_endpoint, + ) + if local_ckpt: + resume_from = local_ckpt + console.print(f"[green]Resumed from HF:[/] {local_ckpt}") + else: + console.print( + "[yellow]--hf-resume: no checkpoint branch found; starting fresh[/]" + ) + # --- Validate logging flags --- if wandb and tensorboard: console.print( @@ -500,6 +561,33 @@ def train( trainer_wrapper = SFTTrainerWrapper(cfg, **trainer_kwargs) trainer_wrapper.setup(dataset) + # --- HF auto-push callback (Part B of v0.29.0) --- + if push_as: + from soup_cli.monitoring.hf_push import build_push_callback + + push_cb = build_push_callback( + repo_id=push_as, + output_dir=cfg.output, + private=False, + ) + if push_cb is None: + console.print( + "[yellow]--push-as: no HF token available; skipping auto-push[/]" + ) + else: + hf_trainer = getattr(trainer_wrapper, "trainer", None) + if hf_trainer is not None and hasattr(hf_trainer, "add_callback"): + hf_trainer.add_callback(push_cb) + console.print( + f"[green]HF auto-push enabled[/] -> {push_as} " + "(one branch per save_steps)" + ) + else: + console.print( + "[yellow]--push-as: trainer does not expose add_callback; " + "auto-push disabled for this run[/]" + ) + # Train with live display and experiment tracking display = TrainingDisplay(cfg, device_name=device_name) console.print("[bold green]Training started![/]\n") diff --git a/soup_cli/monitoring/hf_push.py b/soup_cli/monitoring/hf_push.py new file mode 100644 index 0000000..862e885 --- /dev/null +++ b/soup_cli/monitoring/hf_push.py @@ -0,0 +1,292 @@ +"""HuggingFace auto-push callback and resume helpers (v0.29.0 Part B). + +Hooks into the HF Trainer's ``on_save`` event so every checkpoint saved to +disk is pushed to the Hub as a ``checkpoint-`` branch. Also provides +``prepare_hf_resume`` which downloads the latest checkpoint branch back to +the local ``output_dir`` so a fresh run can pick up where the previous +crashed. + +Network errors are logged and swallowed — we never crash training because +the Hub is unreachable. +""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path +from typing import Optional + +from soup_cli.utils.hf import get_hf_api, resolve_endpoint, resolve_token, validate_repo_id + +logger = logging.getLogger(__name__) + +_CHECKPOINT_BRANCH_RE = re.compile(r"^checkpoint-(\d+)$") + +# Files worth shipping in an auto-pushed checkpoint. Keeps stray .env / +# source files / caches out of auto-pushed revisions if ``output_dir`` +# is ever misconfigured to overlap with the project root. +_CHECKPOINT_ALLOW_PATTERNS = [ + "*.safetensors", + "*.bin", + "*.pt", + "*.json", + "tokenizer*", + "special_tokens_map.json", + "generation_config.json", + "trainer_state.json", + "training_args.bin", + "README.md", +] + + +class HFPushCallback: + """TrainerCallback-shaped auto-pusher. + + We don't inherit from :class:`transformers.TrainerCallback` here to keep + the module importable without transformers (tests mock it anyway). The + ``train`` command attaches an instance via ``trainer.add_callback`` so + duck-typing is sufficient. + """ + + def __init__( + self, + repo_id: str, + token: Optional[str] = None, + endpoint: Optional[str] = None, + output_dir: str = "", + private: bool = False, + ) -> None: + validate_repo_id(repo_id) + self.repo_id = repo_id + self.token = token + self.endpoint = endpoint + # ``output_dir`` is a fallback used only when HF Trainer's + # ``TrainingArguments.output_dir`` is missing (e.g. tests that + # construct a bare SimpleNamespace). Under real training the value + # comes from ``args.output_dir``. + self.output_dir = output_dir + self.private = private + self._repo_created = False + self._repo_failed = False # short-circuits retries after hard failure + + # --- TrainerCallback protocol --- + + def on_train_begin(self, args, state, control, **kwargs) -> None: + # Eagerly create the repo so the first checkpoint upload is not + # delayed. Swallow errors so we don't crash training; the failure + # is retried once on the first on_save, then short-circuited. + self._ensure_repo() + + def on_save(self, args, state, control, **kwargs) -> None: + """Upload the checkpoint directory written at ``global_step``.""" + step = int(getattr(state, "global_step", 0) or 0) + if step <= 0: + return + + out_dir = getattr(args, "output_dir", None) or self.output_dir + if not out_dir: + return + + ckpt_path = Path(out_dir) / f"checkpoint-{step}" + if not ckpt_path.is_dir(): + logger.debug("HFPushCallback: checkpoint dir missing: %s", ckpt_path) + return + + self._upload_checkpoint(ckpt_path, step) + + # --- Helpers --- + + def _ensure_repo(self) -> bool: + """Create the repo if needed. Returns True if ready for uploads.""" + if self._repo_created: + return True + if self._repo_failed: + return False + try: + api = get_hf_api(token=self.token, endpoint=self.endpoint) + api.create_repo(repo_id=self.repo_id, private=self.private, exist_ok=True) + self._repo_created = True + return True + except Exception as exc: + logger.warning( + "HFPushCallback: create_repo failed (%s); auto-push disabled", exc, + ) + self._repo_failed = True + return False + + def _upload_checkpoint(self, ckpt_path: Path, step: int) -> None: + if not self._ensure_repo(): + return + try: + api = get_hf_api(token=self.token, endpoint=self.endpoint) + + revision = f"checkpoint-{step}" + try: + api.create_branch( + repo_id=self.repo_id, + branch=revision, + exist_ok=True, + ) + except Exception as exc: + # create_branch is best-effort — older hub versions lack it. + logger.debug("create_branch failed (continuing): %s", exc) + + api.upload_folder( + folder_path=str(ckpt_path), + repo_id=self.repo_id, + revision=revision, + commit_message=f"Soup auto-push checkpoint-{step}", + allow_patterns=_CHECKPOINT_ALLOW_PATTERNS, + ) + except Exception as exc: + logger.warning("HFPushCallback: upload failed at step %d: %s", step, exc) + + +def resolve_latest_checkpoint_revision( + repo_id: str, + token: Optional[str] = None, + endpoint: Optional[str] = None, +) -> Optional[str]: + """Return the ``checkpoint-`` revision with the largest ``N``, or None. + + Swallows API errors — a missing repo or network failure returns None so + callers can fall back to training from scratch. + """ + try: + api = get_hf_api(token=token, endpoint=endpoint) + refs = api.list_repo_refs(repo_id=repo_id) + except Exception as exc: + logger.debug("list_repo_refs failed (%s); no resume revision", exc) + return None + + branches = getattr(refs, "branches", None) or [] + best_step = -1 + best_name: Optional[str] = None + for branch in branches: + name = getattr(branch, "name", None) + if not isinstance(name, str): + continue + match = _CHECKPOINT_BRANCH_RE.match(name) + if not match: + continue + step = int(match.group(1)) + if step > best_step: + best_step = step + best_name = name + return best_name + + +def _download_checkpoint( + repo_id: str, + revision: str, + local_dir: str, + token: Optional[str], + endpoint: Optional[str], +) -> str: + """Download a revision into ``local_dir``. + + ``local_dir_use_symlinks=False`` forces direct copies — defence against + older ``huggingface_hub`` versions that could symlink the shared cache + into ``local_dir`` and thus let a crafted repo (or future SDK bug) place + symlinks pointing at arbitrary filesystem locations. + """ + try: + from huggingface_hub import snapshot_download + except ImportError as exc: + raise ImportError( + "huggingface_hub is required for --hf-resume. Install huggingface-hub." + ) from exc + + Path(local_dir).mkdir(parents=True, exist_ok=True) + try: + resolved = snapshot_download( + repo_id=repo_id, + revision=revision, + local_dir=local_dir, + token=token, + endpoint=endpoint, + local_dir_use_symlinks=False, + ) + except TypeError: + # Older huggingface_hub versions reject local_dir_use_symlinks. + resolved = snapshot_download( + repo_id=repo_id, + revision=revision, + local_dir=local_dir, + token=token, + endpoint=endpoint, + ) + return resolved or local_dir + + +def prepare_hf_resume( + repo_id: str, + output_dir: str, + token: Optional[str] = None, + endpoint: Optional[str] = None, +) -> Optional[str]: + """Pull the latest checkpoint branch from HF into ``output_dir``. + + Returns the local path of the checkpoint directory, or None if there's + nothing to resume from. + + The ``output_dir`` must stay under the current working directory — an + attacker-controlled ``cfg.output`` (e.g. ``../../../tmp``) would + otherwise place downloaded files outside the project tree. + """ + from soup_cli.utils.paths import is_under_cwd + + validate_repo_id(repo_id) + if not is_under_cwd(output_dir): + raise ValueError( + "output_dir must stay under the current working directory " + f"for --hf-resume; got: {output_dir!r}" + ) + revision = resolve_latest_checkpoint_revision(repo_id, token=token, endpoint=endpoint) + if revision is None: + return None + + # Mirror HF Trainer's on-disk layout: output_dir/ + local_dir = str(Path(output_dir) / revision) + try: + return _download_checkpoint( + repo_id=repo_id, + revision=revision, + local_dir=local_dir, + token=token, + endpoint=endpoint, + ) + except Exception as exc: + logger.warning("HF resume download failed (%s); skipping auto-resume", exc) + return None + + +def build_push_callback( + repo_id: str, + output_dir: str, + explicit_token: Optional[str] = None, + private: bool = False, +) -> Optional[HFPushCallback]: + """Factory that resolves token/endpoint and builds the callback. + + Returns None when no HF token is available — the caller logs and skips + auto-push silently. + """ + token = resolve_token(explicit=explicit_token) + if token is None: + return None + + try: + endpoint = resolve_endpoint() + except ValueError as exc: + logger.warning("HF_ENDPOINT invalid (%s); skipping auto-push", exc) + return None + + return HFPushCallback( + repo_id=repo_id, + token=token, + endpoint=endpoint, + output_dir=output_dir, + private=private, + ) diff --git a/soup_cli/utils/hf.py b/soup_cli/utils/hf.py new file mode 100644 index 0000000..82ff0ae --- /dev/null +++ b/soup_cli/utils/hf.py @@ -0,0 +1,232 @@ +"""HuggingFace Hub integration utilities (v0.29.0). + +Single source of truth for HF token/endpoint resolution, repo ID validation, +and an ``HfApi`` factory used by push, data push, deploy hf-space, and +auto-push callbacks. + +Design goals: +- **No custom token flags**: respect ``HF_TOKEN`` / ``HUGGINGFACE_HUB_TOKEN`` + env vars and the cached login from ``huggingface-cli login``. +- **Self-hosted Hub**: ``HF_ENDPOINT`` env var overrides the public endpoint. +- **SSRF hardening**: only HTTPS remotes allowed; localhost HTTP permitted + for self-hosted dev rigs. +- **Lazy import**: ``huggingface_hub`` is optional and imported inside + ``get_hf_api``. +""" + +from __future__ import annotations + +import ipaddress +import os +import re +from pathlib import Path +from typing import Any, Optional +from urllib.parse import urlparse + +DEFAULT_ENDPOINT = "https://huggingface.co" + +# Loopback hosts that may legitimately use plain HTTP (dev / self-hosted). +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) + +# repo IDs are either "name" or "owner/name"; both parts must be safe. +_REPO_PART_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") + +# Collection slugs use "owner/title-" with longer hashes allowed. +_COLLECTION_SLUG_RE = re.compile(r"^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$") + + +def resolve_token(explicit: Optional[str] = None) -> Optional[str]: + """Return the HF token following the documented precedence. + + Order: + 1. ``explicit`` argument (for tests / programmatic callers). + 2. ``HF_TOKEN`` env var. + 3. ``HUGGINGFACE_HUB_TOKEN`` env var. + 4. Cached login at ``~/.cache/huggingface/token`` (new location). + 5. Legacy cached login at ``~/.huggingface/token``. + 6. ``None`` if no token is available. + """ + if explicit: + stripped = explicit.strip() + if not stripped: + # Whitespace-only — fall through to env/cache lookup rather + # than return an empty string as a token. + pass + elif not stripped.isprintable(): + raise ValueError("explicit token contains non-printable characters") + else: + return stripped + + env_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN") + if env_token: + stripped = env_token.strip() + if stripped: + return stripped + + home = Path.home() + for candidate in ( + home / ".cache" / "huggingface" / "token", + home / ".huggingface" / "token", + ): + if candidate.is_file(): + try: + value = candidate.read_text(encoding="utf-8").strip() + except OSError: + continue + if value: + return value + return None + + +def resolve_endpoint() -> str: + """Return the HF endpoint, validating ``HF_ENDPOINT`` when set. + + Strips one trailing slash, rejects non-HTTPS remotes (localhost HTTP is + allowed for self-hosted dev setups), and rejects null bytes / bogus + schemes. + """ + raw = os.environ.get("HF_ENDPOINT") + if not raw: + return DEFAULT_ENDPOINT + + if "\x00" in raw: + raise ValueError("HF_ENDPOINT must not contain null bytes") + + stripped = raw.rstrip("/") + parsed = urlparse(stripped) + if parsed.scheme not in ("http", "https"): + raise ValueError(f"HF_ENDPOINT must use http/https scheme, got: {parsed.scheme}") + if not parsed.netloc: + raise ValueError("HF_ENDPOINT is missing a host") + + host = parsed.hostname or "" + if host == "0.0.0.0": + raise ValueError( + "HF_ENDPOINT 0.0.0.0 is ambiguous; use 127.0.0.1 or localhost" + ) + if parsed.scheme == "http" and host not in _LOOPBACK_HOSTS: + # Reject plain HTTP for RFC1918 / link-local / cloud metadata too — + # HF_ENDPOINT=http://169.254.169.254 or http://192.168.1.1 would + # otherwise route SDK traffic to internal targets. + if _is_private_or_link_local(host): + raise ValueError( + "HF_ENDPOINT plain HTTP is only allowed for loopback " + "(localhost / 127.0.0.1 / ::1); private/link-local hosts " + "require HTTPS" + ) + raise ValueError( + "HF_ENDPOINT for remote hosts must use HTTPS (localhost HTTP allowed)" + ) + return stripped + + +def _is_private_or_link_local(host: str) -> bool: + """Whether ``host`` resolves to a private / link-local / loopback IP.""" + try: + addr = ipaddress.ip_address(host) + except ValueError: + # Hostname — we don't resolve DNS here (the SDK does), so fall back + # to "treat as public". A malicious DNS record pointing to a private + # IP is out of scope for this local-tool threat model. + return False + return addr.is_private or addr.is_link_local or addr.is_loopback + + +def validate_repo_id(repo_id: str) -> None: + """Validate a HuggingFace repo ID (raises ``ValueError`` on bad input). + + Accepts ``owner/name`` or ``name``. Each component must start with an + alphanumeric character and contain only ``[A-Za-z0-9._-]``, ≤96 chars. + """ + if not isinstance(repo_id, str) or not repo_id: + raise ValueError("repo_id must be a non-empty string") + if "\x00" in repo_id: + raise ValueError("repo_id must not contain null bytes") + if len(repo_id) > 200: + raise ValueError("repo_id too long (max 200 chars)") + if any(ch.isspace() for ch in repo_id): + raise ValueError("repo_id must not contain whitespace") + if ".." in repo_id or repo_id.startswith("/") or repo_id.endswith("/"): + raise ValueError(f"repo_id contains invalid path segments: {repo_id!r}") + + parts = repo_id.split("/") + if len(parts) > 2: + raise ValueError(f"repo_id must be 'owner/name' or 'name', got: {repo_id!r}") + for part in parts: + if not _REPO_PART_RE.match(part): + raise ValueError(f"repo_id component invalid: {part!r}") + + +def validate_collection_slug(slug: str) -> None: + """Validate a HuggingFace collection slug (``owner/slug-hash``).""" + if not isinstance(slug, str) or not slug: + raise ValueError("collection slug must be a non-empty string") + if "\x00" in slug or any(ch.isspace() for ch in slug): + raise ValueError("collection slug must not contain whitespace or null bytes") + if ".." in slug or slug.startswith("/") or slug.endswith("/"): + raise ValueError(f"collection slug has invalid segments: {slug!r}") + if len(slug) > 256: + raise ValueError("collection slug too long (max 256 chars)") + if not _COLLECTION_SLUG_RE.match(slug): + raise ValueError(f"collection slug must be 'owner/slug-hash', got: {slug!r}") + + +def get_hf_api( + token: Optional[str] = None, + endpoint: Optional[str] = None, +) -> Any: + """Return a configured ``HfApi`` instance. + + Lazy-imports ``huggingface_hub`` so the rest of the CLI works without + the optional dependency. Raises ``ImportError`` with install hint when + missing. + """ + try: + from huggingface_hub import HfApi + except ImportError as exc: + raise ImportError( + "huggingface_hub is required for HF Hub operations. " + "Install with: pip install huggingface-hub" + ) from exc + + return HfApi(token=token, endpoint=endpoint) + + +def add_to_collection( + collection_slug: str, + repo_id: str, + token: Optional[str] = None, + endpoint: Optional[str] = None, + item_type: str = "model", + ignore_duplicate: bool = True, +) -> None: + """Add a repo to an existing HF Collection. + + Validates inputs and swallows "already exists" errors when + ``ignore_duplicate`` is set (default) so repeated pushes are idempotent. + """ + validate_collection_slug(collection_slug) + validate_repo_id(repo_id) + if item_type not in ("model", "dataset", "space"): + raise ValueError(f"item_type must be model|dataset|space, got: {item_type!r}") + + api = get_hf_api(token=token, endpoint=endpoint) + try: + api.add_collection_item( + collection_slug=collection_slug, + item_id=repo_id, + item_type=item_type, + ) + except Exception as exc: + if not ignore_duplicate: + raise + # Prefer HTTP status (409 = Conflict) when huggingface_hub raises a + # typed HfHubHTTPError; fall back to a string-match on the exception + # message for older hub versions that lack a typed response. + status = getattr(getattr(exc, "response", None), "status_code", None) + if status == 409: + return + message = str(exc).lower() + if "already" in message or "exists" in message: + return + raise diff --git a/tests/test_hf_integration.py b/tests/test_hf_integration.py new file mode 100644 index 0000000..fb03d73 --- /dev/null +++ b/tests/test_hf_integration.py @@ -0,0 +1,1208 @@ +"""Tests for HF Hub Deep Integration (v0.29.0).""" + +from __future__ import annotations + +import json +import sys +import types +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from soup_cli.cli import app + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# Part A: utils/hf.py — token / endpoint / repo_id resolution +# --------------------------------------------------------------------------- + + +class TestResolveToken: + def test_env_hf_token_wins(self, monkeypatch, tmp_path): + monkeypatch.setenv("HF_TOKEN", "env-token") + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from soup_cli.utils.hf import resolve_token + + assert resolve_token() == "env-token" + + def test_cached_token_new_location(self, monkeypatch, tmp_path): + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + token_file = tmp_path / ".cache" / "huggingface" / "token" + token_file.parent.mkdir(parents=True) + token_file.write_text("cached-token\n") + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from soup_cli.utils.hf import resolve_token + + assert resolve_token() == "cached-token" + + def test_cached_token_legacy_location(self, monkeypatch, tmp_path): + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + token_file = tmp_path / ".huggingface" / "token" + token_file.parent.mkdir(parents=True) + token_file.write_text("legacy-token") + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from soup_cli.utils.hf import resolve_token + + assert resolve_token() == "legacy-token" + + def test_explicit_token_overrides_env(self, monkeypatch): + monkeypatch.setenv("HF_TOKEN", "env-token") + from soup_cli.utils.hf import resolve_token + + assert resolve_token(explicit="explicit-token") == "explicit-token" + + def test_huggingface_hub_token_env(self, monkeypatch, tmp_path): + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.setenv("HUGGINGFACE_HUB_TOKEN", "hh-token") + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from soup_cli.utils.hf import resolve_token + + assert resolve_token() == "hh-token" + + def test_no_token_returns_none(self, monkeypatch, tmp_path): + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from soup_cli.utils.hf import resolve_token + + assert resolve_token() is None + + +class TestResolveEndpoint: + def test_default_endpoint(self, monkeypatch): + monkeypatch.delenv("HF_ENDPOINT", raising=False) + from soup_cli.utils.hf import resolve_endpoint + + assert resolve_endpoint() == "https://huggingface.co" + + def test_custom_endpoint_env(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf.internal.example.com") + from soup_cli.utils.hf import resolve_endpoint + + assert resolve_endpoint() == "https://hf.internal.example.com" + + def test_endpoint_strips_trailing_slash(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf.internal.example.com/") + from soup_cli.utils.hf import resolve_endpoint + + assert resolve_endpoint() == "https://hf.internal.example.com" + + def test_endpoint_rejects_non_https_remote(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "http://evil.example.com") + from soup_cli.utils.hf import resolve_endpoint + + with pytest.raises(ValueError, match="HTTPS"): + resolve_endpoint() + + def test_endpoint_allows_localhost_http(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "http://localhost:8080") + from soup_cli.utils.hf import resolve_endpoint + + assert resolve_endpoint() == "http://localhost:8080" + + def test_endpoint_rejects_null_byte(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "https://hf.example.com\x00") + from soup_cli.utils.hf import resolve_endpoint + + with pytest.raises(ValueError): + resolve_endpoint() + + def test_endpoint_rejects_bad_scheme(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "file:///etc/passwd") + from soup_cli.utils.hf import resolve_endpoint + + with pytest.raises(ValueError): + resolve_endpoint() + + +class TestValidateRepoId: + def test_valid_user_repo(self): + from soup_cli.utils.hf import validate_repo_id + + validate_repo_id("user/my-model") + + def test_valid_repo_only(self): + from soup_cli.utils.hf import validate_repo_id + + validate_repo_id("my-model") + + def test_rejects_empty(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("") + + def test_rejects_slash_at_start(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("/my-model") + + def test_rejects_double_slash(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("user//model") + + def test_rejects_path_traversal(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("../../secrets") + + def test_rejects_null_byte(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("user/model\x00") + + def test_rejects_whitespace(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("user/my model") + + def test_rejects_too_long(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("user/" + ("a" * 200)) + + def test_accepts_dots_underscores_hyphens(self): + from soup_cli.utils.hf import validate_repo_id + + validate_repo_id("user/my_model.v2-beta") + + +class TestGetHfApi: + def test_requires_huggingface_hub(self, monkeypatch): + from soup_cli.utils.hf import get_hf_api + + monkeypatch.setitem(sys.modules, "huggingface_hub", None) + with pytest.raises(ImportError): + get_hf_api() + + def test_passes_token_and_endpoint(self, monkeypatch): + from soup_cli.utils import hf + + fake_hub = types.ModuleType("huggingface_hub") + + calls = {} + + class FakeApi: + def __init__(self, token=None, endpoint=None): + calls["token"] = token + calls["endpoint"] = endpoint + + fake_hub.HfApi = FakeApi + monkeypatch.setitem(sys.modules, "huggingface_hub", fake_hub) + + hf.get_hf_api(token="t1", endpoint="https://e.example.com") + assert calls == {"token": "t1", "endpoint": "https://e.example.com"} + + +# --------------------------------------------------------------------------- +# Part B: Auto-push checkpoints (--push-as flag) +# --------------------------------------------------------------------------- + + +class TestPushAsCLIFlag: + def test_train_shows_push_as_flag_in_help(self): + result = runner.invoke(app, ["train", "--help"]) + out = result.output + assert "--push-as" in out + + def test_push_as_rejects_invalid_repo(self, tmp_path, monkeypatch): + cfg = tmp_path / "soup.yaml" + cfg.write_text("base: meta-llama/Llama-3.1-8B\ntask: sft\ndata:\n train: data.jsonl\n") + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, + [ + "train", + "--config", + str(cfg), + "--push-as", + "../../escape", + "--dry-run", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "push-as" in result.output.lower() + + +class TestHFPushCallback: + def test_callback_import(self): + from soup_cli.monitoring.hf_push import HFPushCallback # noqa: F401 + + def test_callback_uploads_checkpoint_on_save(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + ckpt_dir = tmp_path / "checkpoint-100" + ckpt_dir.mkdir() + (ckpt_dir / "adapter_config.json").write_text("{}") + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api) + + callback = HFPushCallback( + repo_id="user/my-model", + token="t1", + output_dir=str(tmp_path), + ) + + args = types.SimpleNamespace(output_dir=str(tmp_path)) + state = types.SimpleNamespace(global_step=100, epoch=1) + control = types.SimpleNamespace() + callback.on_save(args, state, control) + + fake_api.create_repo.assert_called_once_with( + repo_id="user/my-model", private=False, exist_ok=True, + ) + assert fake_api.upload_folder.called + kwargs = fake_api.upload_folder.call_args.kwargs + assert kwargs["repo_id"] == "user/my-model" + assert kwargs["folder_path"] == str(ckpt_dir) + assert kwargs["revision"] == "checkpoint-100" + assert "checkpoint-100" in kwargs["commit_message"] + + def test_callback_swallows_upload_errors(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + ckpt_dir = tmp_path / "checkpoint-50" + ckpt_dir.mkdir() + (ckpt_dir / "adapter_config.json").write_text("{}") + + fake_api = MagicMock() + fake_api.upload_folder.side_effect = RuntimeError("network") + monkeypatch.setattr("soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api) + + callback = HFPushCallback( + repo_id="user/my-model", + token="t1", + output_dir=str(tmp_path), + ) + + args = types.SimpleNamespace(output_dir=str(tmp_path)) + state = types.SimpleNamespace(global_step=50, epoch=1) + control = types.SimpleNamespace() + callback.on_save(args, state, control) + + def test_callback_skips_missing_checkpoint(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api) + + callback = HFPushCallback( + repo_id="user/my-model", + token="t1", + output_dir=str(tmp_path), + ) + + args = types.SimpleNamespace(output_dir=str(tmp_path)) + state = types.SimpleNamespace(global_step=200, epoch=2) + control = types.SimpleNamespace() + callback.on_save(args, state, control) + + assert not fake_api.upload_folder.called + + +class TestResolveLatestRevision: + def test_returns_latest_step(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import resolve_latest_checkpoint_revision + + fake_api = MagicMock() + fake_api.list_repo_refs.return_value = types.SimpleNamespace( + branches=[ + types.SimpleNamespace(name="main"), + types.SimpleNamespace(name="checkpoint-50"), + types.SimpleNamespace(name="checkpoint-150"), + types.SimpleNamespace(name="checkpoint-100"), + ] + ) + monkeypatch.setattr( + "soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api + ) + result = resolve_latest_checkpoint_revision("user/my-model", token="t1") + assert result == "checkpoint-150" + + def test_returns_none_when_no_checkpoints(self, monkeypatch): + from soup_cli.monitoring.hf_push import resolve_latest_checkpoint_revision + + fake_api = MagicMock() + fake_api.list_repo_refs.return_value = types.SimpleNamespace( + branches=[types.SimpleNamespace(name="main")] + ) + monkeypatch.setattr( + "soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api + ) + result = resolve_latest_checkpoint_revision("user/my-model", token="t1") + assert result is None + + def test_returns_none_on_api_error(self, monkeypatch): + from soup_cli.monitoring.hf_push import resolve_latest_checkpoint_revision + + fake_api = MagicMock() + fake_api.list_repo_refs.side_effect = RuntimeError("no such repo") + monkeypatch.setattr( + "soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api + ) + result = resolve_latest_checkpoint_revision("user/my-model", token="t1") + assert result is None + + +# --------------------------------------------------------------------------- +# Part C: Model Card v2 +# --------------------------------------------------------------------------- + + +class TestModelCardV2: + def test_generate_enhanced_card_basic(self, tmp_path): + from soup_cli.commands.push import generate_model_card_v2 + + adapter_dir = tmp_path / "adapter" + adapter_dir.mkdir() + (adapter_dir / "adapter_config.json").write_text( + json.dumps( + { + "base_model_name_or_path": "meta-llama/Llama-3.1-8B", + "r": 32, + "lora_alpha": 64, + } + ) + ) + card = generate_model_card_v2(adapter_dir, repo_id="user/my-model") + assert "my-model" in card + assert "meta-llama/Llama-3.1-8B" in card + assert "soup-cli" in card + assert "LoRA" in card + + def test_generate_full_model_card(self, tmp_path): + from soup_cli.commands.push import generate_model_card_v2 + + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "config.json").write_text("{}") + card = generate_model_card_v2(model_dir, repo_id="user/full-model") + assert "full-model" in card + assert "soup-cli" in card + + def test_card_includes_training_config_when_present(self, tmp_path): + from soup_cli.commands.push import generate_model_card_v2 + + adapter_dir = tmp_path / "adapter" + adapter_dir.mkdir() + (adapter_dir / "adapter_config.json").write_text("{}") + (adapter_dir / "training_config.yaml").write_text( + "base: meta-llama/Llama-3.1-8B\ntask: sft\ntraining:\n epochs: 3\n lr: 0.0002\n" + ) + card = generate_model_card_v2(adapter_dir, repo_id="user/my-model") + assert "sft" in card or "Training" in card + assert "0.0002" in card or "lr" in card + + def test_card_includes_eval_scorecard_when_registry_available(self, tmp_path): + from soup_cli.commands.push import generate_model_card_v2 + + adapter_dir = tmp_path / "adapter" + adapter_dir.mkdir() + (adapter_dir / "adapter_config.json").write_text("{}") + + eval_data = { + "mmlu": 0.612, + "gsm8k": 0.812, + } + card = generate_model_card_v2( + adapter_dir, + repo_id="user/my-model", + eval_scorecard=eval_data, + ) + assert "mmlu" in card.lower() + assert "0.612" in card or "61.2" in card + + def test_card_safe_against_malformed_config(self, tmp_path): + from soup_cli.commands.push import generate_model_card_v2 + + adapter_dir = tmp_path / "adapter" + adapter_dir.mkdir() + (adapter_dir / "adapter_config.json").write_text("not json") + card = generate_model_card_v2(adapter_dir, repo_id="user/broken") + assert "broken" in card + + +# --------------------------------------------------------------------------- +# Part D: HF Collections +# --------------------------------------------------------------------------- + + +class TestCollections: + def test_push_shows_collection_flag_in_help(self): + result = runner.invoke(app, ["push", "--help"]) + assert "--collection" in result.output + + def test_add_to_collection_calls_api(self, monkeypatch): + from soup_cli.utils.hf import add_to_collection + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + add_to_collection( + collection_slug="user/collection-abc123", + repo_id="user/my-model", + token="t1", + ) + assert fake_api.add_collection_item.called + kwargs = fake_api.add_collection_item.call_args.kwargs + assert kwargs["collection_slug"] == "user/collection-abc123" + assert kwargs["item_id"] == "user/my-model" + assert kwargs["item_type"] == "model" + + def test_add_to_collection_rejects_invalid_slug(self, monkeypatch): + from soup_cli.utils.hf import add_to_collection + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + with pytest.raises(ValueError): + add_to_collection( + collection_slug="../../escape", + repo_id="user/my-model", + token="t1", + ) + + def test_add_to_collection_survives_duplicate(self, monkeypatch): + from soup_cli.utils.hf import add_to_collection + + fake_api = MagicMock() + fake_api.add_collection_item.side_effect = RuntimeError("already exists") + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + add_to_collection( + collection_slug="user/collection-abc123", + repo_id="user/my-model", + token="t1", + ignore_duplicate=True, + ) + + +# --------------------------------------------------------------------------- +# Part E: HF Datasets write +# --------------------------------------------------------------------------- + + +class TestDataPush: + def test_push_subcommand_exists(self): + result = runner.invoke(app, ["data", "push", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "--hf-dataset" in result.output + + def test_push_rejects_missing_file(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, + [ + "data", + "push", + "--input", + "nonexistent.jsonl", + "--hf-dataset", + "user/my-dataset", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "not found" in result.output.lower() or "does not exist" in result.output.lower() + + def test_push_rejects_invalid_dataset_name(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "data.jsonl").write_text('{"text":"a"}\n') + result = runner.invoke( + app, + [ + "data", + "push", + "--input", + "data.jsonl", + "--hf-dataset", + "../../escape", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "hf-dataset" in result.output.lower() or "repo" in result.output.lower() + + def test_push_rejects_path_outside_cwd(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + outside = tmp_path.parent / "outside.jsonl" + outside.write_text('{"text":"a"}\n') + try: + result = runner.invoke( + app, + [ + "data", + "push", + "--input", + str(outside), + "--hf-dataset", + "user/my-dataset", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "current working directory" in result.output.lower() + finally: + outside.unlink(missing_ok=True) + + def test_push_requires_token(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path / "no-creds") + (tmp_path / "data.jsonl").write_text('{"text":"a"}\n') + result = runner.invoke( + app, + [ + "data", + "push", + "--input", + "data.jsonl", + "--hf-dataset", + "user/my-dataset", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "token" in result.output.lower() + + def test_push_dataset_happy_path(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_TOKEN", "t1") + (tmp_path / "data.jsonl").write_text('{"text":"a"}\n{"text":"b"}\n') + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + + result = runner.invoke( + app, + [ + "data", + "push", + "--input", + "data.jsonl", + "--hf-dataset", + "user/my-dataset", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert fake_api.create_repo.called + assert fake_api.upload_file.called or fake_api.upload_folder.called + + +# --------------------------------------------------------------------------- +# Part F: HF Spaces auto-deploy +# --------------------------------------------------------------------------- + + +class TestDeployHfSpace: + def test_hf_space_command_registered(self): + result = runner.invoke(app, ["deploy", "--help"]) + assert "hf-space" in result.output + + def test_hf_space_help_shows_flags(self): + result = runner.invoke(app, ["deploy", "hf-space", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "--model" in result.output + assert "--template" in result.output + + def test_hf_space_rejects_invalid_repo(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_TOKEN", "t1") + result = runner.invoke( + app, + [ + "deploy", + "hf-space", + "--model", + "user/my-model", + "--space", + "../../escape", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "space" in result.output.lower() + + def test_hf_space_rejects_unknown_template(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_TOKEN", "t1") + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + result = runner.invoke( + app, + [ + "deploy", + "hf-space", + "--model", + "user/my-model", + "--space", + "user/my-space", + "--template", + "rocket-launcher", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "template" in result.output.lower() + + def test_hf_space_requires_token(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path / "no-creds") + result = runner.invoke( + app, + [ + "deploy", + "hf-space", + "--model", + "user/my-model", + "--space", + "user/my-space", + "--template", + "gradio-chat", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "token" in result.output.lower() + + def test_hf_space_happy_path_gradio_chat(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_TOKEN", "t1") + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + + result = runner.invoke( + app, + [ + "deploy", + "hf-space", + "--model", + "user/my-model", + "--space", + "user/my-space", + "--template", + "gradio-chat", + "--yes", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert fake_api.create_repo.called + + def test_render_gradio_chat_template(self): + from soup_cli.commands.deploy import render_space_template + + rendered = render_space_template("gradio-chat", model_repo="user/my-model") + assert "gradio" in rendered["app.py"].lower() + assert "user/my-model" in rendered["app.py"] + assert "requirements.txt" in rendered + assert "README.md" in rendered + + def test_render_streamlit_chat_template(self): + from soup_cli.commands.deploy import render_space_template + + rendered = render_space_template("streamlit-chat", model_repo="user/my-model") + assert "streamlit" in rendered["app.py"].lower() + assert "user/my-model" in rendered["app.py"] + + def test_render_unknown_template_raises(self): + from soup_cli.commands.deploy import render_space_template + + with pytest.raises(ValueError): + render_space_template("bogus", model_repo="user/my-model") + + def test_readme_sets_sdk_to_match_template(self): + from soup_cli.commands.deploy import render_space_template + + gradio = render_space_template("gradio-chat", model_repo="user/my-model") + assert "sdk: gradio" in gradio["README.md"] + streamlit = render_space_template("streamlit-chat", model_repo="user/my-model") + assert "sdk: streamlit" in streamlit["README.md"] + + def test_template_escapes_model_repo_id(self): + from soup_cli.commands.deploy import render_space_template + + with pytest.raises(ValueError): + render_space_template("gradio-chat", model_repo="../../escape") + + +# --------------------------------------------------------------------------- +# Auto-resume from HF (Part B) +# --------------------------------------------------------------------------- + + +class TestAutoResume: + def test_train_help_shows_hf_resume(self): + result = runner.invoke(app, ["train", "--help"]) + out = result.output + assert "--hf-resume" in out or "hf-resume" in out.lower().replace("-", "") + + def test_prepare_hf_resume_downloads_latest(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import prepare_hf_resume + + monkeypatch.chdir(tmp_path) + out_dir = tmp_path / "runs" / "experiment" + out_dir.mkdir(parents=True) + + called = {} + + def fake_download(repo_id, revision, local_dir, token, endpoint): + called["repo_id"] = repo_id + called["revision"] = revision + called["local_dir"] = local_dir + return local_dir + + monkeypatch.setattr( + "soup_cli.monitoring.hf_push.resolve_latest_checkpoint_revision", + lambda repo_id, token=None, endpoint=None: "checkpoint-300", + ) + monkeypatch.setattr( + "soup_cli.monitoring.hf_push._download_checkpoint", fake_download + ) + + result = prepare_hf_resume( + repo_id="user/my-model", + output_dir=str(out_dir), + token="t1", + ) + assert result is not None + assert called["revision"] == "checkpoint-300" + + def test_prepare_hf_resume_no_checkpoint_returns_none(self, monkeypatch, tmp_path): + from soup_cli.monitoring.hf_push import prepare_hf_resume + + monkeypatch.chdir(tmp_path) + out_dir = tmp_path / "runs" + out_dir.mkdir() + monkeypatch.setattr( + "soup_cli.monitoring.hf_push.resolve_latest_checkpoint_revision", + lambda repo_id, token=None, endpoint=None: None, + ) + result = prepare_hf_resume( + repo_id="user/my-model", + output_dir=str(out_dir), + token="t1", + ) + assert result is None + + def test_prepare_hf_resume_rejects_output_dir_outside_cwd( + self, monkeypatch, tmp_path + ): + import pytest + + from soup_cli.monitoring.hf_push import prepare_hf_resume + + cwd = tmp_path / "project" + cwd.mkdir() + monkeypatch.chdir(cwd) + outside = tmp_path / "elsewhere" + outside.mkdir() + + with pytest.raises(ValueError, match="under the current"): + prepare_hf_resume( + repo_id="user/my-model", + output_dir=str(outside), + token="t1", + ) + + +# --------------------------------------------------------------------------- +# Integration: utils/hf consolidated behavior +# --------------------------------------------------------------------------- + + +class TestHfUtilsModule: + def test_module_exports(self): + from soup_cli.utils import hf + + assert hasattr(hf, "resolve_token") + assert hasattr(hf, "resolve_endpoint") + assert hasattr(hf, "validate_repo_id") + assert hasattr(hf, "get_hf_api") + assert hasattr(hf, "add_to_collection") + + def test_token_with_whitespace_stripped(self, monkeypatch, tmp_path): + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + token_file = tmp_path / ".cache" / "huggingface" / "token" + token_file.parent.mkdir(parents=True) + token_file.write_text(" padded-token \n") + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path) + from soup_cli.utils.hf import resolve_token + + assert resolve_token() == "padded-token" + + +# --------------------------------------------------------------------------- +# Extra coverage from TDD review: collection-slug negatives, callback factory, +# on_train_begin, short-circuit after repo failure, private-IP SSRF, edge +# cases around resolve_token. +# --------------------------------------------------------------------------- + + +class TestValidateCollectionSlug: + def test_accepts_valid(self): + from soup_cli.utils.hf import validate_collection_slug + + validate_collection_slug("user/my-collection-abc12345") + + def test_rejects_empty(self): + from soup_cli.utils.hf import validate_collection_slug + + with pytest.raises(ValueError): + validate_collection_slug("") + + def test_rejects_whitespace(self): + from soup_cli.utils.hf import validate_collection_slug + + with pytest.raises(ValueError): + validate_collection_slug("user/my coll") + + def test_rejects_null_byte(self): + from soup_cli.utils.hf import validate_collection_slug + + with pytest.raises(ValueError): + validate_collection_slug("user/my-coll\x00") + + def test_rejects_path_traversal(self): + from soup_cli.utils.hf import validate_collection_slug + + with pytest.raises(ValueError): + validate_collection_slug("../../escape") + + def test_rejects_too_long(self): + from soup_cli.utils.hf import validate_collection_slug + + with pytest.raises(ValueError): + validate_collection_slug("user/" + ("a" * 300)) + + def test_rejects_missing_slash(self): + from soup_cli.utils.hf import validate_collection_slug + + with pytest.raises(ValueError): + validate_collection_slug("onlyname") + + +class TestAddToCollectionExtras: + def test_raises_on_duplicate_when_ignore_false(self, monkeypatch): + from soup_cli.utils.hf import add_to_collection + + fake_api = MagicMock() + fake_api.add_collection_item.side_effect = RuntimeError("already exists") + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + with pytest.raises(RuntimeError): + add_to_collection( + collection_slug="user/my-coll-abc12345", + repo_id="user/my-model", + token="t1", + ignore_duplicate=False, + ) + + def test_raises_on_unrelated_error(self, monkeypatch): + from soup_cli.utils.hf import add_to_collection + + fake_api = MagicMock() + fake_api.add_collection_item.side_effect = RuntimeError("auth required") + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + with pytest.raises(RuntimeError): + add_to_collection( + collection_slug="user/my-coll-abc12345", + repo_id="user/my-model", + token="t1", + ) + + def test_rejects_invalid_item_type(self, monkeypatch): + from soup_cli.utils.hf import add_to_collection + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + with pytest.raises(ValueError, match="item_type"): + add_to_collection( + collection_slug="user/my-coll-abc12345", + repo_id="user/my-model", + token="t1", + item_type="notebook", + ) + + +class TestBuildPushCallback: + def test_returns_none_when_no_token(self, monkeypatch, tmp_path): + from soup_cli.monitoring.hf_push import build_push_callback + + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path / "no-creds") + cb = build_push_callback( + repo_id="user/my-model", output_dir=str(tmp_path) + ) + assert cb is None + + def test_returns_none_on_bad_endpoint(self, monkeypatch, tmp_path): + from soup_cli.monitoring.hf_push import build_push_callback + + monkeypatch.setenv("HF_TOKEN", "t1") + monkeypatch.setenv("HF_ENDPOINT", "http://evil.example.com") + cb = build_push_callback( + repo_id="user/my-model", output_dir=str(tmp_path) + ) + assert cb is None + + def test_happy_path_returns_callback(self, monkeypatch, tmp_path): + from soup_cli.monitoring.hf_push import HFPushCallback, build_push_callback + + monkeypatch.setenv("HF_TOKEN", "t1") + monkeypatch.delenv("HF_ENDPOINT", raising=False) + cb = build_push_callback( + repo_id="user/my-model", output_dir=str(tmp_path) + ) + assert isinstance(cb, HFPushCallback) + assert cb.repo_id == "user/my-model" + + +class TestCallbackLifecycle: + def _fake_api(self, monkeypatch): + fake_api = MagicMock() + monkeypatch.setattr( + "soup_cli.monitoring.hf_push.get_hf_api", lambda **_: fake_api + ) + return fake_api + + def test_on_train_begin_creates_repo(self, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + fake_api = self._fake_api(monkeypatch) + cb = HFPushCallback( + repo_id="user/my-model", token="t1", output_dir="/tmp", + ) + cb.on_train_begin( + args=types.SimpleNamespace(output_dir="/tmp"), + state=types.SimpleNamespace(), + control=types.SimpleNamespace(), + ) + fake_api.create_repo.assert_called_once_with( + repo_id="user/my-model", private=False, exist_ok=True, + ) + + def test_on_train_begin_swallows_failure(self, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + fake_api = self._fake_api(monkeypatch) + fake_api.create_repo.side_effect = RuntimeError("auth") + cb = HFPushCallback( + repo_id="user/my-model", token="t1", output_dir="/tmp", + ) + cb.on_train_begin( + args=types.SimpleNamespace(output_dir="/tmp"), + state=types.SimpleNamespace(), + control=types.SimpleNamespace(), + ) + assert cb._repo_failed is True + assert cb._repo_created is False + + def test_create_repo_called_once_across_saves(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + fake_api = self._fake_api(monkeypatch) + for step in (50, 100, 150): + ckpt = tmp_path / f"checkpoint-{step}" + ckpt.mkdir() + (ckpt / "adapter_config.json").write_text("{}") + + cb = HFPushCallback( + repo_id="user/my-model", token="t1", output_dir=str(tmp_path), + ) + for step in (50, 100, 150): + cb.on_save( + args=types.SimpleNamespace(output_dir=str(tmp_path)), + state=types.SimpleNamespace(global_step=step, epoch=1), + control=types.SimpleNamespace(), + ) + assert fake_api.create_repo.call_count == 1 + assert fake_api.upload_folder.call_count == 3 + + def test_no_upload_after_repo_creation_fails(self, tmp_path, monkeypatch): + from soup_cli.monitoring.hf_push import HFPushCallback + + fake_api = self._fake_api(monkeypatch) + fake_api.create_repo.side_effect = RuntimeError("auth forbidden") + + ckpt = tmp_path / "checkpoint-10" + ckpt.mkdir() + (ckpt / "adapter_config.json").write_text("{}") + + cb = HFPushCallback( + repo_id="user/my-model", token="t1", output_dir=str(tmp_path), + ) + cb.on_train_begin( + args=types.SimpleNamespace(output_dir=str(tmp_path)), + state=types.SimpleNamespace(), + control=types.SimpleNamespace(), + ) + cb.on_save( + args=types.SimpleNamespace(output_dir=str(tmp_path)), + state=types.SimpleNamespace(global_step=10, epoch=1), + control=types.SimpleNamespace(), + ) + assert fake_api.upload_folder.call_count == 0 + + +class TestPrivateIPSSRF: + @pytest.mark.parametrize( + "host", + [ + "http://10.0.0.1", + "http://192.168.1.1", + "http://172.16.0.1", + "http://169.254.169.254", + ], + ) + def test_private_ip_http_rejected(self, monkeypatch, host): + monkeypatch.setenv("HF_ENDPOINT", host) + from soup_cli.utils.hf import resolve_endpoint + + with pytest.raises(ValueError): + resolve_endpoint() + + def test_zero_address_rejected(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "http://0.0.0.0:8080") + from soup_cli.utils.hf import resolve_endpoint + + with pytest.raises(ValueError, match="0.0.0.0"): + resolve_endpoint() + + def test_127_allowed(self, monkeypatch): + monkeypatch.setenv("HF_ENDPOINT", "http://127.0.0.1:8080") + from soup_cli.utils.hf import resolve_endpoint + + assert resolve_endpoint() == "http://127.0.0.1:8080" + + +class TestResolveTokenEdgeCases: + def test_whitespace_env_falls_through(self, monkeypatch, tmp_path): + monkeypatch.setenv("HF_TOKEN", " ") + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path / "no-creds") + from soup_cli.utils.hf import resolve_token + + assert resolve_token() is None + + def test_empty_explicit_falls_through_to_env(self, monkeypatch): + monkeypatch.setenv("HF_TOKEN", "from-env") + from soup_cli.utils.hf import resolve_token + + assert resolve_token(explicit="") == "from-env" + + def test_explicit_non_printable_raises(self): + from soup_cli.utils.hf import resolve_token + + with pytest.raises(ValueError): + resolve_token(explicit="abc\x01\x02") + + +class TestValidateRepoIdBoundaries: + def test_accepts_max_length_component(self): + from soup_cli.utils.hf import validate_repo_id + + validate_repo_id("u/" + ("a" * 96)) + + def test_rejects_oversized_component(self): + from soup_cli.utils.hf import validate_repo_id + + with pytest.raises(ValueError): + validate_repo_id("u/" + ("a" * 97)) + + +class TestEvalScorecardEdgeCases: + def test_non_numeric_score(self): + from soup_cli.commands.push import _render_eval_scorecard + + rendered = _render_eval_scorecard({"gsm8k": "N/A"}) + assert "N/A" in rendered + assert "gsm8k" in rendered + + def test_escapes_pipe_in_task_name(self): + from soup_cli.commands.push import _render_eval_scorecard + + rendered = _render_eval_scorecard({"math|injection": 0.5}) + # The pipe must be neutralised to avoid breaking the markdown table. + for line in rendered.splitlines(): + if "0.500" in line: + # Table row — exactly 3 unescaped pipes: leading, middle, trailing. + assert line.count("|") == 3 + + def test_escapes_markdown_injection(self): + from soup_cli.commands.push import _render_eval_scorecard + + rendered = _render_eval_scorecard({"task\n| fake | 0.99": 0.5}) + assert "fake" in rendered # literal preserved as text + # The injection newline / pipe must be neutralised. + assert "\n| fake" not in rendered + + +class TestHfResumeRequiresPushAs: + def test_hf_resume_alone_exits_one(self, tmp_path, monkeypatch): + cfg = tmp_path / "soup.yaml" + cfg.write_text("base: meta-llama/Llama-3.1-8B\ntask: sft\ndata:\n train: data.jsonl\n") + monkeypatch.chdir(tmp_path) + result = runner.invoke( + app, + ["train", "--config", str(cfg), "--hf-resume", "--dry-run"], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "push-as" in result.output.lower() + + +class TestDataPushHappyPathExtras: + def test_create_repo_called_with_dataset_type(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_TOKEN", "t1") + (tmp_path / "data.jsonl").write_text('{"text":"a"}\n') + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + monkeypatch.setattr( + "soup_cli.commands.data.get_hf_api", lambda **_: fake_api, raising=False + ) + + result = runner.invoke( + app, + [ + "data", "push", "--input", "data.jsonl", + "--hf-dataset", "user/my-dataset", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + fake_api.create_repo.assert_called_once_with( + repo_id="user/my-dataset", repo_type="dataset", + private=False, exist_ok=True, + ) + + +class TestHfSpaceHappyPathExtras: + def test_create_repo_called_with_space_type_and_sdk(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HF_TOKEN", "t1") + + fake_api = MagicMock() + monkeypatch.setattr("soup_cli.utils.hf.get_hf_api", lambda **_: fake_api) + + result = runner.invoke( + app, + [ + "deploy", "hf-space", + "--model", "user/my-model", + "--space", "user/my-space", + "--template", "gradio-chat", + "--yes", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + fake_api.create_repo.assert_called_once_with( + repo_id="user/my-space", repo_type="space", + space_sdk="gradio", private=False, exist_ok=True, + ) diff --git a/tests/test_push.py b/tests/test_push.py index 7f4672a..7fe8a7e 100644 --- a/tests/test_push.py +++ b/tests/test_push.py @@ -113,6 +113,11 @@ def test_push_invalid_model_dir(tmp_path: Path): def test_push_no_token(tmp_path: Path, monkeypatch): """Should fail if no HF token is available.""" monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.delenv("HUGGINGFACE_HUB_TOKEN", raising=False) + # Point HOME at an empty dir so the cached-token fallback also misses. + monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path / "no-creds") + # `soup push` now requires --model under cwd. + monkeypatch.chdir(tmp_path) model_dir = tmp_path / "model" model_dir.mkdir()