From db19f64649e43a059acfc07ec532f119352470b6 Mon Sep 17 00:00:00 2001 From: Alpamys Date: Wed, 25 Mar 2026 18:45:34 +0500 Subject: [PATCH] fix: cap max_tokens at 16384 and add trust_remote_code warning in infer - Add min=1, max=16384 bounds to --max-tokens in soup infer (matches serve.py cap, prevents resource exhaustion) - Add visible warning before loading model with trust_remote_code=True --- soup_cli/commands/infer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/soup_cli/commands/infer.py b/soup_cli/commands/infer.py index 62f0fe6..3336943 100644 --- a/soup_cli/commands/infer.py +++ b/soup_cli/commands/infer.py @@ -41,7 +41,9 @@ def infer( max_tokens: int = typer.Option( 256, "--max-tokens", - help="Maximum tokens to generate per response", + min=1, + max=16384, + help="Maximum tokens to generate per response (1-16384)", ), temperature: float = typer.Option( 0.7, @@ -94,6 +96,10 @@ def infer( ) # Load model + console.print( + "[yellow]Warning: loading model with trust_remote_code=True. " + "Only use models you trust.[/]" + ) console.print("[dim]Loading model...[/]") model_obj, tokenizer = _load_model(str(model_path), base, device) console.print("[green]Model loaded.[/]\n")