diff --git a/cron/scheduler.py b/cron/scheduler.py index 57a89e06c1367..f22cc453d7182 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -140,6 +140,24 @@ def _summarize_cron_failure_for_delivery(job: dict, error: str | None) -> str: text = (error or "unknown error").strip() lower = text.lower() + if "skipped to prevent unintended spend: global inference config drifted" in lower: + if "finite one-shot job is consumed" in lower: + remediation = ( + "This finite one-shot is consumed; create a new one-shot job at " + "a future time with an explicit provider and model." + ) + else: + job_id = job.get("id") or "" + remediation = ( + "Pin it explicitly: " + f"`cronjob action=update job_id={job_id} " + "provider= model=`." + ) + return ( + f"⚠️ Cron '{job_name}' skipped before inference to prevent " + f"unintended spend. {remediation}" + ) + # Provider/API failures are the common noisy path. Keep these short. if "429" in text or "rate limit" in lower or "usage limit" in lower: reason = "rate limit" @@ -4192,14 +4210,36 @@ def run_job( _drift.append(f"{_axis} '{_snapshot}' -> '{_current}'") if _drift: _changes = "; ".join(_drift) + # Lifecycle-aware remediation (#72056, @sashmatash): a finite + # one-shot is consumed by this attempted dispatch — telling the + # operator to `cronjob action=update` a spent job is a dead + # end. Recurring/repeatable jobs get the pin command instead. + _repeat = job.get("repeat") if isinstance(job.get("repeat"), dict) else {} + _finite_oneshot = ( + isinstance(job.get("schedule"), dict) + and job["schedule"].get("kind") == "once" + and _repeat.get("times") == 1 + ) + if _finite_oneshot: + _remediation = ( + "This finite one-shot job is consumed by this attempted run; " + "create a new one-shot job at a future time with an explicit " + "provider and model." + ) + else: + _remediation = ( + "To run on the new config, pin it explicitly: " + f"`cronjob action=update job_id={job_id} " + "provider= model=` (or pin the original " + "values to keep them)." + ) logger.warning( "Job '%s': SKIPPED — global inference config drifted since " "creation (%s) and this job is unpinned. Skipped to prevent " - "unintended spend. Pin explicitly to proceed: " - "`cronjob action=update job_id=%s provider=

model=`.", + "unintended spend. %s", job_id, _changes, - job_id, + _remediation, ) # Alert-once (#73506 shape): persist the drift_alerted bit so # only the FIRST drifted tick delivers; run_one_job suppresses @@ -4220,9 +4260,7 @@ def run_job( f"{_drift_marker} Skipped to prevent unintended spend: global " f"inference config drifted since this job was created " f"({_changes}), and this job is unpinned. No inference call " - f"was made. To run on the new config, pin it explicitly: " - f"`cronjob action=update job_id={job_id} provider= " - f"model=` (or pin the original values to keep them). " + f"was made. {_remediation} " f"This alert is sent once; the job stays skipped until the " f"config is pinned or restored. See #44585." ) diff --git a/tests/cron/test_cron_provider_pin.py b/tests/cron/test_cron_provider_pin.py index 104b60a0d2559..b215418160ae8 100644 --- a/tests/cron/test_cron_provider_pin.py +++ b/tests/cron/test_cron_provider_pin.py @@ -27,7 +27,7 @@ import pytest # Ensure project root is importable. sys.path.insert(0, str(Path(__file__).parent.parent.parent)) -from cron.scheduler import run_job +from cron.scheduler import _summarize_cron_failure_for_delivery, run_job def _base_job(**overrides): @@ -90,8 +90,9 @@ class TestProviderDriftGuard: def test_b_unpinned_snapshot_differs_fails_closed(self, tmp_path): """(b) Unpinned job whose snapshot != current provider → fail closed. - The paid call must NOT be made (AIAgent never constructed) and the - delivered error must name both providers and tell the user to pin. + The paid call must NOT be made (AIAgent never constructed), the raw + error must name both providers, and the delivered summary must tell the + user to pin. """ job = _base_job(provider_snapshot="openrouter") success, output, final_response, error, agent_constructed = \ @@ -110,6 +111,9 @@ class TestProviderDriftGuard: assert "cronjob action=update" in blob assert "44585" in blob + delivered = _summarize_cron_failure_for_delivery(job, error).lower() + assert "cronjob action=update" in delivered + def test_c_no_snapshot_runs_backcompat(self, tmp_path): """(c) Pre-existing job with NO provider_snapshot → runs (back-compat). @@ -283,6 +287,29 @@ class TestModelDriftGuard: assert "44585" in blob + def test_finite_oneshot_model_drift_explains_that_recreation_is_required(self, tmp_path): + """A spent one-shot cannot be repaired in place after the guard fires.""" + job = _base_job( + provider_snapshot="openrouter", + model_snapshot="old-model", + schedule={"kind": "once", "run_at": "2030-01-01T00:00:00Z"}, + repeat={"times": 1, "completed": 1}, + ) + success, _output, _final_response, error, agent_constructed = \ + _run_with_current_provider_and_model( + job, "openrouter", "new-model", tmp_path + ) + + assert success is False + assert agent_constructed is False + assert error is not None + assert "create a new one-shot job" in error.lower() + assert "cronjob action=update" not in error.lower() + + delivered = _summarize_cron_failure_for_delivery(job, error).lower() + assert "create a new one-shot job" in delivered + assert "cronjob action=update" not in delivered + def test_no_model_snapshot_backcompat(self, tmp_path): # Pre-existing job without model_snapshot → no model-drift skip. job = _base_job(provider_snapshot="openrouter") # no model_snapshot key set to a value diff --git a/website/docs/user-guide/features/cron.md b/website/docs/user-guide/features/cron.md index e3011168196bc..b764a50a6885f 100644 --- a/website/docs/user-guide/features/cron.md +++ b/website/docs/user-guide/features/cron.md @@ -22,11 +22,12 @@ Cron jobs can: All of this is available to Hermes itself through the `cronjob` tool, so you can create, pause, edit, and remove jobs by asking in plain language — no CLI required. :::tip +<<<<<<< HEAD **Which model does a cron job run on?** Resolution at fire time is: per-job pin → `cron.model` in `config.yaml` → the global default from `hermes model`. - **Per-job pin** — set by *you* via the dashboard, `hermes cron create/edit --model … --provider …`, or by editing `~/.hermes/cron/jobs.json`. Once set, it sticks until you change it. The agent's `cronjob` tool cannot set or change per-job models — inference pins are user-owned. - **`cron.model` / `cron.model_provider`** — a cron-fleet default: every unpinned job runs on this model, independent of your chat model. Set it once (`hermes config set cron.model `) and switching your chat model with `hermes model` or `/model` never touches your cron fleet. -- **Global default** — only when neither of the above is set does a job follow `hermes model`. In this case Hermes **snapshots** the provider and model at creation, and if the global default later changes the job **fails closed**: it skips the run, makes no inference call, and alerts you to pin the provider/model explicitly (#44585). This prevents an unattended job from silently inheriting a switch to a paid provider/model. Setting `cron.model` (or a per-job pin) is the deliberate way to route cron spend, and the drift guard does not engage for an axis covered by it. Operators who instead want unpinned jobs to track the changing global default can [disable the drift guard](#letting-unpinned-jobs-track-global-defaults). +- **Global default** — only when neither of the above is set does a job follow `hermes model`. In this case Hermes **snapshots** the provider and model at creation, and if the global default later changes the job **fails closed**: it skips the run, makes no inference call, and alerts you **once** — the job stays skipped (and silent) on subsequent ticks until you act or the config is restored (#44585). For recurring or otherwise repeatable jobs, pin the provider/model explicitly (`cronjob action=update job_id=… provider=… model=…`) to proceed. A consumed finite one-shot cannot be updated; create a new future one-shot with an explicit provider and model instead. This prevents an unattended job from silently inheriting a switch to a paid provider/model. Setting `cron.model` (or a per-job pin) is the deliberate way to route cron spend, and the drift guard does not engage for an axis covered by it. Operators who instead want unpinned jobs to track the changing global default can [disable the drift guard](#letting-unpinned-jobs-track-global-defaults). `hermes setup --portal` is the lowest-friction option for unattended runs since OAuth refresh is automatic. See [Nous Portal](/integrations/nous-portal). :::