fix(cron): review follow-ups for the fail-closed cwd-lock timeout

- Share one HERMES_CRON_TIMEOUT parser (_cron_inactivity_seconds) between
  run_job's inactivity monitor and the cwd-lock bound so the two sites
  cannot drift - the bound must stay >= the inactivity limit or waiters
  would fail while a healthy holder runs.
- Bind the timeout once per run_job; the raise reports the value that was
  actually used for the wait.
- Timeout message covers the writer-blocked-by-readers path instead of
  always blaming a workdir job.
- The finally's TERMINAL_CWD restore is now gated on _cwd_lock_acquired:
  a fail-closed timeout raised before the env-set, so restoring there
  replayed a pre-wait snapshot over the ACTIVE holder's live override
  (pre-existing microsecond race from the #80912 snapshot placement).
- Comment accuracy: the bound is measured from the waiter's arrival; a
  late-wedging or pre-agent-hung holder can outlive it.
This commit is contained in:
kshitij 2026-08-07 18:07:31 +05:30 committed by kshitij
parent 30679b876c
commit bf2e193a0a
1 changed files with 43 additions and 35 deletions

View File

@ -556,14 +556,16 @@ _terminal_cwd_lock = _ReadWriteLock()
# Ceiling on how long a cron job waits for the TERMINAL_CWD lock before
# FAILING (fail-closed, #79768). Derived from the cron inactivity limit
# (HERMES_CRON_TIMEOUT, default 600s): a *wedged* lock holder stops touching
# its activity clock, so the inactivity monitor kills it and the lock is
# released within roughly that limit — a waiter that outlasts it plus margin
# is facing a holder even the monitor could not reap, and must not run:
# proceeding without the lock lets the holder's process-global TERMINAL_CWD
# override leak into this job's shell/file/code-exec commands (wrong-directory
# execution — the exact corruption _ReadWriteLock exists to prevent, see
# test_reader_never_observes_writer_override). A *healthy* long-running
# (HERMES_CRON_TIMEOUT, default 600s): a wedged lock holder stops touching
# its activity clock, so the inactivity monitor usually reaps it and the
# lock is released within roughly that limit. The bound is measured from
# the WAITER's arrival, so a holder that wedges late (or hangs in pre-agent
# setup before the monitor arms) can still outlive it — waiters then fail
# loudly rather than run: proceeding without the lock lets the holder's
# process-global TERMINAL_CWD override leak into this job's shell/file/
# code-exec commands (wrong-directory execution — the exact corruption
# _ReadWriteLock exists to prevent, see
# test_reader_never_observes_writer_override). A healthy long-running
# workdir job past the bound also fails its waiters loudly rather than
# corrupting them silently; the failure names the holder pattern so the fix
# (stagger schedules / drop the workdir) is actionable.
@ -571,13 +573,27 @@ _CWD_LOCK_TIMEOUT_FLOOR_SECONDS = 120.0
_CWD_LOCK_TIMEOUT_MARGIN_SECONDS = 60.0
def _cron_inactivity_seconds() -> float:
"""Parse HERMES_CRON_TIMEOUT (seconds). 0 = unlimited; bad input = 600.
Shared by run_job's inactivity monitor (which maps 0 to "no limit") and
the cwd-lock bound below (which keeps the wait bounded regardless) so
the two sites cannot drift apart the lock bound must stay at or above
the inactivity limit or waiters would fail while a healthy holder runs.
"""
raw = os.getenv("HERMES_CRON_TIMEOUT", "").strip()
if not raw:
return 600.0
try:
return float(raw)
except (ValueError, TypeError):
logger.warning("Invalid HERMES_CRON_TIMEOUT=%r; using default 600s", raw)
return 600.0
def _cwd_lock_timeout_seconds() -> float:
"""Bound for the TERMINAL_CWD lock wait: inactivity limit + margin."""
raw = os.getenv("HERMES_CRON_TIMEOUT", "").strip()
try:
inactivity = float(raw) if raw else 600.0
except (ValueError, TypeError):
inactivity = 600.0
inactivity = _cron_inactivity_seconds()
if inactivity <= 0: # 0 = unlimited job runtime; keep the wait bounded.
inactivity = 600.0
return (
@ -3236,12 +3252,13 @@ def run_job(
_prior_terminal_cwd = os.environ.get("TERMINAL_CWD", "_UNSET_")
_holds_cwd_write = _job_workdir is not None
_cwd_lock_timeout = _cwd_lock_timeout_seconds()
_cwd_lock_acquired = True
if _holds_cwd_write:
if not _terminal_cwd_lock.acquire_write(timeout=_cwd_lock_timeout_seconds()):
if not _terminal_cwd_lock.acquire_write(timeout=_cwd_lock_timeout):
_cwd_lock_acquired = False
else:
if not _terminal_cwd_lock.acquire_read(timeout=_cwd_lock_timeout_seconds()):
if not _terminal_cwd_lock.acquire_read(timeout=_cwd_lock_timeout):
_cwd_lock_acquired = False
# Everything after the acquire MUST live inside this try, so the finally
@ -3263,11 +3280,11 @@ def run_job(
raise TimeoutError(
f"Timed out waiting for the TERMINAL_CWD "
f"{'write' if _holds_cwd_write else 'read'} lock after "
f"{_cwd_lock_timeout_seconds():.0f}s — another workdir cron "
f"job has held it for longer than the cron inactivity limit "
f"(wedged, or legitimately long-running). Stagger its "
f"schedule or remove its workdir to unblock this job "
f"(#79768)."
f"{_cwd_lock_timeout:.0f}s — another cron job (a workdir "
f"writer, or long-running readers) has held it for longer "
f"than the cron inactivity limit. If a workdir job is the "
f"holder, stagger its schedule or remove its workdir to "
f"unblock this job (#79768)."
)
# Scope cron approval policy to this job. Keep the token so the finally
# restores the pre-job state instead of pinning an explicit empty value,
@ -3693,18 +3710,7 @@ def run_job(
#
# Uses the agent's built-in activity tracker (updated by
# _touch_activity() on every tool call, API call, and stream delta).
_raw_cron_timeout = os.getenv("HERMES_CRON_TIMEOUT", "").strip()
if _raw_cron_timeout:
try:
_cron_timeout = float(_raw_cron_timeout)
except (ValueError, TypeError):
logger.warning(
"Invalid HERMES_CRON_TIMEOUT=%r; using default 600s",
_raw_cron_timeout,
)
_cron_timeout = 600.0
else:
_cron_timeout = 600.0
_cron_timeout = _cron_inactivity_seconds()
_cron_inactivity_limit = _cron_timeout if _cron_timeout > 0 else None
_POLL_INTERVAL = 5.0
# Keep the one-shot run_claim fresh while the run is alive (#62002):
@ -3923,9 +3929,11 @@ def run_job(
finally:
# Restore TERMINAL_CWD to whatever it was before this job ran. We
# only ever mutate it when the job has a workdir; see the setup block
# at the top of run_job for the serialization guarantee.
if _job_workdir:
# only ever mutate it when the job has a workdir AND actually held
# the write lock — a fail-closed timeout raised before the env-set,
# so restoring there would replay a pre-wait snapshot over the
# ACTIVE holder's live override.
if _job_workdir and _cwd_lock_acquired:
if _prior_terminal_cwd == "_UNSET_":
os.environ.pop("TERMINAL_CWD", None)
else: