test(cron): pin fail-closed TERMINAL_CWD lock timeout behavior

- reader/writer run_job timeout paths fail loudly (writer additionally
  proven to never mutate the active holder's TERMINAL_CWD override -
  the fail-open design clobbered it)
- waiter whose holder finishes inside the bound still proceeds
- bound derivation from HERMES_CRON_TIMEOUT (floor, margin, 0/garbage)

The run_job fail-fast test shape follows @necoweb3's #63959.

Co-authored-by: dsad <sswdarius@gmail.com>
This commit is contained in:
kshitij 2026-08-07 17:57:20 +05:30 committed by kshitij
parent 5fcca432f5
commit 30679b876c
1 changed files with 84 additions and 0 deletions

View File

@ -10,6 +10,7 @@ jobs as readers (concurrent with each other, excluded from a writer's run).
These tests assert that contract.
"""
import os
import threading
import time
@ -214,3 +215,86 @@ def test_run_job_releases_cwd_lock_when_body_raises(tmp_path):
t.start()
assert acquired.wait(timeout=5), "writer lock was leaked by run_job on exception"
t.join(timeout=5)
def test_run_job_fails_fast_when_cwd_lock_is_stuck(monkeypatch):
"""Fail-closed (#79768): a reader blocked past the bound FAILS as a
normal cron error instead of proceeding without the lock (which would
expose the stuck writer's TERMINAL_CWD override to its commands)."""
import cron.scheduler as sched
monkeypatch.setattr(sched, "_cwd_lock_timeout_seconds", lambda: 0.05)
sched._terminal_cwd_lock.acquire_write()
try:
start = time.monotonic()
success, _out, _final, error = sched.run_job(
{"id": "blocked-reader", "name": "blocked-reader", "prompt": "hi"}
)
finally:
sched._terminal_cwd_lock.release_write()
assert success is False
assert "TERMINAL_CWD read lock" in (error or "")
assert time.monotonic() - start < 10.0
def test_run_job_writer_fails_fast_and_never_sets_env(monkeypatch, tmp_path):
"""A WRITER that times out must fail before touching TERMINAL_CWD —
the fail-open design let it clobber the active holder's override."""
import cron.scheduler as sched
monkeypatch.setattr(sched, "_cwd_lock_timeout_seconds", lambda: 0.05)
monkeypatch.setenv("TERMINAL_CWD", "/holder/dir")
sched._terminal_cwd_lock.acquire_write()
try:
success, _out, _final, error = sched.run_job(
{
"id": "blocked-writer",
"name": "blocked-writer",
"prompt": "hi",
"workdir": str(tmp_path),
}
)
observed_during = os.environ.get("TERMINAL_CWD")
finally:
sched._terminal_cwd_lock.release_write()
assert success is False
assert "TERMINAL_CWD write lock" in (error or "")
assert observed_during == "/holder/dir", (
"timed-out writer mutated the active holder's TERMINAL_CWD override"
)
assert os.environ.get("TERMINAL_CWD") == "/holder/dir"
def test_lock_wait_shorter_than_bound_still_succeeds(monkeypatch, tmp_path):
"""A waiter whose holder finishes inside the bound proceeds normally —
the fail-closed path only fires past the derived ceiling."""
import cron.scheduler as sched
lock = sched._terminal_cwd_lock
lock.acquire_write()
releaser = threading.Timer(0.1, lock.release_write)
releaser.start()
try:
assert lock.acquire_read(timeout=5.0) is True
lock.release_read()
finally:
releaser.cancel()
def test_cwd_lock_timeout_derivation(monkeypatch):
"""The bound tracks HERMES_CRON_TIMEOUT (+margin) with a floor, and
stays finite when the job runtime is unlimited (0)."""
import cron.scheduler as sched
monkeypatch.delenv("HERMES_CRON_TIMEOUT", raising=False)
assert sched._cwd_lock_timeout_seconds() == 660.0
monkeypatch.setenv("HERMES_CRON_TIMEOUT", "1800")
assert sched._cwd_lock_timeout_seconds() == 1860.0
monkeypatch.setenv("HERMES_CRON_TIMEOUT", "30")
assert sched._cwd_lock_timeout_seconds() == 180.0
monkeypatch.setenv("HERMES_CRON_TIMEOUT", "0")
assert sched._cwd_lock_timeout_seconds() == 660.0
monkeypatch.setenv("HERMES_CRON_TIMEOUT", "bogus")
assert sched._cwd_lock_timeout_seconds() == 660.0