diff --git a/netbox/core/tests/test_management_commands.py b/netbox/core/tests/test_management_commands.py index 17b587c9c..aa05ec093 100644 --- a/netbox/core/tests/test_management_commands.py +++ b/netbox/core/tests/test_management_commands.py @@ -209,6 +209,28 @@ class RQWorkerTestCase(TestCase): with self.assertRaisesMessage(TypeError, 'System job must specify an interval'): call_command('rqworker', stdout=StringIO(), stderr=StringIO()) + def test_reconciles_stale_jobs_before_scheduling(self): + # Recovery of jobs stranded by a killed worker (#22714) must run before enqueue_once(), + # so a stale row can't be mistaken for a live schedule and block re-arming. + job = MagicMock() + job.name = 'TestJob' + manager = MagicMock() + + with ( + patch('core.management.commands.rqworker.registry', {'system_jobs': {job: {'interval': 5}}}), + patch('core.management.commands.rqworker.reconcile_stale_system_jobs') as reconcile, + patch('core.management.commands.rqworker._Command.handle'), + ): + manager.attach_mock(reconcile, 'reconcile') + manager.attach_mock(job.enqueue_once, 'enqueue_once') + call_command('rqworker', stdout=StringIO(), stderr=StringIO()) + + reconcile.assert_called_once_with(job, 5) + self.assertEqual( + [name for name, _, _ in manager.mock_calls], + ['reconcile', 'enqueue_once'], + ) + class SyncDataSourceTestCase(TestCase): class FakeDataSource: diff --git a/netbox/netbox/jobs.py b/netbox/netbox/jobs.py index 2bf6ecf7a..f0f127ea8 100644 --- a/netbox/netbox/jobs.py +++ b/netbox/netbox/jobs.py @@ -97,7 +97,10 @@ def reconcile_stale_system_jobs(job_class, interval): if not running: return - # The timeout is a property of the runner, not the individual job, so resolve it once. + # The timeout is a property of the runner, not the individual job, so resolve it once. Because + # the window sits past the job's own RQ timeout (the deadline at which RQ itself would kill a + # live run), a job still inside it is effectively never a live one, so this can't race a running + # worker's own terminate() under normal operation. grace = resolve_job_timeout(job_class, running[0]) + STALE_RUNNING_JOB_GRACE_SECONDS cutoff = timezone.now() - timedelta(seconds=grace)