diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f9de7ee23..6602f661d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -136,7 +136,9 @@ class ExecutionEngine: self.paused = False def _next_request(self) -> None: - assert self.slot is not None # typing + if self.slot is None: + return + assert self.spider is not None # typing if self.paused: @@ -184,7 +186,8 @@ class ExecutionEngine: d.addErrback(lambda f: logger.info('Error while removing request from slot', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) - d.addBoth(lambda _: self.slot.nextcall.schedule()) + slot = self.slot + d.addBoth(lambda _: slot.nextcall.schedule()) d.addErrback(lambda f: logger.info('Error while scheduling new request', exc_info=failure_to_exc_info(f), extra={'spider': self.spider})) diff --git a/tests/test_engine.py b/tests/test_engine.py index fa7d0c8d4..b8fd341f6 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,9 +12,11 @@ module with the ``runserver`` argument:: import os import re +import subprocess import sys import warnings from collections import defaultdict +from threading import Timer from urllib.parse import urlparse import attr @@ -502,6 +504,37 @@ class EngineTest(unittest.TestCase): self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning) self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated") + def test_short_timeout(self): + args = ( + sys.executable, + '-m', + 'scrapy.cmdline', + 'fetch', + '-s', + 'CLOSESPIDER_TIMEOUT=0.001', + '-s', + 'LOG_LEVEL=DEBUG', + 'http://toscrape.com', + ) + p = subprocess.Popen( + args, + stderr=subprocess.PIPE, + ) + + def kill_proc(): + p.kill() + p.communicate() + assert False, 'Command took too much time to complete' + + timer = Timer(15, kill_proc) + try: + timer.start() + _, stderr = p.communicate() + finally: + timer.cancel() + + self.assertNotIn(b'Traceback', stderr) + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver':