Merge pull request #5440 from Gallaecio/fix-closespider-exception

engine: prevent slot method call after unsetting the slot

scrapy/core/engine.py

@@ -136,7 +136,9 @@ class ExecutionEngine:
         self.paused = False
 
     def _next_request(self) -> None:
-        assert self.slot is not None  # typing
+        if self.slot is None:
+            return
+
         assert self.spider is not None  # typing
 
         if self.paused:
@@ -184,7 +186,8 @@ class ExecutionEngine:
         d.addErrback(lambda f: logger.info('Error while removing request from slot',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))
-        d.addBoth(lambda _: self.slot.nextcall.schedule())
+        slot = self.slot
+        d.addBoth(lambda _: slot.nextcall.schedule())
         d.addErrback(lambda f: logger.info('Error while scheduling new request',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))

tests/test_engine.py

@@ -12,9 +12,11 @@ module with the ``runserver`` argument::
 
 import os
 import re
+import subprocess
 import sys
 import warnings
 from collections import defaultdict
+from threading import Timer
 from urllib.parse import urlparse
 
 import attr
@@ -502,6 +504,37 @@ class EngineTest(unittest.TestCase):
             self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
             self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
 
+    def test_short_timeout(self):
+        args = (
+            sys.executable,
+            '-m',
+            'scrapy.cmdline',
+            'fetch',
+            '-s',
+            'CLOSESPIDER_TIMEOUT=0.001',
+            '-s',
+            'LOG_LEVEL=DEBUG',
+            'http://toscrape.com',
+        )
+        p = subprocess.Popen(
+            args,
+            stderr=subprocess.PIPE,
+        )
+
+        def kill_proc():
+            p.kill()
+            p.communicate()
+            assert False, 'Command took too much time to complete'
+
+        timer = Timer(15, kill_proc)
+        try:
+            timer.start()
+            _, stderr = p.communicate()
+        finally:
+            timer.cancel()
+
+        self.assertNotIn(b'Traceback', stderr)
+
 
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':