Lazy creation of Downloader._slot_gc_loop. (#7210)

scrapy/core/downloader/__init__.py

@@ -108,6 +108,7 @@ def _get_concurrency_delay(
 
 class Downloader:
     DOWNLOAD_SLOT = "download_slot"
+    _SLOT_GC_INTERVAL: float = 60.0  # seconds
 
     def __init__(self, crawler: Crawler):
         self.crawler: Crawler = crawler
@@ -125,10 +126,7 @@ class Downloader:
         self.middleware: DownloaderMiddlewareManager = (
             DownloaderMiddlewareManager.from_crawler(crawler)
         )
-        self._slot_gc_loop: AsyncioLoopingCall | LoopingCall = create_looping_call(
-            self._slot_gc
-        )
-        self._slot_gc_loop.start(60)
+        self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None
         self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(
             "DOWNLOAD_SLOTS"
         )
@@ -172,6 +170,7 @@ class Downloader:
             randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
             new_slot = Slot(conc, delay, randomize_delay)
             self.slots[key] = new_slot
+            self._start_slot_gc()
 
         return key, self.slots[key]
 
@@ -273,7 +272,7 @@ class Downloader:
             queue_dfd.callback(response)  # awaited in _enqueue_request()
 
     def close(self) -> None:
-        self._slot_gc_loop.stop()
+        self._stop_slot_gc()
         for slot in self.slots.values():
             slot.close()
 
@@ -282,3 +281,14 @@ class Downloader:
         for key, slot in list(self.slots.items()):
             if not slot.active and slot.lastseen + slot.delay < mintime:
                 self.slots.pop(key).close()
+
+    def _start_slot_gc(self) -> None:
+        if self._slot_gc_loop:
+            return
+        self._slot_gc_loop = create_looping_call(self._slot_gc)
+        self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False)
+
+    def _stop_slot_gc(self) -> None:
+        if self._slot_gc_loop:
+            self._slot_gc_loop.stop()
+            self._slot_gc_loop = None

tests/test_downloaderslotssettings.py

@@ -99,9 +99,9 @@ def test_params():
     crawler = get_crawler(DefaultSpider, settings_dict=settings)
     crawler.spider = crawler._create_spider()
     downloader = Downloader(crawler)
-    downloader._slot_gc_loop.stop()  # Prevent an unclean reactor.
     request = Request("https://example.com")
     _, actual = downloader._get_slot(request)
+    downloader.close()
     expected = Slot(**params)
     for param in params:
         assert getattr(expected, param) == getattr(actual, param), (
@@ -113,7 +113,6 @@ def test_get_slot_deprecated_spider_arg():
     crawler = get_crawler(DefaultSpider)
     crawler.spider = crawler._create_spider()
     downloader = Downloader(crawler)
-    downloader._slot_gc_loop.stop()  # Prevent an unclean reactor.
     request = Request("https://example.com")
 
     with pytest.warns(
@@ -122,6 +121,7 @@ def test_get_slot_deprecated_spider_arg():
     ):
         key1, slot1 = downloader._get_slot(request, spider=crawler.spider)
     key2, slot2 = downloader._get_slot(request)
+    downloader.close()
 
     assert key1 == key2
     assert slot1 == slot2

tests/test_engine.py

@@ -614,7 +614,6 @@ def test_request_scheduled_signal(caplog):
 
     crawler = get_crawler(MySpider)
     engine = ExecutionEngine(crawler, lambda _: None)
-    engine.downloader._slot_gc_loop.stop()
     scheduler = TestScheduler()
 
     async def start():