Merge pull request #5328 from GeorgeA92/per_slot_settings

Per slot settings

docs/topics/settings.rst

@@ -776,6 +776,31 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
 .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2
 
+.. setting:: DOWNLOAD_SLOTS
+
+DOWNLOAD_SLOTS
+----------------
+
+Default: ``{}``
+
+Allows to define concurrency/delay parameters on per slot(domain) basis:
+
+    .. code-block:: python
+
+        DOWNLOAD_SLOTS = {
+            "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
+            "books.toscrape.com": {"delay": 3, "randomize_delay": False},
+        }
+
+.. note::
+
+    For other downloader slots default settings values will be used:
+
+    -   :setting:`DOWNLOAD_DELAY`: ``delay``
+    -   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
+    -   :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
+
+
 .. setting:: DOWNLOAD_TIMEOUT
 
 DOWNLOAD_TIMEOUT

scrapy/core/downloader/__init__.py

@@ -84,6 +84,7 @@ class Downloader:
         self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
         self._slot_gc_loop = task.LoopingCall(self._slot_gc)
         self._slot_gc_loop.start(60)
+        self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {})
 
     def fetch(self, request, spider):
         def _deactivate(response):
@@ -100,11 +101,18 @@ class Downloader:
     def _get_slot(self, request, spider):
         key = self._get_slot_key(request, spider)
         if key not in self.slots:
+            slot_settings = self.per_slot_settings.get(key, {})
             conc = (
                 self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
             )
             conc, delay = _get_concurrency_delay(conc, spider, self.settings)
-            self.slots[key] = Slot(conc, delay, self.randomize_delay)
+            conc, delay = (
+                slot_settings.get("concurrency", conc),
+                slot_settings.get("delay", delay),
+            )
+            randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
+            new_slot = Slot(conc, delay, randomize_delay)
+            self.slots[key] = new_slot
 
         return key, self.slots[key]
 

tests/test_downloaderslotssettings.py

@@ -0,0 +1,72 @@
+import time
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
+from tests.mockserver import MockServer
+from tests.spiders import MetaSpider
+
+
+class DownloaderSlotsSettingsTestSpider(MetaSpider):
+    name = "downloader_slots"
+
+    custom_settings = {
+        "DOWNLOAD_DELAY": 1,
+        "RANDOMIZE_DOWNLOAD_DELAY": False,
+        "DOWNLOAD_SLOTS": {
+            "quotes.toscrape.com": {
+                "concurrency": 1,
+                "delay": 2,
+                "randomize_delay": False,
+            },
+            "books.toscrape.com": {"delay": 3, "randomize_delay": False},
+        },
+    }
+
+    def start_requests(self):
+        self.times = {None: []}
+
+        slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]
+
+        for slot in slots:
+            url = self.mockserver.url(f"/?downloader_slot={slot}")
+            self.times[slot] = []
+            yield Request(url, callback=self.parse, meta={"download_slot": slot})
+
+    def parse(self, response):
+        slot = response.meta.get("download_slot", None)
+        self.times[slot].append(time.time())
+        url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
+        yield Request(url, callback=self.not_parse, meta={"download_slot": slot})
+
+    def not_parse(self, response):
+        slot = response.meta.get("download_slot", None)
+        self.times[slot].append(time.time())
+
+
+class CrawlTestCase(TestCase):
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_delay(self):
+        crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        slots = crawler.engine.downloader.slots
+        times = crawler.spider.times
+        tolerance = 0.3
+
+        delays_real = {k: v[1] - v[0] for k, v in times.items()}
+        error_delta = {
+            k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
+            for k, v in slots.items()
+        }
+
+        self.assertTrue(max(list(error_delta.values())) < tolerance)