diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 219509c1e..4412b5c1c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -776,6 +776,31 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 +.. setting:: DOWNLOAD_SLOTS + +DOWNLOAD_SLOTS +---------------- + +Default: ``{}`` + +Allows to define concurrency/delay parameters on per slot(domain) basis: + + .. code-block:: python + + DOWNLOAD_SLOTS = { + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + } + +.. note:: + + For other downloader slots default settings values will be used: + + - :setting:`DOWNLOAD_DELAY`: ``delay`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` + - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` + + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0b179da32..1b83c3a8a 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -84,6 +84,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) + self.per_slot_settings = self.settings.getdict("DOWNLOAD_SLOTS", {}) def fetch(self, request, spider): def _deactivate(response): @@ -100,11 +101,18 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: + slot_settings = self.per_slot_settings.get(key, {}) conc = ( self.ip_concurrency if self.ip_concurrency else self.domain_concurrency ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.randomize_delay) + conc, delay = ( + slot_settings.get("concurrency", conc), + slot_settings.get("delay", delay), + ) + randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) + new_slot = Slot(conc, delay, randomize_delay) + self.slots[key] = new_slot return key, self.slots[key] diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py new file mode 100644 index 000000000..9d4072d19 --- /dev/null +++ b/tests/test_downloaderslotssettings.py @@ -0,0 +1,72 @@ +import time + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request +from tests.mockserver import MockServer +from tests.spiders import MetaSpider + + +class DownloaderSlotsSettingsTestSpider(MetaSpider): + name = "downloader_slots" + + custom_settings = { + "DOWNLOAD_DELAY": 1, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "DOWNLOAD_SLOTS": { + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False, + }, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + }, + } + + def start_requests(self): + self.times = {None: []} + + slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] + + for slot in slots: + url = self.mockserver.url(f"/?downloader_slot={slot}") + self.times[slot] = [] + yield Request(url, callback=self.parse, meta={"download_slot": slot}) + + def parse(self, response): + slot = response.meta.get("download_slot", None) + self.times[slot].append(time.time()) + url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") + yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) + + def not_parse(self, response): + slot = response.meta.get("download_slot", None) + self.times[slot].append(time.time()) + + +class CrawlTestCase(TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_delay(self): + crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + yield crawler.crawl(mockserver=self.mockserver) + slots = crawler.engine.downloader.slots + times = crawler.spider.times + tolerance = 0.3 + + delays_real = {k: v[1] - v[0] for k, v in times.items()} + error_delta = { + k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) + for k, v in slots.items() + } + + self.assertTrue(max(list(error_delta.values())) < tolerance)