diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 43069c50d..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False - }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False - } + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } .. note:: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3691df48a..ace483a23 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,13 +103,14 @@ class Downloader: if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) conc, delay = ( - slot_settings.get('concurrency', conc), - slot_settings.get('delay', delay) + slot_settings.get("concurrency", conc), + slot_settings.get("delay", delay), ) - randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) + randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5ee06d5bc..a167161a1 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,31 +1,31 @@ import time +from twisted.internet import defer +from twisted.trial.unittest import TestCase + from scrapy.crawler import CrawlerRunner from scrapy.http import Request from tests.mockserver import MockServer from tests.spiders import MetaSpider -from twisted.internet import defer -from twisted.trial.unittest import TestCase - class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = 'downloader_slots' + name = "downloader_slots" custom_settings = { "DOWNLOAD_DELAY": 1, "RANDOMIZE_DOWNLOAD_DELAY": False, "DOWNLOAD_SLOTS": { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False + "books.toscrape.com": { + "delay": 3, + "randomize_delay": False } } } @@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") self.times[slot] = [] - yield Request(url, callback=self.parse, meta={'download_slot': slot}) + yield Request(url, callback=self.parse, meta={"download_slot": slot}) def parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") - yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) @@ -70,6 +70,9 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} + error_delta = { + k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) + for k, v in slots.items() + } self.assertTrue(max(list(error_delta.values())) < tolerance)