From eea56c4912ebabf94d366b4ead0d0d0a40a1d682 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:13:54 +0200 Subject: [PATCH 01/24] per slot settings: creation of Downloader.Slot objects from per slot settings added --- scrapy/core/downloader/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4f7ab594f..f680b9082 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -83,6 +83,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) + self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) def fetch(self, request, spider): def _deactivate(response): @@ -99,9 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.randomize_delay) + delay = self.per_slot_settings.get(key,{}).get('delay', delay) + randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + new_slot = Slot(conc, delay, randomize_delay) + self.slots[key] = new_slot + return key, self.slots[key] From d5acf88ca55988a6c92046da5ef2bf065b0b3e72 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 20:14:28 +0200 Subject: [PATCH 02/24] per slot settings: logging added (create Slot) --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f680b9082..9aab80171 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,3 +1,4 @@ +import logging import random from time import time from datetime import datetime @@ -12,6 +13,7 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers +logger = logging.getLogger(__name__) class Slot: """Downloader slot""" @@ -106,7 +108,7 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - + logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] From 1b7d7ecfcdb6067f3dccd69fc56e25dc1dbf498f Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:05:24 +0200 Subject: [PATCH 03/24] per slot settings: logging updated (create/close Slot) --- scrapy/core/downloader/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 9aab80171..461b211ab 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -108,7 +108,8 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug(f"Downloader slot created {'from per slot settings' if key in self.per_slot_settings.keys() else ''}: {new_slot}") + logger.debug( + f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -205,4 +206,7 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - self.slots.pop(key).close() + inactive_slot = self.slots.pop(key) + inactive_slot.close() + logger.debug( + f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 8185aa5265c663326474897c24aa0691a2ee0e5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Sat, 20 Nov 2021 21:06:08 +0200 Subject: [PATCH 04/24] per slot settings: codestyle fix --- scrapy/core/downloader/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 461b211ab..d5809ffa0 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -15,6 +15,7 @@ from scrapy.core.downloader.handlers import DownloadHandlers logger = logging.getLogger(__name__) + class Slot: """Downloader slot""" @@ -102,9 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key,{}).get('delay', delay) + delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 115d35270691176e128bd30f6a2a8bcf1289fe5c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:21:31 +0200 Subject: [PATCH 05/24] per slot settings: setting renamed from `PER_SLOT_SETTINGS` to `DOWNLOAD_SLOTS` --- scrapy/core/downloader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d5809ffa0..0e32714a8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,7 +86,7 @@ class Downloader: self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) - self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {}) + self.per_slot_settings = self.settings.getdict('DOWNLOAD_SLOTS', {}) def fetch(self, request, spider): def _deactivate(response): From be97402e46c5e4793e23b713728e1777fbe4d49c Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 11 Feb 2022 01:22:27 +0200 Subject: [PATCH 06/24] per slot settings: log notifications (debug) removed --- scrapy/core/downloader/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 0e32714a8..53dc546de 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -109,8 +109,6 @@ class Downloader: randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot - logger.debug( - f"Downloader slot '{key}' - created {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''}: {new_slot}") return key, self.slots[key] @@ -209,5 +207,3 @@ class Downloader: if not slot.active and slot.lastseen + slot.delay < mintime: inactive_slot = self.slots.pop(key) inactive_slot.close() - logger.debug( - f"Downloader slot '{key}' - closed {'(from per slot settings)' if key in self.per_slot_settings.keys() else ''} : {inactive_slot}") From 2006060688976ca469d794cdd3b753a00bfb83c9 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:29:53 +0300 Subject: [PATCH 07/24] per slot settings: codestyle(flake8) fix, code line length --- scrapy/core/downloader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 53dc546de..511693830 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,7 +103,9 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + conc = self.per_slot_settings.get(key, {}).get( + 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) delay = self.per_slot_settings.get(key, {}).get('delay', delay) randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) From 0ffc52a491e6e6c46196b3aa92767856f64a8ebc Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Wed, 11 May 2022 10:40:12 +0300 Subject: [PATCH 08/24] per slot settings: test added (delays for each download slots) --- tests/test_downloaderslotssettings.py | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 tests/test_downloaderslotssettings.py diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py new file mode 100644 index 000000000..6ea03fb78 --- /dev/null +++ b/tests/test_downloaderslotssettings.py @@ -0,0 +1,75 @@ +import time + +from scrapy.crawler import CrawlerRunner +from scrapy.http import Request + +from tests.mockserver import MockServer +from tests.spiders import MetaSpider + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + + +class DownloaderSlotsSettingsTestSpider(MetaSpider): + + name = 'downloader_slots' + + custom_settings = { + "DOWNLOAD_DELAY": 1, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "DOWNLOAD_SLOTS": { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 1.5, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 2, + 'randomize_delay': False + } + } + } + + def start_requests(self): + self.times = {None: []} + + slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + + for slot in slots: + url = self.mockserver.url(f"/?downloader_slot={slot}") + self.times[slot] = [] + yield Request(url, callback=self.parse, meta={'download_slot': slot}) + + def parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") + yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + + def not_parse(self, response): + slot = response.meta.get('download_slot', None) + self.times[slot].append(time.time()) + + +class CrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.runner = CrawlerRunner() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_delay(self): + crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + yield crawler.crawl(mockserver=self.mockserver) + slots = crawler.engine.downloader.slots + times = crawler.spider.times + tolerance = 0.3 + + delays_real = {k: v[1] - v[0] for k, v in times.items()} + error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + + self.assertTrue(max(list(error_delta.values())) < tolerance) From 4be9c969fd4bdc3206bdabfb5c22e524d9e355ac Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 29 Jul 2022 12:15:39 +0300 Subject: [PATCH 09/24] per slot settings: logger deleted as not used --- scrapy/core/downloader/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 511693830..fa8ac01e5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,3 @@ -import logging import random from time import time from datetime import datetime @@ -13,8 +12,6 @@ from scrapy import signals from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -logger = logging.getLogger(__name__) - class Slot: """Downloader slot""" From aabdd0b657e1b4398a5a1aad79044401e9dbf909 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 12 Aug 2022 15:34:48 +0300 Subject: [PATCH 10/24] per slot settings: logger deleted as not used (step 2) --- scrapy/core/downloader/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index fa8ac01e5..d908f4d7e 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -204,5 +204,4 @@ class Downloader: mintime = time() - age for key, slot in list(self.slots.items()): if not slot.active and slot.lastseen + slot.delay < mintime: - inactive_slot = self.slots.pop(key) - inactive_slot.close() + self.slots.pop(key).close() From 12d52a4f089798f266cbc6f86df3cc9c1cd58257 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:16:18 +0200 Subject: [PATCH 11/24] per slot settings: code optimized --- scrapy/core/downloader/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d908f4d7e..f16afe99b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -100,12 +100,13 @@ class Downloader: def _get_slot(self, request, spider): key = self._get_slot_key(request, spider) if key not in self.slots: - conc = self.per_slot_settings.get(key, {}).get( + slot_settings = self.per_slot_settings.get(key, {}) + conc = slot_settings.get( 'concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = self.per_slot_settings.get(key, {}).get('delay', delay) - randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay) + delay = slot_settings.get('delay', delay) + randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 042012f6bdaf4fbd5b978ff7cc9b6796286ac7c4 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Fri, 18 Nov 2022 14:56:29 +0200 Subject: [PATCH 12/24] per slot settings: error calculation metho updated --- tests/test_downloaderslotssettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 6ea03fb78..a092d01bf 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -70,6 +70,6 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()} + error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} self.assertTrue(max(list(error_delta.values())) < tolerance) From 80a86de5071aef1ac43277c273beace6bd426e80 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 8 Feb 2023 22:40:22 +0200 Subject: [PATCH 13/24] per_slot_settings: test delays increased --- tests/test_downloaderslotssettings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a092d01bf..5ee06d5bc 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -20,11 +20,11 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "DOWNLOAD_SLOTS": { 'quotes.toscrape.com': { 'concurrency': 1, - 'delay': 1.5, + 'delay': 2, 'randomize_delay': False }, 'books.toscrape.com': { - 'delay': 2, + 'delay': 3, 'randomize_delay': False } } From c3033a54b1e37287020d8f1f6b541ce4be8d2971 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Wed, 8 Feb 2023 23:55:07 +0200 Subject: [PATCH 14/24] per_slot_settings(docs): settings 1 --- docs/topics/settings.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d636dc301..bde33a0e9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -771,6 +771,14 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://tools.ietf.org/html/rfc7540#section-8.2 +.. setting:: DOWNLOAD_SLOTS + +DOWNLOAD_SLOTS +---------------- + +Default: ``{}`` + + .. setting:: DOWNLOAD_TIMEOUT DOWNLOAD_TIMEOUT From 5208d436ae26e61b6b00f305079c0a31b8e95063 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 9 Feb 2023 00:33:21 +0200 Subject: [PATCH 15/24] per_slot_settings(docs): settings 2 --- docs/topics/settings.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index bde33a0e9..477c3068f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -778,6 +778,24 @@ DOWNLOAD_SLOTS Default: ``{}`` +Allows to define concurrency/delay parameters on per slot(domain) basis: + + .. code-block:: python + + DOWNLOAD_SLOTS = { + 'quotes.toscrape.com': { + 'concurrency': 1, + 'delay': 2, + 'randomize_delay': False + }, + 'books.toscrape.com': { + 'delay': 3, + 'randomize_delay': False + } + } +wrapper around :func:`~scrapy.core.downloader.Slot` + +.. autofunction:: scrapy.core.downloader.Slot .. setting:: DOWNLOAD_TIMEOUT From 474087be6fadf2dd3440fc77ab757a18be0931c8 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 9 Feb 2023 14:52:28 +0200 Subject: [PATCH 16/24] per_slot_settings(docs): settings 3 --- docs/topics/settings.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 477c3068f..5acf09efe 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -793,9 +793,15 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: 'randomize_delay': False } } -wrapper around :func:`~scrapy.core.downloader.Slot` -.. autofunction:: scrapy.core.downloader.Slot +.. note:: + + For other downloader slots default settings values will be used: + + - ``DOWNLOAD_DELAY`` -> ``delay`` + - ``CONCURRENT_REQUESTS_PER_DOMAIN`` -> ``concurency`` + - ``RANDOMIZE_DOWNLOAD_DELAY`` -> ``randomize_delay`` + .. setting:: DOWNLOAD_TIMEOUT From 98a5958687b4fae213bf053e560bd18a69312f3f Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 14:52:31 +0200 Subject: [PATCH 17/24] per_slot_settings(docs): per slot settings - the highest priority --- scrapy/core/downloader/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 1a1eb819e..3691df48a 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -102,13 +102,13 @@ class Downloader: key = self._get_slot_key(request, spider) if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) - conc = slot_settings.get( - 'concurrency', ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency - ) - ) + conc = ( + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) conc, delay = _get_concurrency_delay(conc, spider, self.settings) - delay = slot_settings.get('delay', delay) + conc, delay = ( + slot_settings.get('concurrency', conc), + slot_settings.get('delay', delay) + ) randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot From 07e1429877e5128786b24ad146dabcad4784ffc2 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 15:14:01 +0200 Subject: [PATCH 18/24] per_slot_settings(docs): docs `:setting` bindings added --- docs/topics/settings.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5acf09efe..43069c50d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -798,9 +798,9 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - - ``DOWNLOAD_DELAY`` -> ``delay`` - - ``CONCURRENT_REQUESTS_PER_DOMAIN`` -> ``concurency`` - - ``RANDOMIZE_DOWNLOAD_DELAY`` -> ``randomize_delay`` + - :setting:`DOWNLOAD_DELAY`: ``delay`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` .. setting:: DOWNLOAD_TIMEOUT From 045092e8d753b9689b24db7d30bcbd7f976fb517 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 17:55:59 +0200 Subject: [PATCH 19/24] per_slot_settings(docs): code style fix (from previous pre-commit check) --- docs/topics/settings.rst | 11 ++------ scrapy/core/downloader/__init__.py | 9 ++++--- tests/test_downloaderslotssettings.py | 37 +++++++++++++++------------ 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 43069c50d..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: .. code-block:: python DOWNLOAD_SLOTS = { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False - }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False - } + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, } .. note:: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3691df48a..ace483a23 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -103,13 +103,14 @@ class Downloader: if key not in self.slots: slot_settings = self.per_slot_settings.get(key, {}) conc = ( - self.ip_concurrency if self.ip_concurrency else self.domain_concurrency) + self.ip_concurrency if self.ip_concurrency else self.domain_concurrency + ) conc, delay = _get_concurrency_delay(conc, spider, self.settings) conc, delay = ( - slot_settings.get('concurrency', conc), - slot_settings.get('delay', delay) + slot_settings.get("concurrency", conc), + slot_settings.get("delay", delay), ) - randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay) + randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) new_slot = Slot(conc, delay, randomize_delay) self.slots[key] = new_slot diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5ee06d5bc..a167161a1 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,31 +1,31 @@ import time +from twisted.internet import defer +from twisted.trial.unittest import TestCase + from scrapy.crawler import CrawlerRunner from scrapy.http import Request from tests.mockserver import MockServer from tests.spiders import MetaSpider -from twisted.internet import defer -from twisted.trial.unittest import TestCase - class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = 'downloader_slots' + name = "downloader_slots" custom_settings = { "DOWNLOAD_DELAY": 1, "RANDOMIZE_DOWNLOAD_DELAY": False, "DOWNLOAD_SLOTS": { - 'quotes.toscrape.com': { - 'concurrency': 1, - 'delay': 2, - 'randomize_delay': False + "quotes.toscrape.com": { + "concurrency": 1, + "delay": 2, + "randomize_delay": False }, - 'books.toscrape.com': { - 'delay': 3, - 'randomize_delay': False + "books.toscrape.com": { + "delay": 3, + "randomize_delay": False } } } @@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def start_requests(self): self.times = {None: []} - slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None] + slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None] for slot in slots: url = self.mockserver.url(f"/?downloader_slot={slot}") self.times[slot] = [] - yield Request(url, callback=self.parse, meta={'download_slot': slot}) + yield Request(url, callback=self.parse, meta={"download_slot": slot}) def parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") - yield Request(url, callback=self.not_parse, meta={'download_slot': slot}) + yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): - slot = response.meta.get('download_slot', None) + slot = response.meta.get("download_slot", None) self.times[slot].append(time.time()) @@ -70,6 +70,9 @@ class CrawlTestCase(TestCase): tolerance = 0.3 delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()} + error_delta = { + k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) + for k, v in slots.items() + } self.assertTrue(max(list(error_delta.values())) < tolerance) From dd5524eb986c0ef1d6ce6ec2887ed8aabe55f7c9 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 17:59:36 +0200 Subject: [PATCH 20/24] per_slot_settings(docs): code style fix 2 (from previous pre-commit check) --- tests/test_downloaderslotssettings.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index a167161a1..b965233a3 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -21,13 +21,10 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): "quotes.toscrape.com": { "concurrency": 1, "delay": 2, - "randomize_delay": False + "randomize_delay": False, }, - "books.toscrape.com": { - "delay": 3, - "randomize_delay": False - } - } + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + }, } def start_requests(self): From 864eee66c77ceff21ba0c8611aa84402f300fc27 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 16 Feb 2023 18:19:36 +0200 Subject: [PATCH 21/24] per_slot_settings(docs): code style fix 3 (from previous pre-commit check) --- tests/test_downloaderslotssettings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index b965233a3..9d4072d19 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -5,13 +5,11 @@ from twisted.trial.unittest import TestCase from scrapy.crawler import CrawlerRunner from scrapy.http import Request - from tests.mockserver import MockServer from tests.spiders import MetaSpider class DownloaderSlotsSettingsTestSpider(MetaSpider): - name = "downloader_slots" custom_settings = { @@ -49,7 +47,6 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): class CrawlTestCase(TestCase): - def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() From 68ba25cb69cb70c0144c9473c1ada46051c7af5b Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sat, 4 Mar 2023 22:15:06 +0200 Subject: [PATCH 22/24] periodic log stats: typo in docs fixed --- conftest.py | 2 +- docs/topics/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index e1d4b1213..01a83d94d 100644 --- a/conftest.py +++ b/conftest.py @@ -20,7 +20,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path("tests/ignores.txt").open(encoding="utf-8") as reader: +with Path(r"C:/Users/georg/PycharmProjects/scrapy/tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 175e8f7f0..fbd32ca7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From be52fe4f67647cea9670197feebbcdb05af6cf96 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 6 Mar 2023 16:24:28 +0200 Subject: [PATCH 23/24] Revert "periodic log stats: typo in docs fixed" This reverts commit 68ba25cb69cb70c0144c9473c1ada46051c7af5b. --- conftest.py | 2 +- docs/topics/settings.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 01a83d94d..e1d4b1213 100644 --- a/conftest.py +++ b/conftest.py @@ -20,7 +20,7 @@ collect_ignore = [ *_py_files("tests/CrawlerRunner"), ] -with Path(r"C:/Users/georg/PycharmProjects/scrapy/tests/ignores.txt").open(encoding="utf-8") as reader: +with Path("tests/ignores.txt").open(encoding="utf-8") as reader: for line in reader: file_path = line.strip() if file_path and file_path[0] != "#": diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fbd32ca7a..175e8f7f0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` From 218829b1db5e086db640a3f2ef3cb8ae76bb66bb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Mon, 6 Mar 2023 16:25:49 +0200 Subject: [PATCH 24/24] per slot settings: typo fixed --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 175e8f7f0..fbd32ca7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -792,7 +792,7 @@ Allows to define concurrency/delay parameters on per slot(domain) basis: For other downloader slots default settings values will be used: - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurency`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``