From a752fa072e1acbb233191dbf04728db1fd6712a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 23 Nov 2020 22:58:54 +0100 Subject: [PATCH 01/12] Implement retry request functions and mixin --- scrapy/downloadermiddlewares/retry.py | 134 ++++++++++++++++++++------ 1 file changed, 106 insertions(+), 28 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 51fe59254..023ab7d60 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -31,6 +31,105 @@ from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) +def get_retry_request( + request, + *, + reason, + spider, + max_retry_times=None, + priority_adjust=None, +): + settings = spider.crawler.settings + stats = spider.crawler.stats + retry_times = request.meta.get('retry_times', 0) + 1 + request_max_retry_times = request.meta.get( + 'max_retry_times', + max_retry_times, + ) + if request_max_retry_times is None: + request_max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= request_max_retry_times: + logger.debug( + "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider} + ) + new_request = request.copy() + new_request.meta['retry_times'] = retry_times + new_request.dont_filter = True + if priority_adjust is None: + priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + new_request.priority = request.priority + priority_adjust + + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + stats.inc_value('retry/count') + stats.inc_value(f'retry/reason_count/{reason}') + return new_request + else: + stats.inc_value('retry/max_reached') + logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}) + return None + + +def retry_request( + request, + *, + reason, + spider, + max_retry_times=None, + priority_adjust=None, +): + new_request = get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + if new_request: + return [new_request] + return [] + + +class RetrySpiderMixin: + + def get_retry_request( + self, + request, + *, + reason, + max_retry_times=None, + priority_adjust=None, + ): + return get_retry_request( + request, + reason=reason, + spider=self, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + + def retry_request( + self, + request, + *, + reason, + max_retry_times=None, + priority_adjust=None, + ): + return retry_request( + request, + reason=reason, + spider=self, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, + ) + + class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to @@ -67,31 +166,10 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): - retries = request.meta.get('retry_times', 0) + 1 - - retry_times = self.max_retry_times - - if 'max_retry_times' in request.meta: - retry_times = request.meta['max_retry_times'] - - stats = spider.crawler.stats - if retries <= retry_times: - logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) - retryreq = request.copy() - retryreq.meta['retry_times'] = retries - retryreq.dont_filter = True - retryreq.priority = request.priority + self.priority_adjust - - if isinstance(reason, Exception): - reason = global_object_name(reason.__class__) - - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') - return retryreq - else: - stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", - {'request': request, 'retries': retries, 'reason': reason}, - extra={'spider': spider}) + return get_retry_request( + request, + reason=reason, + spider=spider, + max_retry_times=self.max_retry_times, + priority_adjust=self.priority_adjust, + ) From 5fc27b1e6fc8532dc5bc08fc5abf5a0daa8ef0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:09:06 +0100 Subject: [PATCH 02/12] Remove RetrySpiderMixin and retry_request --- scrapy/downloadermiddlewares/retry.py | 55 --------------------------- 1 file changed, 55 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 023ab7d60..5963dacdf 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -75,61 +75,6 @@ def get_retry_request( return None -def retry_request( - request, - *, - reason, - spider, - max_retry_times=None, - priority_adjust=None, -): - new_request = get_retry_request( - request, - reason=reason, - spider=spider, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - if new_request: - return [new_request] - return [] - - -class RetrySpiderMixin: - - def get_retry_request( - self, - request, - *, - reason, - max_retry_times=None, - priority_adjust=None, - ): - return get_retry_request( - request, - reason=reason, - spider=self, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - - def retry_request( - self, - request, - *, - reason, - max_retry_times=None, - priority_adjust=None, - ): - return retry_request( - request, - reason=reason, - spider=self, - max_retry_times=max_retry_times, - priority_adjust=priority_adjust, - ) - - class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to From 825462615a8df8e2274cf71c8a9bbea68c89262a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:09:48 +0100 Subject: [PATCH 03/12] =?UTF-8?q?get=5Fretry=5Frequest:=20set=20the=20defa?= =?UTF-8?q?ult=20retry=20reason=20to=20=E2=80=9Cunspecified=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5963dacdf..b8ead12ce 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -34,8 +34,8 @@ logger = logging.getLogger(__name__) def get_retry_request( request, *, - reason, spider, + reason='unspecified', max_retry_times=None, priority_adjust=None, ): From ec836dcc9290f50bad27c874cd0c25a87781735c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:15:28 +0100 Subject: [PATCH 04/12] Solve style issues --- scrapy/downloadermiddlewares/retry.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index b8ead12ce..5f9bc756c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -69,9 +69,12 @@ def get_retry_request( return new_request else: stats.inc_value('retry/max_reached') - logger.error("Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", - {'request': request, 'retry_times': retry_times, 'reason': reason}, - extra={'spider': spider}) + logger.error( + "Gave up retrying %(request)s (failed %(retry_times)d times): " + "%(reason)s", + {'request': request, 'retry_times': retry_times, 'reason': reason}, + extra={'spider': spider}, + ) return None From 6ab990181c6502624ceb0cca6783d99b30d30c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 14:48:03 +0100 Subject: [PATCH 05/12] Document get_retry_requests --- docs/topics/downloader-middleware.rst | 17 ++++++++++ docs/topics/settings.rst | 14 -------- scrapy/downloadermiddlewares/retry.py | 49 +++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6801adc9c..b539c23df 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -892,6 +892,11 @@ settings (see the settings documentation for more info): If :attr:`Request.meta ` has ``dont_retry`` key set to True, the request will be ignored by this middleware. +To retry requests from a spider callback, you can use the +:func:`get_retry_request` function: + +.. autofunction:: get_retry_request + RetryMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because it is a common code used to indicate server overload. It is not included by default because HTTP specs say so. +.. setting:: RETRY_PRIORITY_ADJUST + +RETRY_PRIORITY_ADJUST +--------------------- + +Default: ``-1`` + +Adjust retry request priority relative to original request: + +- a positive priority adjust means higher priority. +- **a negative priority adjust (default) means lower priority.** + .. _topics-dlmw-robots: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0086a6c74..7c5e9ef6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1188,20 +1188,6 @@ Adjust redirect request priority relative to original request: - **a positive priority adjust (default) means higher priority.** - a negative priority adjust means lower priority. -.. setting:: RETRY_PRIORITY_ADJUST - -RETRY_PRIORITY_ADJUST ---------------------- - -Default: ``-1`` - -Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware`` - -Adjust retry request priority relative to original request: - -- a positive priority adjust means higher priority. -- **a negative priority adjust (default) means lower priority.** - .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5f9bc756c..046e3ea71 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -39,16 +39,51 @@ def get_retry_request( max_retry_times=None, priority_adjust=None, ): + """ + Returns a new :class:`~scrapy.Request` object to retry the specified + request, or ``None`` if retries of the specified request have been + exhausted. + + For example, in a :class:`~scrapy.Spider` callback, you could use it as + follows:: + + def parse(self, response): + if not response.text: + new_request = get_retry_request( + response.request, + spider=self, + reason='empty', + ) + if new_request: + yield new_request + return + + *spider* is the :class:`~scrapy.Spider` instance which is asking for the + retry request. It is used to access the :ref:`settings ` + and :ref:`stats `, and to provide extra logging context (see + :func:`logging.debug`). + + *reason* is a string or an :class:`Exception` object that indicates the + reason why the request needs to be retried. It is used to name retry stats. + + *max_retry_times* is a number that determines the maximum number of times + that *request* can be retried. If not specified or ``None``, the number is + read from the :reqmeta:`max_retry_times` meta key of the request. If the + :reqmeta:`max_retry_times` meta key is not defined or ``None``, the number + is read from the :setting:`RETRY_TIMES` setting. + + *priority_adjust* is a number that determines how the priority of the new + request changes in relation to *request*. If not specified, the number is + read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + """ settings = spider.crawler.settings stats = spider.crawler.stats retry_times = request.meta.get('retry_times', 0) + 1 - request_max_retry_times = request.meta.get( - 'max_retry_times', - max_retry_times, - ) - if request_max_retry_times is None: - request_max_retry_times = settings.getint('RETRY_TIMES') - if retry_times <= request_max_retry_times: + if max_retry_times is None: + max_retry_times = request.meta.get('max_retry_times') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') + if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", {'request': request, 'retry_times': retry_times, 'reason': reason}, From 80f5003c88f8ee4c5529c1a6d7fc3981efef511d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:38:38 +0100 Subject: [PATCH 06/12] Add tests for get_retry_request --- scrapy/downloadermiddlewares/retry.py | 13 +- tests/test_downloadermiddleware_retry.py | 498 ++++++++++++++++++++--- 2 files changed, 459 insertions(+), 52 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 046e3ea71..0f24e5d28 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -10,6 +10,7 @@ Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ import logging +from inspect import isclass from twisted.internet import defer from twisted.internet.error import ( @@ -81,8 +82,8 @@ def get_retry_request( retry_times = request.meta.get('retry_times', 0) + 1 if max_retry_times is None: max_retry_times = request.meta.get('max_retry_times') - if max_retry_times is None: - max_retry_times = settings.getint('RETRY_TIMES') + if max_retry_times is None: + max_retry_times = settings.getint('RETRY_TIMES') if retry_times <= max_retry_times: logger.debug( "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", @@ -96,6 +97,8 @@ def get_retry_request( priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') new_request.priority = request.priority + priority_adjust + if isclass(reason): + reason = reason() if isinstance(reason, Exception): reason = global_object_name(reason.__class__) @@ -149,10 +152,12 @@ class RetryMiddleware: return self._retry(request, exception, spider) def _retry(self, request, reason, spider): + max_retry_times = request.meta.get('max_retry_times', self.max_retry_times) + priority_adjust = request.meta.get('priority_adjust', self.priority_adjust) return get_retry_request( request, reason=reason, spider=spider, - max_retry_times=self.max_retry_times, - priority_adjust=self.priority_adjust, + max_retry_times=max_retry_times, + priority_adjust=priority_adjust, ) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 364ce0c89..cf01a7dff 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -4,18 +4,22 @@ from twisted.internet.error import ( ConnectError, ConnectionDone, ConnectionLost, - ConnectionRefusedError, DNSLookupError, TCPTimedOutError, - TimeoutError, ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import RetryMiddleware -from scrapy.spiders import Spider +from scrapy.downloadermiddlewares.retry import ( + get_retry_request, + RetryMiddleware, +) +from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from testfixtures import LogCapture + class RetryTest(unittest.TestCase): def setUp(self): @@ -119,82 +123,480 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): - def setUp(self): - self.crawler = get_crawler(Spider) - self.spider = self.crawler._create_spider('foo') - self.mw = RetryMiddleware.from_crawler(self.crawler) - self.mw.max_retry_times = 2 - self.invalid_url = 'http://www.scrapytest.org/invalid_url' + + invalid_url = 'http://www.scrapytest.org/invalid_url' + + def get_spider_and_middleware(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + spider = crawler._create_spider('foo') + middleware = RetryMiddleware.from_crawler(crawler) + return spider, middleware def test_with_settings_zero(self): - - # SETTINGS: RETRY_TIMES = 0 - self.mw.max_retry_times = 0 - + max_retry_times = 0 + settings = {'RETRY_TIMES': max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) - self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_zero(self): - - # SETTINGS: meta(max_retry_times) = 0 - meta_max_retry_times = 0 - - req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) - self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) + max_retry_times = 0 + spider, middleware = self.get_spider_and_middleware() + meta = {'max_retry_times': max_retry_times} + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_without_metakey(self): - - # SETTINGS: RETRY_TIMES is NON-ZERO - self.mw.max_retry_times = 5 - + max_retry_times = 5 + settings = {'RETRY_TIMES': max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) - self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + self._test_retry( + req, + DNSLookupError('foo'), + max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_greater(self): - - # SETINGS: RETRY_TIMES < meta(max_retry_times) - self.mw.max_retry_times = 2 meta_max_retry_times = 3 + middleware_max_retry_times = 2 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_metakey_lesser(self): - - # SETINGS: RETRY_TIMES > meta(max_retry_times) - self.mw.max_retry_times = 5 meta_max_retry_times = 4 + middleware_max_retry_times = 5 req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) req2 = Request(self.invalid_url) - self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + settings = {'RETRY_TIMES': middleware_max_retry_times} + spider, middleware = self.get_spider_and_middleware(settings) + + self._test_retry( + req1, + DNSLookupError('foo'), + meta_max_retry_times, + spider=spider, + middleware=middleware, + ) + self._test_retry( + req2, + DNSLookupError('foo'), + middleware_max_retry_times, + spider=spider, + middleware=middleware, + ) def test_with_dont_retry(self): + max_retry_times = 4 + spider, middleware = self.get_spider_and_middleware() + meta = { + 'max_retry_times': max_retry_times, + 'dont_retry': True, + } + req = Request(self.invalid_url, meta=meta) + self._test_retry( + req, + DNSLookupError('foo'), + 0, + spider=spider, + middleware=middleware, + ) - # SETTINGS: meta(max_retry_times) = 4 - meta_max_retry_times = 4 - - req = Request(self.invalid_url, meta={ - 'max_retry_times': meta_max_retry_times, 'dont_retry': True - }) - - self._test_retry(req, DNSLookupError('foo'), 0) - - def _test_retry(self, req, exception, max_retry_times): + def _test_retry( + self, + req, + exception, + max_retry_times, + spider=None, + middleware=None, + ): + spider = spider or self.spider + middleware = middleware or self.mw for i in range(0, max_retry_times): - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) assert isinstance(req, Request) # discard it - req = self.mw.process_exception(req, exception, self.spider) + req = middleware.process_exception(req, exception, spider) self.assertEqual(req, None) +class GetRetryRequestTest(unittest.TestCase): + + def get_spider(self, settings=None): + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider('foo') + + def test_basic_usage(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_max_retries_reached(self): + request = Request('https://example.com') + spider = self.get_spider() + max_retry_times = 0 + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_one_retry(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=1, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = 1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -1) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_two_retries(self): + spider = self.get_spider() + request = Request('https://example.com') + new_request = request + max_retry_times = 2 + for index in range(max_retry_times): + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertIsInstance(new_request, Request) + self.assertNotEqual(new_request, request) + self.assertEqual(new_request.dont_filter, True) + expected_retry_times = index+1 + self.assertEqual(new_request.meta['retry_times'], expected_retry_times) + self.assertEqual(new_request.priority, -expected_retry_times) + expected_reason = "unspecified" + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + value = spider.crawler.stats.get_value(stat) + self.assertEqual(value, expected_retry_times) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + with LogCapture() as log: + new_request = get_retry_request( + new_request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + self.assertEqual( + spider.crawler.stats.get_value('retry/max_reached'), + 1 + ) + failure_count = max_retry_times + 1 + expected_reason = "unspecified" + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed {failure_count} times): " + f"{expected_reason}", + ) + ) + + def test_no_spider(self): + request = Request('https://example.com') + with self.assertRaises(TypeError): + get_retry_request(request) + + def test_max_retry_times_setting(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_meta(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request, None) + + def test_max_retry_times_argument(self): + max_retry_times = 0 + spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1}) + meta = {'max_retry_times': max_retry_times + 1} + request = Request('https://example.com', meta=meta) + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=max_retry_times, + ) + self.assertEqual(new_request, None) + + def test_priority_adjust_setting(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_priority_adjust_argument(self): + priority_adjust = 1 + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1}) + request = Request('https://example.com') + new_request = get_retry_request( + request, + spider=spider, + priority_adjust=priority_adjust, + ) + self.assertEqual(new_request.priority, priority_adjust) + + def test_log_extra_retry_success(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + new_request = get_retry_request( + request, + spider=spider, + ) + log.check_present(spider) + + def test_log_extra_retries_exceeded(self): + request = Request('https://example.com') + spider = self.get_spider() + with LogCapture(attributes=('spider',)) as log: + new_request = get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present(spider) + + def test_reason_string(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = 'because' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + for stat in ('retry/count', f'retry/reason_count/{expected_reason}'): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError() + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_builtin_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = NotImplementedError + expected_reason_string = 'builtins.NotImplementedError' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest() + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + def test_reason_custom_exception_class(self): + request = Request('https://example.com') + spider = self.get_spider() + expected_reason = IgnoreRequest + expected_reason_string = 'scrapy.exceptions.IgnoreRequest' + with LogCapture() as log: + new_request = get_retry_request( + request, + spider=spider, + reason=expected_reason, + ) + expected_retry_times = 1 + stat = spider.crawler.stats.get_value( + f'retry/reason_count/{expected_reason_string}' + ) + self.assertEqual(stat, 1) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "DEBUG", + f"Retrying {request} (failed {expected_retry_times} times): " + f"{expected_reason}", + ) + ) + + if __name__ == "__main__": unittest.main() From 722a33a2ac8ebc22bb7a7056598898dcb76e98a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:42:38 +0100 Subject: [PATCH 07/12] Fix style issues --- tests/test_downloadermiddleware_retry.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index cf01a7dff..61c0aaf2f 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -357,7 +357,7 @@ class GetRetryRequestTest(unittest.TestCase): self.assertIsInstance(new_request, Request) self.assertNotEqual(new_request, request) self.assertEqual(new_request.dont_filter, True) - expected_retry_times = index+1 + expected_retry_times = index + 1 self.assertEqual(new_request.meta['retry_times'], expected_retry_times) self.assertEqual(new_request.priority, -expected_retry_times) expected_reason = "unspecified" @@ -445,7 +445,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_priority_adjust_argument(self): priority_adjust = 1 - spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust+1}) + spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1}) request = Request('https://example.com') new_request = get_retry_request( request, @@ -458,7 +458,7 @@ class GetRetryRequestTest(unittest.TestCase): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider',)) as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, ) @@ -468,7 +468,7 @@ class GetRetryRequestTest(unittest.TestCase): request = Request('https://example.com') spider = self.get_spider() with LogCapture(attributes=('spider',)) as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, max_retry_times=0, @@ -480,7 +480,7 @@ class GetRetryRequestTest(unittest.TestCase): spider = self.get_spider() expected_reason = 'because' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -503,7 +503,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = NotImplementedError() expected_reason_string = 'builtins.NotImplementedError' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -528,7 +528,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = NotImplementedError expected_reason_string = 'builtins.NotImplementedError' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -553,7 +553,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = IgnoreRequest() expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, @@ -578,7 +578,7 @@ class GetRetryRequestTest(unittest.TestCase): expected_reason = IgnoreRequest expected_reason_string = 'scrapy.exceptions.IgnoreRequest' with LogCapture() as log: - new_request = get_retry_request( + get_retry_request( request, spider=spider, reason=expected_reason, From 1f7665c4cfb955ca4e81d7bdb249cd88ada788c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 22 Feb 2021 16:48:10 +0100 Subject: [PATCH 08/12] Silence a PyLint check on a mistake made for testing purposes --- tests/test_downloadermiddleware_retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 61c0aaf2f..46e525f99 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -398,7 +398,7 @@ class GetRetryRequestTest(unittest.TestCase): def test_no_spider(self): request = Request('https://example.com') with self.assertRaises(TypeError): - get_retry_request(request) + get_retry_request(request) # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 From 9e62355271fa39e67b06f00f5601cdb848c7894e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 2 Mar 2021 12:09:10 -0300 Subject: [PATCH 09/12] Allow logger/stats customization in get_retry_request --- scrapy/downloadermiddlewares/retry.py | 16 ++++++--- tests/test_downloadermiddleware_retry.py | 44 ++++++++++++++++++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 0f24e5d28..8955c7e4f 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -29,7 +29,8 @@ from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError from scrapy.utils.python import global_object_name -logger = logging.getLogger(__name__) + +retry_logger = logging.getLogger(__name__) def get_retry_request( @@ -39,6 +40,8 @@ def get_retry_request( reason='unspecified', max_retry_times=None, priority_adjust=None, + logger=retry_logger, + stats_base_key='retry', ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -76,6 +79,11 @@ def get_retry_request( *priority_adjust* is a number that determines how the priority of the new request changes in relation to *request*. If not specified, the number is read from the :setting:`RETRY_PRIORITY_ADJUST` setting. + + *logger* is the logging.Logger object to be used when logging messages + + *stats_base_key* is a string to be used as the base key for the + retry-related job stats """ settings = spider.crawler.settings stats = spider.crawler.stats @@ -102,11 +110,11 @@ def get_retry_request( if isinstance(reason, Exception): reason = global_object_name(reason.__class__) - stats.inc_value('retry/count') - stats.inc_value(f'retry/reason_count/{reason}') + stats.inc_value(f'{stats_base_key}/count') + stats.inc_value(f'{stats_base_key}/reason_count/{reason}') return new_request else: - stats.inc_value('retry/max_reached') + stats.inc_value(f'{stats_base_key}/max_reached') logger.error( "Gave up retrying %(request)s (failed %(retry_times)d times): " "%(reason)s", diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 46e525f99..915bd3a3e 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,4 +1,7 @@ +import logging import unittest + +from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.error import ( ConnectError, @@ -9,17 +12,12 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.downloadermiddlewares.retry import ( - get_retry_request, - RetryMiddleware, -) +from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from testfixtures import LogCapture - class RetryTest(unittest.TestCase): def setUp(self): @@ -597,6 +595,40 @@ class GetRetryRequestTest(unittest.TestCase): ) ) + def test_custom_logger(self): + logger = logging.getLogger("custom-logger") + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + reason=expected_reason, + logger=logger, + ) + log.check_present( + ( + "custom-logger", + "DEBUG", + f"Retrying {request} (failed 1 times): {expected_reason}", + ) + ) + + def test_custom_stats_key(self): + request = Request("https://example.com") + spider = self.get_spider() + expected_reason = "because" + stats_key = "custom_retry" + get_retry_request( + request, + spider=spider, + reason=expected_reason, + stats_base_key=stats_key, + ) + for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"): + self.assertEqual(spider.crawler.stats.get_value(stat), 1) + if __name__ == "__main__": unittest.main() From c0f3ca193873cd4dbf4de730dcceb38967efcc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 12 Mar 2021 14:02:48 +0100 Subject: [PATCH 10/12] get_retry_request: add typing information --- scrapy/downloadermiddlewares/retry.py | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 8955c7e4f..5e49a284a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -9,8 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry Failed pages are collected on the scraping process and rescheduled at the end, once the spider has finished crawling all regular (non failed) pages. """ -import logging -from inspect import isclass +from logging import getLogger, Logger +from typing import Optional, Union from twisted.internet import defer from twisted.internet.error import ( @@ -24,24 +24,26 @@ from twisted.internet.error import ( ) from twisted.web.client import ResponseFailed -from scrapy.exceptions import NotConfigured -from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.exceptions import NotConfigured +from scrapy.http.request import Request +from scrapy.spiders import Spider from scrapy.utils.python import global_object_name +from scrapy.utils.response import response_status_message -retry_logger = logging.getLogger(__name__) +retry_logger = getLogger(__name__) def get_retry_request( - request, + request: Request, *, - spider, - reason='unspecified', - max_retry_times=None, - priority_adjust=None, - logger=retry_logger, - stats_base_key='retry', + spider: Spider, + reason: Union[str, Exception] = 'unspecified', + max_retry_times: Optional[int] = None, + priority_adjust: Union[int, float, None] = None, + logger: Logger = retry_logger, + stats_base_key: str = 'retry', ): """ Returns a new :class:`~scrapy.Request` object to retry the specified @@ -105,7 +107,7 @@ def get_retry_request( priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') new_request.priority = request.priority + priority_adjust - if isclass(reason): + if callable(reason): reason = reason() if isinstance(reason, Exception): reason = global_object_name(reason.__class__) From 94201612bcad7b74c60a2a7ab70f40ba87714ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 18 Mar 2021 23:35:47 +0100 Subject: [PATCH 11/12] Simplify the get_retry_request code example --- scrapy/downloadermiddlewares/retry.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5e49a284a..2721db7cf 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -55,14 +55,12 @@ def get_retry_request( def parse(self, response): if not response.text: - new_request = get_retry_request( + new_request_or_none = get_retry_request( response.request, spider=self, reason='empty', ) - if new_request: - yield new_request - return + return new_request_or_none *spider* is the :class:`~scrapy.Spider` instance which is asking for the retry request. It is used to access the :ref:`settings ` From d458ccff3b2d6df94df1aa86eeb7d2505d62f2d6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 1 Apr 2021 12:27:35 -0300 Subject: [PATCH 12/12] Retry request: priority_adjust cannot be float (Request.priority is int) --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 2721db7cf..5965a1c6c 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -41,7 +41,7 @@ def get_retry_request( spider: Spider, reason: Union[str, Exception] = 'unspecified', max_retry_times: Optional[int] = None, - priority_adjust: Union[int, float, None] = None, + priority_adjust: Optional[int] = None, logger: Logger = retry_logger, stats_base_key: str = 'retry', ):