mirror of https://github.com/scrapy/scrapy.git
Merge pull request #4902 from Gallaecio/retry-request
Implement a retry request function
This commit is contained in:
Mikhail Korobov
GitHub
commit
d0e23486b4
|
|
@ -892,6 +892,11 @@ settings (see the settings documentation for more info):
|
|||
If :attr:`Request.meta <scrapy.http.Request.meta>` has ``dont_retry`` key
|
||||
set to True, the request will be ignored by this middleware.
|
||||
|
||||
To retry requests from a spider callback, you can use the
|
||||
:func:`get_retry_request` function:
|
||||
|
||||
.. autofunction:: get_retry_request
|
||||
|
||||
RetryMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
@ -932,6 +937,18 @@ In some cases you may want to add 400 to :setting:`RETRY_HTTP_CODES` because
|
|||
it is a common code used to indicate server overload. It is not included by
|
||||
default because HTTP specs say so.
|
||||
|
||||
.. setting:: RETRY_PRIORITY_ADJUST
|
||||
|
||||
RETRY_PRIORITY_ADJUST
|
||||
---------------------
|
||||
|
||||
Default: ``-1``
|
||||
|
||||
Adjust retry request priority relative to original request:
|
||||
|
||||
- a positive priority adjust means higher priority.
|
||||
- **a negative priority adjust (default) means lower priority.**
|
||||
|
||||
|
||||
.. _topics-dlmw-robots:
|
||||
|
||||
|
|
|
|||
|
|
@ -1229,20 +1229,6 @@ Adjust redirect request priority relative to original request:
|
|||
- **a positive priority adjust (default) means higher priority.**
|
||||
- a negative priority adjust means lower priority.
|
||||
|
||||
.. setting:: RETRY_PRIORITY_ADJUST
|
||||
|
||||
RETRY_PRIORITY_ADJUST
|
||||
---------------------
|
||||
|
||||
Default: ``-1``
|
||||
|
||||
Scope: ``scrapy.downloadermiddlewares.retry.RetryMiddleware``
|
||||
|
||||
Adjust retry request priority relative to original request:
|
||||
|
||||
- a positive priority adjust means higher priority.
|
||||
- **a negative priority adjust (default) means lower priority.**
|
||||
|
||||
.. setting:: ROBOTSTXT_OBEY
|
||||
|
||||
ROBOTSTXT_OBEY
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
|
|||
Failed pages are collected on the scraping process and rescheduled at the end,
|
||||
once the spider has finished crawling all regular (non failed) pages.
|
||||
"""
|
||||
import logging
|
||||
from logging import getLogger, Logger
|
||||
from typing import Optional, Union
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import (
|
||||
|
|
@ -23,12 +24,104 @@ from twisted.internet.error import (
|
|||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.utils.response import response_status_message
|
||||
from scrapy.core.downloader.handlers.http11 import TunnelError
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.python import global_object_name
|
||||
from scrapy.utils.response import response_status_message
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
retry_logger = getLogger(__name__)
|
||||
|
||||
|
||||
def get_retry_request(
|
||||
request: Request,
|
||||
*,
|
||||
spider: Spider,
|
||||
reason: Union[str, Exception] = 'unspecified',
|
||||
max_retry_times: Optional[int] = None,
|
||||
priority_adjust: Optional[int] = None,
|
||||
logger: Logger = retry_logger,
|
||||
stats_base_key: str = 'retry',
|
||||
):
|
||||
"""
|
||||
Returns a new :class:`~scrapy.Request` object to retry the specified
|
||||
request, or ``None`` if retries of the specified request have been
|
||||
exhausted.
|
||||
|
||||
For example, in a :class:`~scrapy.Spider` callback, you could use it as
|
||||
follows::
|
||||
|
||||
def parse(self, response):
|
||||
if not response.text:
|
||||
new_request_or_none = get_retry_request(
|
||||
response.request,
|
||||
spider=self,
|
||||
reason='empty',
|
||||
)
|
||||
return new_request_or_none
|
||||
|
||||
*spider* is the :class:`~scrapy.Spider` instance which is asking for the
|
||||
retry request. It is used to access the :ref:`settings <topics-settings>`
|
||||
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
|
||||
:func:`logging.debug`).
|
||||
|
||||
*reason* is a string or an :class:`Exception` object that indicates the
|
||||
reason why the request needs to be retried. It is used to name retry stats.
|
||||
|
||||
*max_retry_times* is a number that determines the maximum number of times
|
||||
that *request* can be retried. If not specified or ``None``, the number is
|
||||
read from the :reqmeta:`max_retry_times` meta key of the request. If the
|
||||
:reqmeta:`max_retry_times` meta key is not defined or ``None``, the number
|
||||
is read from the :setting:`RETRY_TIMES` setting.
|
||||
|
||||
*priority_adjust* is a number that determines how the priority of the new
|
||||
request changes in relation to *request*. If not specified, the number is
|
||||
read from the :setting:`RETRY_PRIORITY_ADJUST` setting.
|
||||
|
||||
*logger* is the logging.Logger object to be used when logging messages
|
||||
|
||||
*stats_base_key* is a string to be used as the base key for the
|
||||
retry-related job stats
|
||||
"""
|
||||
settings = spider.crawler.settings
|
||||
stats = spider.crawler.stats
|
||||
retry_times = request.meta.get('retry_times', 0) + 1
|
||||
if max_retry_times is None:
|
||||
max_retry_times = request.meta.get('max_retry_times')
|
||||
if max_retry_times is None:
|
||||
max_retry_times = settings.getint('RETRY_TIMES')
|
||||
if retry_times <= max_retry_times:
|
||||
logger.debug(
|
||||
"Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
|
||||
{'request': request, 'retry_times': retry_times, 'reason': reason},
|
||||
extra={'spider': spider}
|
||||
)
|
||||
new_request = request.copy()
|
||||
new_request.meta['retry_times'] = retry_times
|
||||
new_request.dont_filter = True
|
||||
if priority_adjust is None:
|
||||
priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
|
||||
new_request.priority = request.priority + priority_adjust
|
||||
|
||||
if callable(reason):
|
||||
reason = reason()
|
||||
if isinstance(reason, Exception):
|
||||
reason = global_object_name(reason.__class__)
|
||||
|
||||
stats.inc_value(f'{stats_base_key}/count')
|
||||
stats.inc_value(f'{stats_base_key}/reason_count/{reason}')
|
||||
return new_request
|
||||
else:
|
||||
stats.inc_value(f'{stats_base_key}/max_reached')
|
||||
logger.error(
|
||||
"Gave up retrying %(request)s (failed %(retry_times)d times): "
|
||||
"%(reason)s",
|
||||
{'request': request, 'retry_times': retry_times, 'reason': reason},
|
||||
extra={'spider': spider},
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
class RetryMiddleware:
|
||||
|
|
@ -67,31 +160,12 @@ class RetryMiddleware:
|
|||
return self._retry(request, exception, spider)
|
||||
|
||||
def _retry(self, request, reason, spider):
|
||||
retries = request.meta.get('retry_times', 0) + 1
|
||||
|
||||
retry_times = self.max_retry_times
|
||||
|
||||
if 'max_retry_times' in request.meta:
|
||||
retry_times = request.meta['max_retry_times']
|
||||
|
||||
stats = spider.crawler.stats
|
||||
if retries <= retry_times:
|
||||
logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
|
||||
{'request': request, 'retries': retries, 'reason': reason},
|
||||
extra={'spider': spider})
|
||||
retryreq = request.copy()
|
||||
retryreq.meta['retry_times'] = retries
|
||||
retryreq.dont_filter = True
|
||||
retryreq.priority = request.priority + self.priority_adjust
|
||||
|
||||
if isinstance(reason, Exception):
|
||||
reason = global_object_name(reason.__class__)
|
||||
|
||||
stats.inc_value('retry/count')
|
||||
stats.inc_value(f'retry/reason_count/{reason}')
|
||||
return retryreq
|
||||
else:
|
||||
stats.inc_value('retry/max_reached')
|
||||
logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
|
||||
{'request': request, 'retries': retries, 'reason': reason},
|
||||
extra={'spider': spider})
|
||||
max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
|
||||
priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
|
||||
return get_retry_request(
|
||||
request,
|
||||
reason=reason,
|
||||
spider=spider,
|
||||
max_retry_times=max_retry_times,
|
||||
priority_adjust=priority_adjust,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,19 +1,21 @@
|
|||
import logging
|
||||
import unittest
|
||||
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import (
|
||||
ConnectError,
|
||||
ConnectionDone,
|
||||
ConnectionLost,
|
||||
ConnectionRefusedError,
|
||||
DNSLookupError,
|
||||
TCPTimedOutError,
|
||||
TimeoutError,
|
||||
)
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy.downloadermiddlewares.retry import RetryMiddleware
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.downloadermiddlewares.retry import get_retry_request, RetryMiddleware
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
@ -119,82 +121,514 @@ class RetryTest(unittest.TestCase):
|
|||
|
||||
|
||||
class MaxRetryTimesTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.spider = self.crawler._create_spider('foo')
|
||||
self.mw = RetryMiddleware.from_crawler(self.crawler)
|
||||
self.mw.max_retry_times = 2
|
||||
self.invalid_url = 'http://www.scrapytest.org/invalid_url'
|
||||
|
||||
invalid_url = 'http://www.scrapytest.org/invalid_url'
|
||||
|
||||
def get_spider_and_middleware(self, settings=None):
|
||||
crawler = get_crawler(Spider, settings or {})
|
||||
spider = crawler._create_spider('foo')
|
||||
middleware = RetryMiddleware.from_crawler(crawler)
|
||||
return spider, middleware
|
||||
|
||||
def test_with_settings_zero(self):
|
||||
|
||||
# SETTINGS: RETRY_TIMES = 0
|
||||
self.mw.max_retry_times = 0
|
||||
|
||||
max_retry_times = 0
|
||||
settings = {'RETRY_TIMES': max_retry_times}
|
||||
spider, middleware = self.get_spider_and_middleware(settings)
|
||||
req = Request(self.invalid_url)
|
||||
self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
|
||||
self._test_retry(
|
||||
req,
|
||||
DNSLookupError('foo'),
|
||||
max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
|
||||
def test_with_metakey_zero(self):
|
||||
|
||||
# SETTINGS: meta(max_retry_times) = 0
|
||||
meta_max_retry_times = 0
|
||||
|
||||
req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
|
||||
self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
|
||||
max_retry_times = 0
|
||||
spider, middleware = self.get_spider_and_middleware()
|
||||
meta = {'max_retry_times': max_retry_times}
|
||||
req = Request(self.invalid_url, meta=meta)
|
||||
self._test_retry(
|
||||
req,
|
||||
DNSLookupError('foo'),
|
||||
max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
|
||||
def test_without_metakey(self):
|
||||
|
||||
# SETTINGS: RETRY_TIMES is NON-ZERO
|
||||
self.mw.max_retry_times = 5
|
||||
|
||||
max_retry_times = 5
|
||||
settings = {'RETRY_TIMES': max_retry_times}
|
||||
spider, middleware = self.get_spider_and_middleware(settings)
|
||||
req = Request(self.invalid_url)
|
||||
self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
|
||||
self._test_retry(
|
||||
req,
|
||||
DNSLookupError('foo'),
|
||||
max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
|
||||
def test_with_metakey_greater(self):
|
||||
|
||||
# SETINGS: RETRY_TIMES < meta(max_retry_times)
|
||||
self.mw.max_retry_times = 2
|
||||
meta_max_retry_times = 3
|
||||
middleware_max_retry_times = 2
|
||||
|
||||
req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
|
||||
req2 = Request(self.invalid_url)
|
||||
|
||||
self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
|
||||
self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
|
||||
settings = {'RETRY_TIMES': middleware_max_retry_times}
|
||||
spider, middleware = self.get_spider_and_middleware(settings)
|
||||
|
||||
self._test_retry(
|
||||
req1,
|
||||
DNSLookupError('foo'),
|
||||
meta_max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
self._test_retry(
|
||||
req2,
|
||||
DNSLookupError('foo'),
|
||||
middleware_max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
|
||||
def test_with_metakey_lesser(self):
|
||||
|
||||
# SETINGS: RETRY_TIMES > meta(max_retry_times)
|
||||
self.mw.max_retry_times = 5
|
||||
meta_max_retry_times = 4
|
||||
middleware_max_retry_times = 5
|
||||
|
||||
req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
|
||||
req2 = Request(self.invalid_url)
|
||||
|
||||
self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
|
||||
self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
|
||||
settings = {'RETRY_TIMES': middleware_max_retry_times}
|
||||
spider, middleware = self.get_spider_and_middleware(settings)
|
||||
|
||||
self._test_retry(
|
||||
req1,
|
||||
DNSLookupError('foo'),
|
||||
meta_max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
self._test_retry(
|
||||
req2,
|
||||
DNSLookupError('foo'),
|
||||
middleware_max_retry_times,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
|
||||
def test_with_dont_retry(self):
|
||||
max_retry_times = 4
|
||||
spider, middleware = self.get_spider_and_middleware()
|
||||
meta = {
|
||||
'max_retry_times': max_retry_times,
|
||||
'dont_retry': True,
|
||||
}
|
||||
req = Request(self.invalid_url, meta=meta)
|
||||
self._test_retry(
|
||||
req,
|
||||
DNSLookupError('foo'),
|
||||
0,
|
||||
spider=spider,
|
||||
middleware=middleware,
|
||||
)
|
||||
|
||||
# SETTINGS: meta(max_retry_times) = 4
|
||||
meta_max_retry_times = 4
|
||||
|
||||
req = Request(self.invalid_url, meta={
|
||||
'max_retry_times': meta_max_retry_times, 'dont_retry': True
|
||||
})
|
||||
|
||||
self._test_retry(req, DNSLookupError('foo'), 0)
|
||||
|
||||
def _test_retry(self, req, exception, max_retry_times):
|
||||
def _test_retry(
|
||||
self,
|
||||
req,
|
||||
exception,
|
||||
max_retry_times,
|
||||
spider=None,
|
||||
middleware=None,
|
||||
):
|
||||
spider = spider or self.spider
|
||||
middleware = middleware or self.mw
|
||||
|
||||
for i in range(0, max_retry_times):
|
||||
req = self.mw.process_exception(req, exception, self.spider)
|
||||
req = middleware.process_exception(req, exception, spider)
|
||||
assert isinstance(req, Request)
|
||||
|
||||
# discard it
|
||||
req = self.mw.process_exception(req, exception, self.spider)
|
||||
req = middleware.process_exception(req, exception, spider)
|
||||
self.assertEqual(req, None)
|
||||
|
||||
|
||||
class GetRetryRequestTest(unittest.TestCase):
|
||||
|
||||
def get_spider(self, settings=None):
|
||||
crawler = get_crawler(Spider, settings or {})
|
||||
return crawler._create_spider('foo')
|
||||
|
||||
def test_basic_usage(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
with LogCapture() as log:
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
)
|
||||
self.assertIsInstance(new_request, Request)
|
||||
self.assertNotEqual(new_request, request)
|
||||
self.assertEqual(new_request.dont_filter, True)
|
||||
expected_retry_times = 1
|
||||
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
|
||||
self.assertEqual(new_request.priority, -1)
|
||||
expected_reason = "unspecified"
|
||||
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
|
||||
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_max_retries_reached(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
max_retry_times = 0
|
||||
with LogCapture() as log:
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
max_retry_times=max_retry_times,
|
||||
)
|
||||
self.assertEqual(new_request, None)
|
||||
self.assertEqual(
|
||||
spider.crawler.stats.get_value('retry/max_reached'),
|
||||
1
|
||||
)
|
||||
failure_count = max_retry_times + 1
|
||||
expected_reason = "unspecified"
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"ERROR",
|
||||
f"Gave up retrying {request} (failed {failure_count} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_one_retry(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
with LogCapture() as log:
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
max_retry_times=1,
|
||||
)
|
||||
self.assertIsInstance(new_request, Request)
|
||||
self.assertNotEqual(new_request, request)
|
||||
self.assertEqual(new_request.dont_filter, True)
|
||||
expected_retry_times = 1
|
||||
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
|
||||
self.assertEqual(new_request.priority, -1)
|
||||
expected_reason = "unspecified"
|
||||
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
|
||||
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_two_retries(self):
|
||||
spider = self.get_spider()
|
||||
request = Request('https://example.com')
|
||||
new_request = request
|
||||
max_retry_times = 2
|
||||
for index in range(max_retry_times):
|
||||
with LogCapture() as log:
|
||||
new_request = get_retry_request(
|
||||
new_request,
|
||||
spider=spider,
|
||||
max_retry_times=max_retry_times,
|
||||
)
|
||||
self.assertIsInstance(new_request, Request)
|
||||
self.assertNotEqual(new_request, request)
|
||||
self.assertEqual(new_request.dont_filter, True)
|
||||
expected_retry_times = index + 1
|
||||
self.assertEqual(new_request.meta['retry_times'], expected_retry_times)
|
||||
self.assertEqual(new_request.priority, -expected_retry_times)
|
||||
expected_reason = "unspecified"
|
||||
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
|
||||
value = spider.crawler.stats.get_value(stat)
|
||||
self.assertEqual(value, expected_retry_times)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
with LogCapture() as log:
|
||||
new_request = get_retry_request(
|
||||
new_request,
|
||||
spider=spider,
|
||||
max_retry_times=max_retry_times,
|
||||
)
|
||||
self.assertEqual(new_request, None)
|
||||
self.assertEqual(
|
||||
spider.crawler.stats.get_value('retry/max_reached'),
|
||||
1
|
||||
)
|
||||
failure_count = max_retry_times + 1
|
||||
expected_reason = "unspecified"
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"ERROR",
|
||||
f"Gave up retrying {request} (failed {failure_count} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_no_spider(self):
|
||||
request = Request('https://example.com')
|
||||
with self.assertRaises(TypeError):
|
||||
get_retry_request(request) # pylint: disable=missing-kwoa
|
||||
|
||||
def test_max_retry_times_setting(self):
|
||||
max_retry_times = 0
|
||||
spider = self.get_spider({'RETRY_TIMES': max_retry_times})
|
||||
request = Request('https://example.com')
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
)
|
||||
self.assertEqual(new_request, None)
|
||||
|
||||
def test_max_retry_times_meta(self):
|
||||
max_retry_times = 0
|
||||
spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
|
||||
meta = {'max_retry_times': max_retry_times}
|
||||
request = Request('https://example.com', meta=meta)
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
)
|
||||
self.assertEqual(new_request, None)
|
||||
|
||||
def test_max_retry_times_argument(self):
|
||||
max_retry_times = 0
|
||||
spider = self.get_spider({'RETRY_TIMES': max_retry_times + 1})
|
||||
meta = {'max_retry_times': max_retry_times + 1}
|
||||
request = Request('https://example.com', meta=meta)
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
max_retry_times=max_retry_times,
|
||||
)
|
||||
self.assertEqual(new_request, None)
|
||||
|
||||
def test_priority_adjust_setting(self):
|
||||
priority_adjust = 1
|
||||
spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust})
|
||||
request = Request('https://example.com')
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
)
|
||||
self.assertEqual(new_request.priority, priority_adjust)
|
||||
|
||||
def test_priority_adjust_argument(self):
|
||||
priority_adjust = 1
|
||||
spider = self.get_spider({'RETRY_PRIORITY_ADJUST': priority_adjust + 1})
|
||||
request = Request('https://example.com')
|
||||
new_request = get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
priority_adjust=priority_adjust,
|
||||
)
|
||||
self.assertEqual(new_request.priority, priority_adjust)
|
||||
|
||||
def test_log_extra_retry_success(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
with LogCapture(attributes=('spider',)) as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
)
|
||||
log.check_present(spider)
|
||||
|
||||
def test_log_extra_retries_exceeded(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
with LogCapture(attributes=('spider',)) as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
max_retry_times=0,
|
||||
)
|
||||
log.check_present(spider)
|
||||
|
||||
def test_reason_string(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
expected_reason = 'because'
|
||||
with LogCapture() as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
)
|
||||
expected_retry_times = 1
|
||||
for stat in ('retry/count', f'retry/reason_count/{expected_reason}'):
|
||||
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_reason_builtin_exception(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
expected_reason = NotImplementedError()
|
||||
expected_reason_string = 'builtins.NotImplementedError'
|
||||
with LogCapture() as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
)
|
||||
expected_retry_times = 1
|
||||
stat = spider.crawler.stats.get_value(
|
||||
f'retry/reason_count/{expected_reason_string}'
|
||||
)
|
||||
self.assertEqual(stat, 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_reason_builtin_exception_class(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
expected_reason = NotImplementedError
|
||||
expected_reason_string = 'builtins.NotImplementedError'
|
||||
with LogCapture() as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
)
|
||||
expected_retry_times = 1
|
||||
stat = spider.crawler.stats.get_value(
|
||||
f'retry/reason_count/{expected_reason_string}'
|
||||
)
|
||||
self.assertEqual(stat, 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_reason_custom_exception(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
expected_reason = IgnoreRequest()
|
||||
expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
|
||||
with LogCapture() as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
)
|
||||
expected_retry_times = 1
|
||||
stat = spider.crawler.stats.get_value(
|
||||
f'retry/reason_count/{expected_reason_string}'
|
||||
)
|
||||
self.assertEqual(stat, 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_reason_custom_exception_class(self):
|
||||
request = Request('https://example.com')
|
||||
spider = self.get_spider()
|
||||
expected_reason = IgnoreRequest
|
||||
expected_reason_string = 'scrapy.exceptions.IgnoreRequest'
|
||||
with LogCapture() as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
)
|
||||
expected_retry_times = 1
|
||||
stat = spider.crawler.stats.get_value(
|
||||
f'retry/reason_count/{expected_reason_string}'
|
||||
)
|
||||
self.assertEqual(stat, 1)
|
||||
log.check_present(
|
||||
(
|
||||
"scrapy.downloadermiddlewares.retry",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed {expected_retry_times} times): "
|
||||
f"{expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_custom_logger(self):
|
||||
logger = logging.getLogger("custom-logger")
|
||||
request = Request("https://example.com")
|
||||
spider = self.get_spider()
|
||||
expected_reason = "because"
|
||||
with LogCapture() as log:
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
logger=logger,
|
||||
)
|
||||
log.check_present(
|
||||
(
|
||||
"custom-logger",
|
||||
"DEBUG",
|
||||
f"Retrying {request} (failed 1 times): {expected_reason}",
|
||||
)
|
||||
)
|
||||
|
||||
def test_custom_stats_key(self):
|
||||
request = Request("https://example.com")
|
||||
spider = self.get_spider()
|
||||
expected_reason = "because"
|
||||
stats_key = "custom_retry"
|
||||
get_retry_request(
|
||||
request,
|
||||
spider=spider,
|
||||
reason=expected_reason,
|
||||
stats_base_key=stats_key,
|
||||
)
|
||||
for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"):
|
||||
self.assertEqual(spider.crawler.stats.get_value(stat), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue