RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add flags into retry message.

This commit is contained in:
bulat 2021-11-21 15:49:45 +05:00
parent c316ca45a5
commit 3a4239fbc8
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/downloadermiddlewares/retry.py
View File

@ -87,6 +87,7 @@ def get_retry_request(
"""
settings = spider.crawler.settings
stats = spider.crawler.stats
request_flags = f' {request.flags if request.flags else ""}'
retry_times = request.meta.get('retry_times', 0) + 1
if max_retry_times is None:
max_retry_times = request.meta.get('max_retry_times')
@ -94,8 +95,8 @@ def get_retry_request(
max_retry_times = settings.getint('RETRY_TIMES')
if retry_times <= max_retry_times:
logger.debug(
"Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
"Retrying %(request)s%(flags)s (failed %(retry_times)d times): %(reason)s",
{'request': request, 'flags': request_flags, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider}
)
new_request: Request = request.copy()
@ -116,9 +117,9 @@ def get_retry_request(
else:
stats.inc_value(f'{stats_base_key}/max_reached')
logger.error(
"Gave up retrying %(request)s (failed %(retry_times)d times): "
"Gave up retrying %(request)s%(flags)s (failed %(retry_times)d times): "
"%(reason)s",
{'request': request, 'retry_times': retry_times, 'reason': reason},
{'request': request, 'flags': request_flags, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider},
)
return None