diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index dd2897ee4..feff15453 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -95,6 +95,7 @@ def get_retry_request( """ settings = spider.crawler.settings stats = spider.crawler.stats + flags = f" {request.flags}" if request.flags else "" retry_times = request.meta.get("retry_times", 0) + 1 if max_retry_times is None: max_retry_times = request.meta.get("max_retry_times") @@ -102,8 +103,7 @@ def get_retry_request( max_retry_times = settings.getint("RETRY_TIMES") if retry_times <= max_retry_times: logger.debug( - "Retrying %(request)s (failed %(retry_times)d times): %(reason)s", - {"request": request, "retry_times": retry_times, "reason": reason}, + f"Retrying {request}{flags} (failed {retry_times} times): {reason}", extra={"spider": spider}, ) new_request: Request = request.copy() @@ -131,8 +131,7 @@ def get_retry_request( stats.inc_value(f"{stats_base_key}/max_reached") logger.log( give_up_log_level, - "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", - {"request": request, "retry_times": retry_times, "reason": reason}, + f"Gave up retrying {request}{flags} (failed {retry_times} times): {reason}", extra={"spider": spider}, ) return None diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 1c8a61aa5..17c71b9fd 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -743,3 +743,27 @@ class TestGetRetryRequest: f"{stats_key}/reason_count/{expected_reason}", ): assert spider.crawler.stats.get_value(stat) == 1 + + def test_flags(self, caplog: pytest.LogCaptureFixture) -> None: + flags = ["foo", "bar"] + request = Request("https://example.com", flags=flags) + spider = self.get_spider() + with caplog.at_level(logging.DEBUG): + get_retry_request(request, spider=spider) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.DEBUG, + f"Retrying {request} {flags} (failed 1 times): unspecified", + ) in caplog.record_tuples + + def test_flags_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None: + flags = ["foo", "bar"] + request = Request("https://example.com", flags=flags) + spider = self.get_spider() + with caplog.at_level(logging.DEBUG): + get_retry_request(request, spider=spider, max_retry_times=0) + assert ( + "scrapy.downloadermiddlewares.retry", + logging.ERROR, + f"Gave up retrying {request} {flags} (failed 1 times): unspecified", + ) in caplog.record_tuples