Merge 3b3c9299df into e28e56aa61

scrapy/downloadermiddlewares/retry.py

@@ -95,6 +95,7 @@ def get_retry_request(
     """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
+    flags = f" {request.flags}" if request.flags else ""
     retry_times = request.meta.get("retry_times", 0) + 1
     if max_retry_times is None:
         max_retry_times = request.meta.get("max_retry_times")
@@ -102,8 +103,7 @@ def get_retry_request(
             max_retry_times = settings.getint("RETRY_TIMES")
     if retry_times <= max_retry_times:
         logger.debug(
-            "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
-            {"request": request, "retry_times": retry_times, "reason": reason},
+            f"Retrying {request}{flags} (failed {retry_times} times): {reason}",
             extra={"spider": spider},
         )
         new_request: Request = request.copy()
@@ -131,8 +131,7 @@ def get_retry_request(
     stats.inc_value(f"{stats_base_key}/max_reached")
     logger.log(
         give_up_log_level,
-        "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
-        {"request": request, "retry_times": retry_times, "reason": reason},
+        f"Gave up retrying {request}{flags} (failed {retry_times} times): {reason}",
         extra={"spider": spider},
     )
     return None

tests/test_downloadermiddleware_retry.py

@@ -743,3 +743,27 @@ class TestGetRetryRequest:
             f"{stats_key}/reason_count/{expected_reason}",
         ):
             assert spider.crawler.stats.get_value(stat) == 1
+
+    def test_flags(self, caplog: pytest.LogCaptureFixture) -> None:
+        flags = ["foo", "bar"]
+        request = Request("https://example.com", flags=flags)
+        spider = self.get_spider()
+        with caplog.at_level(logging.DEBUG):
+            get_retry_request(request, spider=spider)
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} {flags} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
+
+    def test_flags_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None:
+        flags = ["foo", "bar"]
+        request = Request("https://example.com", flags=flags)
+        spider = self.get_spider()
+        with caplog.at_level(logging.DEBUG):
+            get_retry_request(request, spider=spider, max_retry_times=0)
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} {flags} (failed 1 times): unspecified",
+        ) in caplog.record_tuples