From 3a4239fbc8ef9ca01a7c22b0a6e5e60670a1af62 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Sun, 21 Nov 2021 15:49:45 +0500
Subject: [PATCH 1/3] add flags into retry message.

---
 scrapy/downloadermiddlewares/retry.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index c6cc7c56d..5564d7613 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -87,6 +87,7 @@ def get_retry_request(
     """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
+    request_flags = f' {request.flags if request.flags else ""}'
     retry_times = request.meta.get('retry_times', 0) + 1
     if max_retry_times is None:
         max_retry_times = request.meta.get('max_retry_times')
@@ -94,8 +95,8 @@ def get_retry_request(
             max_retry_times = settings.getint('RETRY_TIMES')
     if retry_times <= max_retry_times:
         logger.debug(
-            "Retrying %(request)s (failed %(retry_times)d times): %(reason)s",
-            {'request': request, 'retry_times': retry_times, 'reason': reason},
+            "Retrying %(request)s%(flags)s (failed %(retry_times)d times): %(reason)s",
+            {'request': request, 'flags': request_flags, 'retry_times': retry_times, 'reason': reason},
             extra={'spider': spider}
         )
         new_request: Request = request.copy()
@@ -116,9 +117,9 @@ def get_retry_request(
     else:
         stats.inc_value(f'{stats_base_key}/max_reached')
         logger.error(
-            "Gave up retrying %(request)s (failed %(retry_times)d times): "
+            "Gave up retrying %(request)s%(flags)s (failed %(retry_times)d times): "
             "%(reason)s",
-            {'request': request, 'retry_times': retry_times, 'reason': reason},
+            {'request': request, 'flags': request_flags, 'retry_times': retry_times, 'reason': reason},
             extra={'spider': spider},
         )
         return None

From 04d5cbc4ce96ccce7506f815da49ce3da1b631c4 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Sun, 21 Nov 2021 15:59:34 +0500
Subject: [PATCH 2/3] Fix blank symbol.

---
 scrapy/downloadermiddlewares/retry.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 5564d7613..2fc331f1e 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -87,7 +87,7 @@ def get_retry_request(
     """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
-    request_flags = f' {request.flags if request.flags else ""}'
+    request_flags = f' {request.flags}' if request.flags else ""
     retry_times = request.meta.get('retry_times', 0) + 1
     if max_retry_times is None:
         max_retry_times = request.meta.get('max_retry_times')

From 677de46bcccc3fab25e0a981643a1883f4973eb3 Mon Sep 17 00:00:00 2001
From: bulat <xb.bolo@gmail.com>
Date: Sun, 21 Nov 2021 16:45:30 +0500
Subject: [PATCH 3/3] add flags tests.

---
 tests/test_downloadermiddleware_retry.py | 46 ++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 915bd3a3e..0afbd2ca8 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -629,6 +629,52 @@ class GetRetryRequestTest(unittest.TestCase):
         for stat in (f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}"):
             self.assertEqual(spider.crawler.stats.get_value(stat), 1)
 
+    def test_flags_present(self):
+        flags = ["test_flag1", "test_flag2"]
+        request = Request(
+            'https://example.com', flags=flags
+        )
+        spider = self.get_spider()
+        with LogCapture() as log:
+            get_retry_request(
+                request,
+                spider=spider,
+            )
+        expected_retry_times = 1
+        expected_reason = "unspecified"
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "DEBUG",
+                f"Retrying {request} {flags} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            )
+        )
+
+    def test_flags_max_retries_reached(self):
+        flags = ["test_flag1", "test_flag2"]
+        max_retry_times = 0
+        request = Request(
+            'https://example.com', flags=flags
+        )
+        spider = self.get_spider()
+        with LogCapture() as log:
+            get_retry_request(
+                request,
+                spider=spider,
+                max_retry_times=max_retry_times,
+            )
+        failure_count = max_retry_times + 1
+        expected_reason = "unspecified"
+        log.check_present(
+            (
+                "scrapy.downloadermiddlewares.retry",
+                "ERROR",
+                f"Gave up retrying {request} {flags} (failed {failure_count} times): "
+                f"{expected_reason}",
+            )
+        )
+
 
 if __name__ == "__main__":
     unittest.main()