Merge pull request #6050 from wRAR/retrymw-fix

Improve the backwards compatibility for RetryMiddleware.EXCEPTIONS_TO_RETRY

scrapy/downloadermiddlewares/retry.py

@@ -24,9 +24,8 @@ from scrapy.utils.response import response_status_message
 retry_logger = getLogger(__name__)
 
 
-class BackwardsCompatibilityMetaclass(type):
-    @property
-    def EXCEPTIONS_TO_RETRY(cls):
+def backwards_compatibility_getattr(self, name):
+    if name == "EXCEPTIONS_TO_RETRY":
         warnings.warn(
             "Attribute RetryMiddleware.EXCEPTIONS_TO_RETRY is deprecated. "
             "Use the RETRY_EXCEPTIONS setting instead.",
@@ -37,6 +36,13 @@ class BackwardsCompatibilityMetaclass(type):
             load_object(x) if isinstance(x, str) else x
             for x in Settings().getlist("RETRY_EXCEPTIONS")
         )
+    raise AttributeError(
+        f"{self.__class__.__name__!r} object has no attribute {name!r}"
+    )
+
+
+class BackwardsCompatibilityMetaclass(type):
+    __getattr__ = backwards_compatibility_getattr
 
 
 def get_retry_request(
@@ -137,15 +143,14 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
         )
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
-        if not hasattr(
-            self, "EXCEPTIONS_TO_RETRY"
-        ):  # If EXCEPTIONS_TO_RETRY is not "overriden"
+        try:
+            self.exceptions_to_retry = self.__getattribute__("EXCEPTIONS_TO_RETRY")
+        except AttributeError:
+            # If EXCEPTIONS_TO_RETRY is not "overridden"
             self.exceptions_to_retry = tuple(
                 load_object(x) if isinstance(x, str) else x
                 for x in settings.getlist("RETRY_EXCEPTIONS")
             )
-        else:
-            self.exceptions_to_retry = self.EXCEPTIONS_TO_RETRY
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -175,3 +180,5 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
             max_retry_times=max_retry_times,
             priority_adjust=priority_adjust,
         )
+
+    __getattr__ = backwards_compatibility_getattr

tests/test_downloadermiddleware_retry.py

@@ -122,7 +122,7 @@ class RetryTest(unittest.TestCase):
         req = Request(f"http://www.scrapytest.org/{exc.__name__}")
         self._test_retry_exception(req, exc("foo"), mw)
 
-    def test_exception_to_retry_customMiddleware(self):
+    def test_exception_to_retry_custom_middleware(self):
         exc = ValueError
 
         with warnings.catch_warnings(record=True) as warns:
@@ -138,6 +138,21 @@ class RetryTest(unittest.TestCase):
         assert isinstance(req, Request)
         self.assertEqual(req.meta["retry_times"], 1)
 
+    def test_exception_to_retry_custom_middleware_self(self):
+        class MyRetryMiddleware(RetryMiddleware):
+            def process_exception(self, request, exception, spider):
+                if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
+                    return self._retry(request, exception, spider)
+
+        exc = OSError
+        mw2 = MyRetryMiddleware.from_crawler(self.crawler)
+        req = Request(f"http://www.scrapytest.org/{exc.__name__}")
+        with warnings.catch_warnings(record=True) as warns:
+            req = mw2.process_exception(req, exc("foo"), self.spider)
+        assert isinstance(req, Request)
+        self.assertEqual(req.meta["retry_times"], 1)
+        self.assertEqual(len(warns), 1)
+
     def _test_retry_exception(self, req, exception, mw=None):
         if mw is None:
             mw = self.mw