Remove custom redirection priority of request returned by downloadermiddleware

docs/ref/settings.rst

@@ -699,6 +699,14 @@ Default: ``100``
 Some sites use meta-refresh for redirecting to a session expired page, so we
 restrict automatic redirection to a maximum delay (in seconds)
 
+REDIRECT_PRIORITY_ADJUST
+------------------------------
+
+Default: ``-2``
+
+Adjust redirect request priority relative to original request.
+A negative priority adjust means more priority.
+
 .. setting:: REQUESTS_QUEUE_SIZE
 
 REQUESTS_PER_DOMAIN

scrapy/conf/default_settings.py

@@ -141,6 +141,7 @@ NEWSPIDER_MODULE = ''
 
 REDIRECT_MAX_METAREFRESH_DELAY = 100
 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
+REDIRECT_PRIORITY_ADJUST = -2
 
 REQUESTS_QUEUE_SIZE = 0
 REQUESTS_PER_DOMAIN = 8     # max simultaneous requests per domain
@@ -148,6 +149,7 @@ REQUESTS_PER_DOMAIN = 8     # max simultaneous requests per domain
 # contrib.middleware.retry.RetryMiddleware default settings
 RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = ['500', '503', '504', '400', '408']
+RETRY_PRIORITY_ADJUST = +1
 
 ROBOTSTXT_OBEY = False
 

scrapy/contrib/downloadermiddleware/redirect.py

@@ -10,6 +10,7 @@ class RedirectMiddleware(object):
     def __init__(self):
         self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY')
         self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
+        self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
 
     def process_response(self, request, response, spider):
         domain = spider.domain_name
@@ -41,6 +42,7 @@ class RedirectMiddleware(object):
             redirected.meta['redirect_times'] = redirects
             redirected.meta['redirect_ttl'] = ttl - 1
             redirected.dont_filter = request.dont_filter
+            redirected.priority = request.priority + self.priority_adjust
             log.msg("Redirecting (%s) to %s from %s" % (reason, redirected, request),
                     domain=spider.domain_name, level=log.DEBUG)
             return redirected

scrapy/contrib/downloadermiddleware/retry.py

@@ -41,6 +41,7 @@ class RetryMiddleware(object):
     def __init__(self):
         self.max_retry_times = settings.getint('RETRY_TIMES')
         self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES'))
+        self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
 
     def process_response(self, request, response, spider):
         if response.status in self.retry_http_codes:
@@ -61,6 +62,7 @@ class RetryMiddleware(object):
             retryreq = request.copy()
             retryreq.meta['retry_times'] = retries
             retryreq.dont_filter = True
+            retryreq.priority = request.priority + self.priority_adjust
             return retryreq
         else:
             log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason),

scrapy/core/engine.py

@@ -44,10 +44,6 @@ class ExecutionEngine(object):
     process for that domain.
     """
 
-    # Scheduler priority of redirected requests. Negative means high priority.
-    # We use high priority to avoid hogging memory with pending redirected requests
-    REDIRECTION_PRIORITY = -10
-
     def __init__(self):
         self.configured = False
         self.keep_alive = False
@@ -323,10 +319,9 @@ class ExecutionEngine(object):
                 log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain)
                 return response
             elif isinstance(response, Request):
-                redirected = response # proper alias
-                redirected.priority = self.REDIRECTION_PRIORITY
-                schd = self.schedule(redirected, spider)
-                chain_deferred(schd, redirected.deferred)
+                newrequest = response # proper alias
+                schd = self.schedule(newrequest, spider)
+                chain_deferred(schd, newrequest.deferred)
                 return schd
 
         def _on_error(_failure):

scrapy/tests/test_downloadermiddleware_redirect.py

@@ -12,6 +12,12 @@ class RedirectMiddlewareTest(unittest.TestCase):
         self.spider = spiders.fromdomain('scrapytest.org')
         self.mw = RedirectMiddleware()
 
+    def test_priority_adjust(self):
+        req = Request('http://a.com')
+        rsp = Response('http://a.com', headers={'Location': 'http://a.com/redirected'}, status=301)
+        req2 = self.mw.process_response(req, rsp, self.spider)
+        assert req2.priority < req.priority
+
     def test_redirect_301(self):
         url = 'http://www.example.com/301'
         url2 = 'http://www.example.com/redirected'

scrapy/tests/test_downloadermiddleware_retry.py

@@ -16,6 +16,12 @@ class RetryTest(unittest.TestCase):
         self.mw = RetryMiddleware()
         self.mw.max_retry_times = 2
 
+    def test_priority_adjust(self):
+        req = Request('http://www.scrapytest.org/503')
+        rsp = Response('http://www.scrapytest.org/503', body='', status=503)
+        req2 = self.mw.process_response(req, rsp, self.spider)
+        assert req2.priority > req.priority
+
     def test_404(self):
         req = Request('http://www.scrapytest.org/404')
         rsp = Response('http://www.scrapytest.org/404', body='', status=404)