diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index 10f923ab8..e327bb766 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -699,6 +699,14 @@ Default: ``100`` Some sites use meta-refresh for redirecting to a session expired page, so we restrict automatic redirection to a maximum delay (in seconds) +REDIRECT_PRIORITY_ADJUST +------------------------------ + +Default: ``-2`` + +Adjust redirect request priority relative to original request. +A negative priority adjust means more priority. + .. setting:: REQUESTS_QUEUE_SIZE REQUESTS_PER_DOMAIN diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 0741ceb0b..4c4a825ce 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -141,6 +141,7 @@ NEWSPIDER_MODULE = '' REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting +REDIRECT_PRIORITY_ADJUST = -2 REQUESTS_QUEUE_SIZE = 0 REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain @@ -148,6 +149,7 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain # contrib.middleware.retry.RetryMiddleware default settings RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408'] +RETRY_PRIORITY_ADJUST = +1 ROBOTSTXT_OBEY = False diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 02ce77ec8..b5af78324 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -10,6 +10,7 @@ class RedirectMiddleware(object): def __init__(self): self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY') self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES') + self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST') def process_response(self, request, response, spider): domain = spider.domain_name @@ -41,6 +42,7 @@ class RedirectMiddleware(object): redirected.meta['redirect_times'] = redirects redirected.meta['redirect_ttl'] = ttl - 1 redirected.dont_filter = request.dont_filter + redirected.priority = request.priority + self.priority_adjust log.msg("Redirecting (%s) to %s from %s" % (reason, redirected, request), domain=spider.domain_name, level=log.DEBUG) return redirected diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index a69f77782..e8f1a1735 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -41,6 +41,7 @@ class RetryMiddleware(object): def __init__(self): self.max_retry_times = settings.getint('RETRY_TIMES') self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES')) + self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') def process_response(self, request, response, spider): if response.status in self.retry_http_codes: @@ -61,6 +62,7 @@ class RetryMiddleware(object): retryreq = request.copy() retryreq.meta['retry_times'] = retries retryreq.dont_filter = True + retryreq.priority = request.priority + self.priority_adjust return retryreq else: log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason), diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index e7dbe3db2..25284dea9 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -44,10 +44,6 @@ class ExecutionEngine(object): process for that domain. """ - # Scheduler priority of redirected requests. Negative means high priority. - # We use high priority to avoid hogging memory with pending redirected requests - REDIRECTION_PRIORITY = -10 - def __init__(self): self.configured = False self.keep_alive = False @@ -323,10 +319,9 @@ class ExecutionEngine(object): log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain) return response elif isinstance(response, Request): - redirected = response # proper alias - redirected.priority = self.REDIRECTION_PRIORITY - schd = self.schedule(redirected, spider) - chain_deferred(schd, redirected.deferred) + newrequest = response # proper alias + schd = self.schedule(newrequest, spider) + chain_deferred(schd, newrequest.deferred) return schd def _on_error(_failure): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index 400c1feeb..7572a21c0 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -12,6 +12,12 @@ class RedirectMiddlewareTest(unittest.TestCase): self.spider = spiders.fromdomain('scrapytest.org') self.mw = RedirectMiddleware() + def test_priority_adjust(self): + req = Request('http://a.com') + rsp = Response('http://a.com', headers={'Location': 'http://a.com/redirected'}, status=301) + req2 = self.mw.process_response(req, rsp, self.spider) + assert req2.priority < req.priority + def test_redirect_301(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index d646ffab5..ef8f249b8 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -16,6 +16,12 @@ class RetryTest(unittest.TestCase): self.mw = RetryMiddleware() self.mw.max_retry_times = 2 + def test_priority_adjust(self): + req = Request('http://www.scrapytest.org/503') + rsp = Response('http://www.scrapytest.org/503', body='', status=503) + req2 = self.mw.process_response(req, rsp, self.spider) + assert req2.priority > req.priority + def test_404(self): req = Request('http://www.scrapytest.org/404') rsp = Response('http://www.scrapytest.org/404', body='', status=404)