From 47970e91bc5a370ca58f1a1ee951ed100e0b864f Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Sat, 20 Jun 2009 19:23:26 -0300 Subject: [PATCH] core: Invert request priority meaning, a higher request.priority value means more priority --- docs/ref/settings.rst | 2 +- scrapy/conf/default_settings.py | 4 ++-- scrapy/core/scheduler/schedulers.py | 2 +- scrapy/tests/test_downloadermiddleware_redirect.py | 2 +- scrapy/tests/test_downloadermiddleware_retry.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index e327bb766..60bea9154 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -702,7 +702,7 @@ restrict automatic redirection to a maximum delay (in seconds) REDIRECT_PRIORITY_ADJUST ------------------------------ -Default: ``-2`` +Default: ``+2`` Adjust redirect request priority relative to original request. A negative priority adjust means more priority. diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 4c4a825ce..9a1420f10 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -141,7 +141,7 @@ NEWSPIDER_MODULE = '' REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting -REDIRECT_PRIORITY_ADJUST = -2 +REDIRECT_PRIORITY_ADJUST = +2 REQUESTS_QUEUE_SIZE = 0 REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain @@ -149,7 +149,7 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain # contrib.middleware.retry.RetryMiddleware default settings RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408'] -RETRY_PRIORITY_ADJUST = +1 +RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False diff --git a/scrapy/core/scheduler/schedulers.py b/scrapy/core/scheduler/schedulers.py index 8facf88e6..95a2f605f 100644 --- a/scrapy/core/scheduler/schedulers.py +++ b/scrapy/core/scheduler/schedulers.py @@ -54,7 +54,7 @@ class Scheduler(object): def _enqueue_request(self, domain, request): dfd = defer.Deferred() - self.pending_requests[domain].push((request, dfd), request.priority) + self.pending_requests[domain].push((request, dfd), -request.priority) return dfd def next_request(self, domain): diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index 7572a21c0..0db49e485 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -16,7 +16,7 @@ class RedirectMiddlewareTest(unittest.TestCase): req = Request('http://a.com') rsp = Response('http://a.com', headers={'Location': 'http://a.com/redirected'}, status=301) req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority < req.priority + assert req2.priority > req.priority def test_redirect_301(self): url = 'http://www.example.com/301' diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index ef8f249b8..d0f6f273f 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -20,7 +20,7 @@ class RetryTest(unittest.TestCase): req = Request('http://www.scrapytest.org/503') rsp = Response('http://www.scrapytest.org/503', body='', status=503) req2 = self.mw.process_response(req, rsp, self.spider) - assert req2.priority > req.priority + assert req2.priority < req.priority def test_404(self): req = Request('http://www.scrapytest.org/404')