mirror of https://github.com/scrapy/scrapy.git
core: Invert request priority meaning, a higher request.priority value means more priority
This commit is contained in:
parent
18b6fecc47
commit
47970e91bc
|
|
@ -702,7 +702,7 @@ restrict automatic redirection to a maximum delay (in seconds)
|
|||
REDIRECT_PRIORITY_ADJUST
|
||||
------------------------------
|
||||
|
||||
Default: ``-2``
|
||||
Default: ``+2``
|
||||
|
||||
Adjust redirect request priority relative to original request.
|
||||
A negative priority adjust means more priority.
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ NEWSPIDER_MODULE = ''
|
|||
|
||||
REDIRECT_MAX_METAREFRESH_DELAY = 100
|
||||
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
||||
REDIRECT_PRIORITY_ADJUST = -2
|
||||
REDIRECT_PRIORITY_ADJUST = +2
|
||||
|
||||
REQUESTS_QUEUE_SIZE = 0
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
|
|
@ -149,7 +149,7 @@ REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
|||
# contrib.middleware.retry.RetryMiddleware default settings
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408']
|
||||
RETRY_PRIORITY_ADJUST = +1
|
||||
RETRY_PRIORITY_ADJUST = -1
|
||||
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class Scheduler(object):
|
|||
|
||||
def _enqueue_request(self, domain, request):
|
||||
dfd = defer.Deferred()
|
||||
self.pending_requests[domain].push((request, dfd), request.priority)
|
||||
self.pending_requests[domain].push((request, dfd), -request.priority)
|
||||
return dfd
|
||||
|
||||
def next_request(self, domain):
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
req = Request('http://a.com')
|
||||
rsp = Response('http://a.com', headers={'Location': 'http://a.com/redirected'}, status=301)
|
||||
req2 = self.mw.process_response(req, rsp, self.spider)
|
||||
assert req2.priority < req.priority
|
||||
assert req2.priority > req.priority
|
||||
|
||||
def test_redirect_301(self):
|
||||
url = 'http://www.example.com/301'
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class RetryTest(unittest.TestCase):
|
|||
req = Request('http://www.scrapytest.org/503')
|
||||
rsp = Response('http://www.scrapytest.org/503', body='', status=503)
|
||||
req2 = self.mw.process_response(req, rsp, self.spider)
|
||||
assert req2.priority > req.priority
|
||||
assert req2.priority < req.priority
|
||||
|
||||
def test_404(self):
|
||||
req = Request('http://www.scrapytest.org/404')
|
||||
|
|
|
|||
Loading…
Reference in New Issue