diff --git a/docs/ref/request-response.rst b/docs/ref/request-response.rst index 6631e3270..8150d6dee 100644 --- a/docs/ref/request-response.rst +++ b/docs/ref/request-response.rst @@ -25,7 +25,7 @@ below in :ref:`ref-request-subclasses` and :ref:`ref-response-subclasses`. Request objects =============== -.. class:: Request(url[, callback, method='GET', body, headers, cookies, meta, encoding='utf-8', dont_filter=False, errback]) +.. class:: Request(url[, callback, method='GET', body, headers, cookies, meta, encoding='utf-8', priority=0.0, dont_filter=False, errback]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -81,6 +81,12 @@ Request objects body to ``str`` (if given as ``unicode``). :type encoding: string + :param priority: the priority of this request (defaults to ``0.0``). + The priority is used by the scheduler to define the order used to return + requests. It can also be used to feed priorities externally, for + example, using an offline long-term scheduler. + :type encoding: int or float + :param dont_filter: indicates that this request should not be filtered by the scheduler. This is used when you want to perform an identical request multiple times, to ignore the duplicates filter. Use it with diff --git a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py index a6bf9a095..2753d4fae 100644 --- a/scrapy/contrib/schedulermiddleware/duplicatesfilter.py +++ b/scrapy/contrib/schedulermiddleware/duplicatesfilter.py @@ -9,7 +9,7 @@ from scrapy.dupefilter import dupefilter class DuplicatesFilterMiddleware(object): """Filter out already seen requests to avoid visiting pages more than once.""" - def enqueue_request(self, domain, request, priority): + def enqueue_request(self, domain, request): added = dupefilter.add(domain, request) if not (added or request.dont_filter): raise IgnoreRequest('Skipped (already seen request)') diff --git a/scrapy/contrib_exp/history/scheduler.py b/scrapy/contrib_exp/history/scheduler.py index 1f4152675..56a858349 100644 --- a/scrapy/contrib_exp/history/scheduler.py +++ b/scrapy/contrib_exp/history/scheduler.py @@ -33,7 +33,7 @@ class RulesScheduler(Scheduler): # How often we should process pages that have not changed (need to include depth) MIN_PROCESS_UNCHANGED_DAYS = 12 - def enqueue_request(self, domain, request, priority=1): + def enqueue_request(self, domain, request): """Add a page to be scraped for a domain that is currently being scraped. The url will only be added if we have not checked it already within @@ -55,7 +55,7 @@ class RulesScheduler(Scheduler): return # put the version in the pending pages to avoid querying DB again record = (request, version, now) - self.pending_requests[domain].put(record, priority) + self.pending_requests[domain].push(record, request.priority) def next_request(self, domain): """Get the next page from the superclass. This will add a callback diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 78fc246c6..565603878 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -219,7 +219,7 @@ class ExecutionEngine(object): def open_domains(self): return self.downloader.sites.keys() - def crawl(self, request, spider, priority=0, domain_priority=0): + def crawl(self, request, spider, domain_priority=0): domain = spider.domain_name def _process(response): @@ -242,7 +242,7 @@ class ExecutionEngine(object): piped.addBoth(_onpipelinefinish, item) elif isinstance(item, Request): signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response) - self.crawl(request=item, spider=spider, priority=priority) + self.crawl(request=item, spider=spider) elif item is None: pass # may be next time. else: @@ -276,14 +276,14 @@ class ExecutionEngine(object): request.deferred.addErrback(lambda _:None) request.deferred.errback(_failure) #TODO: merge into spider middleware. - schd = self.schedule(request, spider, priority, domain_priority) + schd = self.schedule(request, spider, domain_priority) schd.addCallbacks(_process, _cleanfailure) return schd def scrape(self, request, response, spider): return self.spidermiddleware.scrape(request, response, spider) - def schedule(self, request, spider, priority=0, domain_priority=0): + def schedule(self, request, spider, domain_priority=0): domain = spider.domain_name if not self.scheduler.domain_is_open(domain): if self.debug_mode: @@ -291,7 +291,7 @@ class ExecutionEngine(object): return self._add_starter(request, spider, domain_priority) if self.debug_mode: log.msg('Scheduling %s (now)' % request_info(request), log.DEBUG) - schd = self.schedulermiddleware.enqueue_request(domain, request, priority) + schd = self.schedulermiddleware.enqueue_request(domain, request) self.next_request(spider) return schd @@ -343,7 +343,8 @@ class ExecutionEngine(object): return response elif isinstance(response, Request): redirected = response # proper alias - schd = self.schedule(redirected, spider, priority=self.REDIRECTION_PRIORITY) + redirected.priority = self.REDIRECTION_PRIORITY + schd = self.schedule(redirected, spider) chain_deferred(schd, redirected.deferred) return schd diff --git a/scrapy/core/scheduler/middleware.py b/scrapy/core/scheduler/middleware.py index 68e7080e1..6972e859c 100644 --- a/scrapy/core/scheduler/middleware.py +++ b/scrapy/core/scheduler/middleware.py @@ -44,16 +44,16 @@ class SchedulerMiddlewareManager(object): log.msg("Enabled scheduler middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) self.loaded = True - def enqueue_request(self, domain, request, priority): + def enqueue_request(self, domain, request): def _enqueue_request(request): for method in self.mw_enqueue_request: - result = method(domain=domain, request=request, priority=priority) + result = method(domain=domain, request=request) assert result is None or isinstance(result, (Response, Deferred)), \ 'Middleware %s.enqueue_request must return None, Response or Deferred, got %s' % \ (method.im_self.__class__.__name__, result.__class__.__name__) if result: return result - return self.scheduler.enqueue_request(domain=domain, request=request, priority=priority) + return self.scheduler.enqueue_request(domain=domain, request=request) deferred = mustbe_deferred(_enqueue_request, request) return deferred diff --git a/scrapy/core/scheduler/schedulers.py b/scrapy/core/scheduler/schedulers.py index 55251b180..0fbd88e13 100644 --- a/scrapy/core/scheduler/schedulers.py +++ b/scrapy/core/scheduler/schedulers.py @@ -62,7 +62,7 @@ class Scheduler(object) : def next_domain(self) : """Return next domain available to scrape and remove it from available domains queue""" if self.pending_domains_count: - domain, priority = self.domains_queue.pop() + domain = self.domains_queue.pop()[0] if self.pending_domains_count[domain] == 1: del self.pending_domains_count[domain] else: @@ -90,10 +90,10 @@ class Scheduler(object) : Priority = PriorityStack if self.dfo else PriorityQueue self.pending_requests[domain] = Priority() - def enqueue_request(self, domain, request, priority=0): + def enqueue_request(self, domain, request): """Enqueue a request to be downloaded for a domain that is currently being scraped.""" dfd = defer.Deferred() - self.pending_requests[domain].push((request, dfd), priority) + self.pending_requests[domain].push((request, dfd), request.priority) return dfd def next_request(self, domain): @@ -107,9 +107,8 @@ class Scheduler(object) : """ try: - # The second value is the request scheduled priority, returns the first one. - return self.pending_requests[domain].pop()[0] - except (KeyError, IndexError), ex: + return self.pending_requests[domain].pop()[0] # [1] is priority + except (KeyError, IndexError): return (None, None) def close_domain(self, domain) : diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 0055c7139..2556b7bb3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -5,7 +5,6 @@ requests in Scrapy. See documentation in docs/ref/request-response.rst """ -import urllib import copy from twisted.internet import defer @@ -13,18 +12,18 @@ from twisted.internet import defer from scrapy.http.url import Url from scrapy.http.headers import Headers from scrapy.utils.url import safe_url_string -from scrapy.utils.defer import chain_deferred class Request(object): def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', dont_filter=False, - errback=None): + cookies=None, meta=None, encoding='utf-8', priority=0.0, + dont_filter=False, errback=None): self._encoding = encoding # this one has to be set first self.method = method.upper() self.set_url(url) self.set_body(body) + self.priority = priority if callable(callback): callback = defer.Deferred().addCallbacks(callback, errback) diff --git a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py index 75becd859..029d3fe08 100644 --- a/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py +++ b/scrapy/tests/test_schedulermiddleware_duplicatesfilter.py @@ -1,7 +1,6 @@ import unittest -from scrapy.spider import spiders -from scrapy.http import Request, Response +from scrapy.http import Request from scrapy.core.exceptions import IgnoreRequest from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware from scrapy.dupefilter import dupefilter @@ -25,7 +24,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase): r3 = Request('http://scrapytest.org/2') r4 = Request('http://scrapytest.org/1') - assert not mw.enqueue_request(domain, r1, 1) - assert not mw.enqueue_request(domain, r2, 1) - self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r3, 1) - self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r4, 1) + assert not mw.enqueue_request(domain, r1) + assert not mw.enqueue_request(domain, r2) + self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r3) + self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r4)