RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added 'priority' attribute to Requests and removed old 'priority' argument passed through engine, scheduler and scheduler middleware calls

This commit is contained in:
Pablo Hoffman 2009-06-11 22:25:47 -03:00
parent 962dbeba88
commit 4a1a01354b
8 changed files with 33 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/ref/request-response.rst
View File

@ -25,7 +25,7 @@ below in :ref:`ref-request-subclasses` and :ref:`ref-response-subclasses`.
Request objects
===============
.. class:: Request(url[, callback, method='GET', body, headers, cookies, meta, encoding='utf-8', dont_filter=False, errback])
.. class:: Request(url[, callback, method='GET', body, headers, cookies, meta, encoding='utf-8', priority=0.0, dont_filter=False, errback])
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
@ -81,6 +81,12 @@ Request objects
body to ``str`` (if given as ``unicode``).
:type encoding: string
:param priority: the priority of this request (defaults to ``0.0``).
The priority is used by the scheduler to define the order used to return
requests. It can also be used to feed priorities externally, for
example, using an offline long-term scheduler.
:type encoding: int or float
:param dont_filter: indicates that this request should not be filtered by
the scheduler. This is used when you want to perform an identical
request multiple times, to ignore the duplicates filter. Use it with

2
scrapy/contrib/schedulermiddleware/duplicatesfilter.py
View File

@ -9,7 +9,7 @@ from scrapy.dupefilter import dupefilter
class DuplicatesFilterMiddleware(object):
"""Filter out already seen requests to avoid visiting pages more than once."""
def enqueue_request(self, domain, request, priority):
def enqueue_request(self, domain, request):
added = dupefilter.add(domain, request)
if not (added or request.dont_filter):
raise IgnoreRequest('Skipped (already seen request)')

4
scrapy/contrib_exp/history/scheduler.py
View File

@ -33,7 +33,7 @@ class RulesScheduler(Scheduler):
# How often we should process pages that have not changed (need to include depth)
MIN_PROCESS_UNCHANGED_DAYS = 12
def enqueue_request(self, domain, request, priority=1):
def enqueue_request(self, domain, request):
"""Add a page to be scraped for a domain that is currently being scraped.
The url will only be added if we have not checked it already within
@ -55,7 +55,7 @@ class RulesScheduler(Scheduler):
return
# put the version in the pending pages to avoid querying DB again
record = (request, version, now)
self.pending_requests[domain].put(record, priority)
self.pending_requests[domain].push(record, request.priority)
def next_request(self, domain):
"""Get the next page from the superclass. This will add a callback

13
scrapy/core/engine.py
View File

@ -219,7 +219,7 @@ class ExecutionEngine(object):
def open_domains(self):
return self.downloader.sites.keys()
def crawl(self, request, spider, priority=0, domain_priority=0):
def crawl(self, request, spider, domain_priority=0):
domain = spider.domain_name
def _process(response):
@ -242,7 +242,7 @@ class ExecutionEngine(object):
piped.addBoth(_onpipelinefinish, item)
elif isinstance(item, Request):
signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response)
self.crawl(request=item, spider=spider, priority=priority)
self.crawl(request=item, spider=spider)
elif item is None:
pass # may be next time.
else:
@ -276,14 +276,14 @@ class ExecutionEngine(object):
request.deferred.addErrback(lambda _:None)
request.deferred.errback(_failure) #TODO: merge into spider middleware.
schd = self.schedule(request, spider, priority, domain_priority)
schd = self.schedule(request, spider, domain_priority)
schd.addCallbacks(_process, _cleanfailure)
return schd
def scrape(self, request, response, spider):
return self.spidermiddleware.scrape(request, response, spider)
def schedule(self, request, spider, priority=0, domain_priority=0):
def schedule(self, request, spider, domain_priority=0):
domain = spider.domain_name
if not self.scheduler.domain_is_open(domain):
if self.debug_mode:
@ -291,7 +291,7 @@ class ExecutionEngine(object):
return self._add_starter(request, spider, domain_priority)
if self.debug_mode:
log.msg('Scheduling %s (now)' % request_info(request), log.DEBUG)
schd = self.schedulermiddleware.enqueue_request(domain, request, priority)
schd = self.schedulermiddleware.enqueue_request(domain, request)
self.next_request(spider)
return schd
@ -343,7 +343,8 @@ class ExecutionEngine(object):
return response
elif isinstance(response, Request):
redirected = response # proper alias
schd = self.schedule(redirected, spider, priority=self.REDIRECTION_PRIORITY)
redirected.priority = self.REDIRECTION_PRIORITY
schd = self.schedule(redirected, spider)
chain_deferred(schd, redirected.deferred)
return schd

6
scrapy/core/scheduler/middleware.py
View File

@ -44,16 +44,16 @@ class SchedulerMiddlewareManager(object):
log.msg("Enabled scheduler middlewares: %s" % ", ".join([type(m).__name__ for m in mws]))
self.loaded = True
def enqueue_request(self, domain, request, priority):
def enqueue_request(self, domain, request):
def _enqueue_request(request):
for method in self.mw_enqueue_request:
result = method(domain=domain, request=request, priority=priority)
result = method(domain=domain, request=request)
assert result is None or isinstance(result, (Response, Deferred)), \
'Middleware %s.enqueue_request must return None, Response or Deferred, got %s' % \
(method.im_self.__class__.__name__, result.__class__.__name__)
if result:
return result
return self.scheduler.enqueue_request(domain=domain, request=request, priority=priority)
return self.scheduler.enqueue_request(domain=domain, request=request)
deferred = mustbe_deferred(_enqueue_request, request)
return deferred

11
scrapy/core/scheduler/schedulers.py
View File

@ -62,7 +62,7 @@ class Scheduler(object) :
def next_domain(self) :
"""Return next domain available to scrape and remove it from available domains queue"""
if self.pending_domains_count:
domain, priority = self.domains_queue.pop()
domain = self.domains_queue.pop()[0]
if self.pending_domains_count[domain] == 1:
del self.pending_domains_count[domain]
else:
@ -90,10 +90,10 @@ class Scheduler(object) :
Priority = PriorityStack if self.dfo else PriorityQueue
self.pending_requests[domain] = Priority()
def enqueue_request(self, domain, request, priority=0):
def enqueue_request(self, domain, request):
"""Enqueue a request to be downloaded for a domain that is currently being scraped."""
dfd = defer.Deferred()
self.pending_requests[domain].push((request, dfd), priority)
self.pending_requests[domain].push((request, dfd), request.priority)
return dfd
def next_request(self, domain):
@ -107,9 +107,8 @@ class Scheduler(object) :
"""
try:
# The second value is the request scheduled priority, returns the first one.
return self.pending_requests[domain].pop()[0]
except (KeyError, IndexError), ex:
return self.pending_requests[domain].pop()[0] # [1] is priority
except (KeyError, IndexError):
return (None, None)
def close_domain(self, domain) :

7
scrapy/http/request/__init__.py
View File

@ -5,7 +5,6 @@ requests in Scrapy.
See documentation in docs/ref/request-response.rst
"""
import urllib
import copy
from twisted.internet import defer
@ -13,18 +12,18 @@ from twisted.internet import defer
from scrapy.http.url import Url
from scrapy.http.headers import Headers
from scrapy.utils.url import safe_url_string
from scrapy.utils.defer import chain_deferred
class Request(object):
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', dont_filter=False,
errback=None):
cookies=None, meta=None, encoding='utf-8', priority=0.0,
dont_filter=False, errback=None):
self._encoding = encoding # this one has to be set first
self.method = method.upper()
self.set_url(url)
self.set_body(body)
self.priority = priority
if callable(callback):
callback = defer.Deferred().addCallbacks(callback, errback)

11
scrapy/tests/test_schedulermiddleware_duplicatesfilter.py
View File

@ -1,7 +1,6 @@
import unittest
from scrapy.spider import spiders
from scrapy.http import Request, Response
from scrapy.http import Request
from scrapy.core.exceptions import IgnoreRequest
from scrapy.contrib.schedulermiddleware.duplicatesfilter import DuplicatesFilterMiddleware
from scrapy.dupefilter import dupefilter
@ -25,7 +24,7 @@ class DuplicatesFilterMiddlewareTest(unittest.TestCase):
r3 = Request('http://scrapytest.org/2')
r4 = Request('http://scrapytest.org/1')
assert not mw.enqueue_request(domain, r1, 1)
assert not mw.enqueue_request(domain, r2, 1)
self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r3, 1)
self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r4, 1)
assert not mw.enqueue_request(domain, r1)
assert not mw.enqueue_request(domain, r2)
self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r3)
self.assertRaises(IgnoreRequest, mw.enqueue_request, domain, r4)