RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added support for Requests without callbacks (#166) - the Spider.parse() method 

is used in those cases.

Also removed Request.deferred attribute.
This commit is contained in:
Pablo Hoffman 2010-06-08 18:18:02 -03:00
parent 76ed9d442b
commit 73305b1eb3
9 changed files with 38 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/topics/request-response.rst
View File

@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
Request objects
===============
.. class:: Request(url[, callback, method='GET', body, headers, cookies, meta, encoding='utf-8', priority=0.0, dont_filter=False, errback])
.. class:: Request(url[, method='GET', body, headers, cookies, meta, encoding='utf-8', priority=0.0, dont_filter=False, callback, errback])
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
@ -33,11 +33,6 @@ Request objects
:param url: the URL of this request
:type url: string
:param callback: the function that will be called with the response of this
request (once its downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
:type callback: callable
:param method: the HTTP method of this request. Defaults to ``'GET'``.
:type method: string
@ -92,6 +87,13 @@ Request objects
care, or you will get into crawling loops. Default to ``False``.
:type dont_filter: boolean
:param callback: the function that will be called with the response of this
request (once its downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.spider.BaseSpider.parse` method will be used.
:type callback: callable
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance

13
docs/topics/spiders.rst
View File

@ -136,12 +136,19 @@ BaseSpider
.. method:: parse(response)
This is the default callback used by the :meth:`start_requests` method, and
will be used to parse the first pages crawled by the spider.
This is the default callback used by Scrapy to process downloaded
responses, when their requests don't specify a callback.
The ``parse`` method is in charge of processing the response and returning
scraped data and/or more URLs to follow. Other Requests callbacks have
the same requirements as the BaseSpider class.
the same requirements as the :class:`BaseSpider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.http.Request~ and/or
:class:`~scrapy.item.Item` objects.
:param response: the response to parse
:type reponse: :class:~scrapy.http.Response`
.. method:: log(message, [level, component])

2
scrapy/contrib/pipeline/media.py
View File

@ -46,7 +46,7 @@ class MediaPipeline(object):
return DeferredList(dlist, consumeErrors=1).addCallback(self.item_completed, item, info)
def _enqueue(self, request, info):
wad = request.deferred or Deferred()
wad = Deferred().addCallbacks(request.callback, request.errback)
fp = request_fingerprint(request)
# if already downloaded, return cached result.

6
scrapy/contrib/spiders/crawl.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/spiders.rst
"""
import copy
from functools import partial
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
@ -97,9 +98,10 @@ class CrawlSpider(InitSpider):
links = rule.process_links(links)
seen = seen.union(links)
for link in links:
r = Request(url=link.url)
callback = partial(self._response_downloaded, callback=rule.callback, \
cb_kwargs=rule.cb_kwargs, follow=rule.follow)
r = Request(url=link.url, callback=callback)
r.meta['link_text'] = link.text
r.deferred.addCallback(self._response_downloaded, rule.callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow)
yield r
def _response_downloaded(self, response, callback, cb_kwargs, follow):

5
scrapy/contrib_exp/spiderprofiler.py
View File

@ -33,10 +33,7 @@ class SpiderProfiler(object):
dispatcher.connect(self._request_received, signals.request_received)
def _request_received(self, request, spider):
old_cbs = request.deferred.callbacks[0]
new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], \
old_cbs[0][2]), old_cbs[1])
request.deferred.callbacks[0] = new_cbs
request.callback = self._profiled_callback(request.callback, spider)
def _profiled_callback(self, function, spider):
def new_callback(*args, **kwargs):

12
scrapy/core/engine.py
View File

@ -149,10 +149,6 @@ class ExecutionEngine(object):
def crawl(self, request, spider):
assert spider in self.open_spiders, \
"Spider %r not opened when crawling: %s" % (spider.name, request)
if not request.deferred.callbacks:
log.msg("Unable to crawl Request with no callback: %s" % request,
level=log.ERROR, spider=spider)
return
if spider in self.closing: # ignore requests for spiders being closed
return
schd = mustbe_deferred(self.schedule, request, spider)
@ -181,9 +177,11 @@ class ExecutionEngine(object):
return response
elif isinstance(response, Request):
newrequest = response
schd = mustbe_deferred(self.schedule, newrequest, spider)
schd.chainDeferred(newrequest.deferred)
return newrequest.deferred
dfd = mustbe_deferred(self.schedule, newrequest, spider)
if newrequest.callback:
# XXX: this is a bit hacky and should be removed
dfd.addCallbacks(newrequest.callback, newrequest.errback)
return dfd
def _on_error(_failure):
"""handle an error processing a page"""

5
scrapy/core/scraper.py
View File

@ -132,8 +132,9 @@ class Scraper(object):
request_result, request, spider)
def call_spider(self, result, request, spider):
defer_result(result).chainDeferred(request.deferred)
return request.deferred.addCallback(iterate_spider_output)
dfd = defer_result(result)
dfd.addCallbacks(request.callback or spider.parse, request.errback)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, spider, propagated_failure=None):
referer = request.headers.get('Referer', None)

10
scrapy/http/request/__init__.py
View File

@ -7,8 +7,6 @@ See documentation in docs/topics/request-response.rst
import copy
from twisted.internet import defer
from scrapy.http.headers import Headers
from scrapy.utils.url import safe_url_string
from scrapy.utils.trackref import object_ref
@ -16,7 +14,7 @@ from scrapy.utils.trackref import object_ref
class Request(object_ref):
__slots__ = ['_encoding', 'method', '_url', '_body', '_meta', \
'dont_filter', 'headers', 'cookies', 'deferred', 'priority', \
'dont_filter', 'headers', 'cookies', 'callback', 'errback', 'priority', \
'__weakref__']
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
@ -29,9 +27,9 @@ class Request(object_ref):
self._set_body(body)
self.priority = priority
if callable(callback):
callback = defer.Deferred().addCallbacks(callback, errback)
self.deferred = callback or defer.Deferred()
assert callback or not errback, "Cannot use errback without a callback"
self.callback = callback
self.errback = errback
self.cookies = cookies or {}
self.headers = Headers(headers or {}, encoding=encoding)

7
scrapy/spider/models.py
View File

@ -71,13 +71,10 @@ class BaseSpider(object_ref):
return reqs
def make_requests_from_url(self, url):
return Request(url, callback=self.parse, dont_filter=True)
return Request(url, dont_filter=True)
def parse(self, response):
"""This is the default callback function used to parse the start
requests, although it can be overrided in descendant spiders.
"""
pass
raise NotImplementedError
def __str__(self):
return "<%s %r>" % (type(self).__name__, self.name)