diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 084108164..d7b66878c 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', body, headers, cookies, meta, encoding='utf-8', priority=0.0, dont_filter=False, errback]) +.. class:: Request(url[, method='GET', body, headers, cookies, meta, encoding='utf-8', priority=0.0, dont_filter=False, callback, errback]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -33,11 +33,6 @@ Request objects :param url: the URL of this request :type url: string - :param callback: the function that will be called with the response of this - request (once its downloaded) as its first parameter. For more information - see :ref:`topics-request-response-ref-request-callback-arguments` below. - :type callback: callable - :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: string @@ -92,6 +87,13 @@ Request objects care, or you will get into crawling loops. Default to ``False``. :type dont_filter: boolean + :param callback: the function that will be called with the response of this + request (once its downloaded) as its first parameter. For more information + see :ref:`topics-request-response-ref-request-callback-arguments` below. + If a Request doesn't specify a callback, the spider's + :meth:`~scrapy.spider.BaseSpider.parse` method will be used. + :type callback: callable + :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b69f9a445..0504a4e70 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -136,12 +136,19 @@ BaseSpider .. method:: parse(response) - This is the default callback used by the :meth:`start_requests` method, and - will be used to parse the first pages crawled by the spider. + This is the default callback used by Scrapy to process downloaded + responses, when their requests don't specify a callback. The ``parse`` method is in charge of processing the response and returning scraped data and/or more URLs to follow. Other Requests callbacks have - the same requirements as the BaseSpider class. + the same requirements as the :class:`BaseSpider` class. + + This method, as well as any other Request callback, must return an + iterable of :class:`~scrapy.http.Request~ and/or + :class:`~scrapy.item.Item` objects. + + :param response: the response to parse + :type reponse: :class:~scrapy.http.Response` .. method:: log(message, [level, component]) diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 10debf84d..195af8928 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -46,7 +46,7 @@ class MediaPipeline(object): return DeferredList(dlist, consumeErrors=1).addCallback(self.item_completed, item, info) def _enqueue(self, request, info): - wad = request.deferred or Deferred() + wad = Deferred().addCallbacks(request.callback, request.errback) fp = request_fingerprint(request) # if already downloaded, return cached result. diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py index 2e74dab4e..14d4fe40e 100644 --- a/scrapy/contrib/spiders/crawl.py +++ b/scrapy/contrib/spiders/crawl.py @@ -6,6 +6,7 @@ See documentation in docs/topics/spiders.rst """ import copy +from functools import partial from scrapy.http import Request from scrapy.utils.spider import iterate_spider_output @@ -97,9 +98,10 @@ class CrawlSpider(InitSpider): links = rule.process_links(links) seen = seen.union(links) for link in links: - r = Request(url=link.url) + callback = partial(self._response_downloaded, callback=rule.callback, \ + cb_kwargs=rule.cb_kwargs, follow=rule.follow) + r = Request(url=link.url, callback=callback) r.meta['link_text'] = link.text - r.deferred.addCallback(self._response_downloaded, rule.callback, cb_kwargs=rule.cb_kwargs, follow=rule.follow) yield r def _response_downloaded(self, response, callback, cb_kwargs, follow): diff --git a/scrapy/contrib_exp/spiderprofiler.py b/scrapy/contrib_exp/spiderprofiler.py index 0587c6997..70de83d28 100644 --- a/scrapy/contrib_exp/spiderprofiler.py +++ b/scrapy/contrib_exp/spiderprofiler.py @@ -33,10 +33,7 @@ class SpiderProfiler(object): dispatcher.connect(self._request_received, signals.request_received) def _request_received(self, request, spider): - old_cbs = request.deferred.callbacks[0] - new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], \ - old_cbs[0][2]), old_cbs[1]) - request.deferred.callbacks[0] = new_cbs + request.callback = self._profiled_callback(request.callback, spider) def _profiled_callback(self, function, spider): def new_callback(*args, **kwargs): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 9b0e252da..8b8d5b83f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -149,10 +149,6 @@ class ExecutionEngine(object): def crawl(self, request, spider): assert spider in self.open_spiders, \ "Spider %r not opened when crawling: %s" % (spider.name, request) - if not request.deferred.callbacks: - log.msg("Unable to crawl Request with no callback: %s" % request, - level=log.ERROR, spider=spider) - return if spider in self.closing: # ignore requests for spiders being closed return schd = mustbe_deferred(self.schedule, request, spider) @@ -181,9 +177,11 @@ class ExecutionEngine(object): return response elif isinstance(response, Request): newrequest = response - schd = mustbe_deferred(self.schedule, newrequest, spider) - schd.chainDeferred(newrequest.deferred) - return newrequest.deferred + dfd = mustbe_deferred(self.schedule, newrequest, spider) + if newrequest.callback: + # XXX: this is a bit hacky and should be removed + dfd.addCallbacks(newrequest.callback, newrequest.errback) + return dfd def _on_error(_failure): """handle an error processing a page""" diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index d1e82fb98..10fefe8cd 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -132,8 +132,9 @@ class Scraper(object): request_result, request, spider) def call_spider(self, result, request, spider): - defer_result(result).chainDeferred(request.deferred) - return request.deferred.addCallback(iterate_spider_output) + dfd = defer_result(result) + dfd.addCallbacks(request.callback or spider.parse, request.errback) + return dfd.addCallback(iterate_spider_output) def handle_spider_error(self, _failure, request, spider, propagated_failure=None): referer = request.headers.get('Referer', None) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 4dce92880..1274127eb 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -7,8 +7,6 @@ See documentation in docs/topics/request-response.rst import copy -from twisted.internet import defer - from scrapy.http.headers import Headers from scrapy.utils.url import safe_url_string from scrapy.utils.trackref import object_ref @@ -16,7 +14,7 @@ from scrapy.utils.trackref import object_ref class Request(object_ref): __slots__ = ['_encoding', 'method', '_url', '_body', '_meta', \ - 'dont_filter', 'headers', 'cookies', 'deferred', 'priority', \ + 'dont_filter', 'headers', 'cookies', 'callback', 'errback', 'priority', \ '__weakref__'] def __init__(self, url, callback=None, method='GET', headers=None, body=None, @@ -29,9 +27,9 @@ class Request(object_ref): self._set_body(body) self.priority = priority - if callable(callback): - callback = defer.Deferred().addCallbacks(callback, errback) - self.deferred = callback or defer.Deferred() + assert callback or not errback, "Cannot use errback without a callback" + self.callback = callback + self.errback = errback self.cookies = cookies or {} self.headers = Headers(headers or {}, encoding=encoding) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index c1ac2c1d4..09e9f1771 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -71,13 +71,10 @@ class BaseSpider(object_ref): return reqs def make_requests_from_url(self, url): - return Request(url, callback=self.parse, dont_filter=True) + return Request(url, dont_filter=True) def parse(self, response): - """This is the default callback function used to parse the start - requests, although it can be overrided in descendant spiders. - """ - pass + raise NotImplementedError def __str__(self): return "<%s %r>" % (type(self).__name__, self.name)