From 38f184e42ad5f0e6e21780ba30581341d54caba0 Mon Sep 17 00:00:00 2001 From: dgrana Date: Tue, 23 Jun 2009 12:53:13 -0700 Subject: [PATCH] remove calls to chain_deferred and deferred_imap --- scrapy/core/engine.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 443f8f865..7566766d2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -23,7 +23,6 @@ from scrapy.item import ScrapedItem from scrapy.item.pipeline import ItemPipelineManager from scrapy.spider import spiders from scrapy.spider.middleware import SpiderMiddlewareManager -from scrapy.utils.defer import chain_deferred, deferred_imap from scrapy.utils.request import request_info from scrapy.utils.misc import load_object from scrapy.utils.defer import mustbe_deferred @@ -222,7 +221,6 @@ class ExecutionEngine(object): signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, item=item, spider=spider, response=response, exception=pipe_result.value) else: signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, item=item, spider=spider, response=response, pipe_output=pipe_result) - self.next_request(spider) if domain in self.closing: return @@ -237,9 +235,10 @@ class ExecutionEngine(object): elif output is None: pass # may be next time. else: - log.msg("Spider must return Request, ScrapedItem or None, got '%s' while processing %s" % (type(output).__name__, request), log.WARNING, domain=domain) + log.msg("Spider must return Request, ScrapedItem or None, got '%s' while processing %s" \ + % (type(output).__name__, request), log.WARNING, domain=domain) - return deferred_imap(cb_spider_output, spmw_result) + return task.coiterate(cb_spider_output, spmw_result) def eb_user(_failure): if not isinstance(_failure.value, IgnoreRequest): @@ -251,7 +250,7 @@ class ExecutionEngine(object): log.msg('FRAMEWORK BUG processing %s: %s' % (request, _failure), log.ERROR, domain=domain) - scd = self.spidermiddleware.scrape(request, response, spider) + scd = mustbe_deferred(self.spidermiddleware.scrape, request, response, spider) scd.addCallbacks(cb_spidermiddleware_output, eb_user) scd.addErrback(eb_framework) @@ -266,7 +265,7 @@ class ExecutionEngine(object): request.deferred.addErrback(lambda _:None) request.deferred.errback(_failure) # TODO: merge into spider middleware. - schd = self.schedule(request, spider) + schd = mustbe_deferred(self.schedule, request, spider) schd.addCallbacks(_process_response, _cleanfailure) return schd @@ -305,15 +304,16 @@ class ExecutionEngine(object): assert isinstance(response, (Response, Request)) if self.debug_mode: log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain) + if isinstance(response, Response): response.request = request # tie request to obtained response log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain) return response elif isinstance(response, Request): newrequest = response # proper alias - schd = self.schedule(newrequest, spider) - chain_deferred(schd, newrequest.deferred) - return schd + schd = mustbe_deferred(self.schedule, newrequest, spider) + schd.chainDeferred(newrequest.deferred) + return newrequest.deferred def _on_error(_failure): """handle an error processing a page""" @@ -324,13 +324,12 @@ class ExecutionEngine(object): def _on_complete(_): self.next_request(spider) + return _ - dwld = self.downloader.fetch(request, spider) + dwld = mustbe_deferred(self.downloader.fetch, request, spider) dwld.addCallbacks(_on_success, _on_error) - deferred = defer.Deferred() - chain_deferred(dwld, deferred) dwld.addBoth(_on_complete) - return deferred + return dwld def open_domain(self, domain, spider=None): log.msg("Domain opened", domain=domain)