RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove calls to chain_deferred and deferred_imap

This commit is contained in:
dgrana 2009-06-23 12:53:13 -07:00
parent a8a44aa035
commit 38f184e42a
1 changed files with 12 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
scrapy/core/engine.py
View File

@ -23,7 +23,6 @@ from scrapy.item import ScrapedItem
from scrapy.item.pipeline import ItemPipelineManager
from scrapy.spider import spiders
from scrapy.spider.middleware import SpiderMiddlewareManager
from scrapy.utils.defer import chain_deferred, deferred_imap
from scrapy.utils.request import request_info
from scrapy.utils.misc import load_object
from scrapy.utils.defer import mustbe_deferred
@ -222,7 +221,6 @@ class ExecutionEngine(object):
signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, item=item, spider=spider, response=response, exception=pipe_result.value)
else:
signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, item=item, spider=spider, response=response, pipe_output=pipe_result)
self.next_request(spider)
if domain in self.closing:
return
@ -237,9 +235,10 @@ class ExecutionEngine(object):
elif output is None:
pass # may be next time.
else:
log.msg("Spider must return Request, ScrapedItem or None, got '%s' while processing %s" % (type(output).__name__, request), log.WARNING, domain=domain)
log.msg("Spider must return Request, ScrapedItem or None, got '%s' while processing %s" \
% (type(output).__name__, request), log.WARNING, domain=domain)
return deferred_imap(cb_spider_output, spmw_result)
return task.coiterate(cb_spider_output, spmw_result)
def eb_user(_failure):
if not isinstance(_failure.value, IgnoreRequest):
@ -251,7 +250,7 @@ class ExecutionEngine(object):
log.msg('FRAMEWORK BUG processing %s: %s' % (request, _failure), log.ERROR, domain=domain)
scd = self.spidermiddleware.scrape(request, response, spider)
scd = mustbe_deferred(self.spidermiddleware.scrape, request, response, spider)
scd.addCallbacks(cb_spidermiddleware_output, eb_user)
scd.addErrback(eb_framework)
@ -266,7 +265,7 @@ class ExecutionEngine(object):
request.deferred.addErrback(lambda _:None)
request.deferred.errback(_failure) # TODO: merge into spider middleware.
schd = self.schedule(request, spider)
schd = mustbe_deferred(self.schedule, request, spider)
schd.addCallbacks(_process_response, _cleanfailure)
return schd
@ -305,15 +304,16 @@ class ExecutionEngine(object):
assert isinstance(response, (Response, Request))
if self.debug_mode:
log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain)
if isinstance(response, Response):
response.request = request # tie request to obtained response
log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain)
return response
elif isinstance(response, Request):
newrequest = response # proper alias
schd = self.schedule(newrequest, spider)
chain_deferred(schd, newrequest.deferred)
return schd
schd = mustbe_deferred(self.schedule, newrequest, spider)
schd.chainDeferred(newrequest.deferred)
return newrequest.deferred
def _on_error(_failure):
"""handle an error processing a page"""
@ -324,13 +324,12 @@ class ExecutionEngine(object):
def _on_complete(_):
self.next_request(spider)
return _
dwld = self.downloader.fetch(request, spider)
dwld = mustbe_deferred(self.downloader.fetch, request, spider)
dwld.addCallbacks(_on_success, _on_error)
deferred = defer.Deferred()
chain_deferred(dwld, deferred)
dwld.addBoth(_on_complete)
return deferred
return dwld
def open_domain(self, domain, spider=None):
log.msg("Domain opened", domain=domain)