RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

engine: simplified next_request and removed 'domain in self.closing' check

This commit is contained in:
Pablo Hoffman 2009-06-22 19:59:36 -03:00
parent ab0950b67e
commit b3a624ed6b
1 changed files with 4 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapy/core/engine.py
View File

@ -26,6 +26,7 @@ from scrapy.spider.middleware import SpiderMiddlewareManager
from scrapy.utils.defer import chain_deferred, deferred_imap
from scrapy.utils.request import request_info
from scrapy.utils.misc import load_object
from scrapy.utils.defer import mustbe_deferred
class ExecutionEngine(object):
"""
@ -182,7 +183,6 @@ class ExecutionEngine(object):
domain = spider.domain_name
if not self.running or \
domain in self.closing or \
self.domain_is_closed(domain) or \
self.downloader.sites[domain].needs_backout():
return
@ -190,18 +190,9 @@ class ExecutionEngine(object):
# Next pending request from scheduler
request, deferred = self.scheduler.next_request(domain)
if request:
try:
dwld = self.download(request, spider)
except IgnoreRequest, ex:
log.msg(ex.message, log.WARNING, domain=domain)
except Exception, ex:
log.exc("Bug in download code: %s" % request, domain=domain)
self._domain_idle(domain)
else:
chain_deferred(dwld, deferred)
else:
if self.domain_is_idle(domain):
self._domain_idle(domain)
mustbe_deferred(self.download, request, spider).chainDeferred(deferred)
elif self.domain_is_idle(domain):
self._domain_idle(domain)
def domain_is_idle(self, domain):
scraping = self._scraping.get(domain)