diff --git a/scrapy/contrib/delayedclosedomain.py b/scrapy/contrib/delayedclosedomain.py index ef79deaaa..78b9de4c9 100644 --- a/scrapy/contrib/delayedclosedomain.py +++ b/scrapy/contrib/delayedclosedomain.py @@ -25,7 +25,10 @@ class DelayedCloseDomain(object): dispatcher.connect(self.domain_closed, signal=signals.domain_closed) def domain_idle(self, domain): - lastseen = scrapyengine.downloader.lastseen(domain) + try: + lastseen = scrapyengine.downloader.sites[domain].lastseen + except KeyError: + lastseen = None if not lastseen: lastseen = self.opened_at[domain] diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 3518065c3..e69035680 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -55,7 +55,8 @@ class LiveStats(object): s += "DomainItems
ScrapedPages
CrawledScheduler
PendingDownloader
PendingDownloader
ActiveStart timeFinish timeRun time\n" for d in sorted(self.domains.keys()): scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0 - active, pending = len(dwl.active_requests(d)), len(dwl.request_queue(d)) + active = len(dwl.sites[d].active) if d in dwl.sites else 0 + pending = len(dwl.sites[d].queue) if d in dwl.queue else 0 stats = self.domains[d] runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index 2d48efd92..43cc7c273 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -151,10 +151,6 @@ class Downloader(object): def domain_is_open(self, domain): return domain in self.sites - def lastseen(self, domain): - if domain in self.sites: - return self.sites[domain].lastseen - def outstanding(self, domain): """The number of outstanding requests for a domain This includes both active requests and pending requests. @@ -163,17 +159,6 @@ class Downloader(object): if site: return site.outstanding() - def domain_is_idle(self, domain): - return not self.outstanding(domain) - - def request_queue(self, domain): - site = self.sites.get(domain) - return site.queue if site else [] - - def active_requests(self, domain): - site = self.sites.get(domain) - return site.active if site else [] - def has_capacity(self): """Does the downloader have capacity to handle more domains""" return len(self.sites) < self.concurrent_domains diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 732721f35..f360a8dcf 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -208,7 +208,7 @@ class ExecutionEngine(object): def domain_is_idle(self, domain): scraping = self._scraping.get(domain) pending = self.scheduler.domain_has_pending_requests(domain) - downloading = not self.downloader.domain_is_idle(domain) + downloading = domain in self.downloader.sites and self.downloader.sites[domain].outstanding() > 0 haspipe = not self.pipeline.domain_is_idle(domain) return not (pending or downloading or haspipe or scraping) @@ -426,11 +426,14 @@ class ExecutionEngine(object): ] domain_tests = [ "self.domain_is_idle(domain)", + "domain in self.cancelled", "self.scheduler.domain_has_pending_requests(domain)", "len(self.scheduler.pending_requests[domain])", - "self.downloader.outstanding(domain)", - "len(self.downloader.request_queue(domain))", - "len(self.downloader.active_requests(domain))", + "len(self.downloader.sites[domain].queue)", + "len(self.downloader.sites[domain].active)", + "len(self.downloader.sites[domain].transferring)", + "self.downloader.sites[domain].closed", + "self.downloader.sites[domain].lastseen", "self.pipeline.domain_is_idle(domain)", "len(self.pipeline.domaininfo[domain])", "len(self._scraping[domain])",