diff --git a/scrapy/contrib/delayedclosedomain.py b/scrapy/contrib/delayedclosedomain.py
index ef79deaaa..78b9de4c9 100644
--- a/scrapy/contrib/delayedclosedomain.py
+++ b/scrapy/contrib/delayedclosedomain.py
@@ -25,7 +25,10 @@ class DelayedCloseDomain(object):
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
def domain_idle(self, domain):
- lastseen = scrapyengine.downloader.lastseen(domain)
+ try:
+ lastseen = scrapyengine.downloader.sites[domain].lastseen
+ except KeyError:
+ lastseen = None
if not lastseen:
lastseen = self.opened_at[domain]
diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py
index 3518065c3..e69035680 100644
--- a/scrapy/contrib/webconsole/livestats.py
+++ b/scrapy/contrib/webconsole/livestats.py
@@ -55,7 +55,8 @@ class LiveStats(object):
s += "
| Domain | Items Scraped | Pages Crawled | Scheduler Pending | Downloader Pending | Downloader Active | Start time | Finish time | Run time |
\n"
for d in sorted(self.domains.keys()):
scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0
- active, pending = len(dwl.active_requests(d)), len(dwl.request_queue(d))
+ active = len(dwl.sites[d].active) if d in dwl.sites else 0
+ pending = len(dwl.sites[d].queue) if d in dwl.queue else 0
stats = self.domains[d]
runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started
diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py
index 2d48efd92..43cc7c273 100644
--- a/scrapy/core/downloader/manager.py
+++ b/scrapy/core/downloader/manager.py
@@ -151,10 +151,6 @@ class Downloader(object):
def domain_is_open(self, domain):
return domain in self.sites
- def lastseen(self, domain):
- if domain in self.sites:
- return self.sites[domain].lastseen
-
def outstanding(self, domain):
"""The number of outstanding requests for a domain
This includes both active requests and pending requests.
@@ -163,17 +159,6 @@ class Downloader(object):
if site:
return site.outstanding()
- def domain_is_idle(self, domain):
- return not self.outstanding(domain)
-
- def request_queue(self, domain):
- site = self.sites.get(domain)
- return site.queue if site else []
-
- def active_requests(self, domain):
- site = self.sites.get(domain)
- return site.active if site else []
-
def has_capacity(self):
"""Does the downloader have capacity to handle more domains"""
return len(self.sites) < self.concurrent_domains
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 732721f35..f360a8dcf 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -208,7 +208,7 @@ class ExecutionEngine(object):
def domain_is_idle(self, domain):
scraping = self._scraping.get(domain)
pending = self.scheduler.domain_has_pending_requests(domain)
- downloading = not self.downloader.domain_is_idle(domain)
+ downloading = domain in self.downloader.sites and self.downloader.sites[domain].outstanding() > 0
haspipe = not self.pipeline.domain_is_idle(domain)
return not (pending or downloading or haspipe or scraping)
@@ -426,11 +426,14 @@ class ExecutionEngine(object):
]
domain_tests = [
"self.domain_is_idle(domain)",
+ "domain in self.cancelled",
"self.scheduler.domain_has_pending_requests(domain)",
"len(self.scheduler.pending_requests[domain])",
- "self.downloader.outstanding(domain)",
- "len(self.downloader.request_queue(domain))",
- "len(self.downloader.active_requests(domain))",
+ "len(self.downloader.sites[domain].queue)",
+ "len(self.downloader.sites[domain].active)",
+ "len(self.downloader.sites[domain].transferring)",
+ "self.downloader.sites[domain].closed",
+ "self.downloader.sites[domain].lastseen",
"self.pipeline.domain_is_idle(domain)",
"len(self.pipeline.domaininfo[domain])",
"len(self._scraping[domain])",