additional simplifications to downloader (several methods removed) and added more info to engine getstatus() method

scrapy/contrib/delayedclosedomain.py

@@ -25,7 +25,10 @@ class DelayedCloseDomain(object):
         dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
 
     def domain_idle(self, domain):
-        lastseen = scrapyengine.downloader.lastseen(domain)
+        try:
+            lastseen = scrapyengine.downloader.sites[domain].lastseen
+        except KeyError:
+            lastseen = None
         if not lastseen:
             lastseen = self.opened_at[domain]
 

scrapy/contrib/webconsole/livestats.py

@@ -55,7 +55,8 @@ class LiveStats(object):
         s += "<tr><th>Domain</th><th>Items<br>Scraped</th><th>Pages<br>Crawled</th><th>Scheduler<br>Pending</th><th>Downloader<br/>Pending</th><th>Downloader<br/>Active</th><th>Start time</th><th>Finish time</th><th>Run time</th></tr>\n"
         for d in sorted(self.domains.keys()):
             scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0
-            active, pending = len(dwl.active_requests(d)), len(dwl.request_queue(d))
+            active = len(dwl.sites[d].active) if d in dwl.sites else 0
+            pending = len(dwl.sites[d].queue) if d in dwl.queue else 0
             stats = self.domains[d]
             runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started
 

scrapy/core/downloader/manager.py

@@ -151,10 +151,6 @@ class Downloader(object):
     def domain_is_open(self, domain):
         return domain in self.sites
 
-    def lastseen(self, domain):
-        if domain in self.sites:
-            return self.sites[domain].lastseen
-
     def outstanding(self, domain):
         """The number of outstanding requests for a domain
         This includes both active requests and pending requests.
@@ -163,17 +159,6 @@ class Downloader(object):
         if site:
             return site.outstanding()
 
-    def domain_is_idle(self, domain):
-        return not self.outstanding(domain)
-
-    def request_queue(self, domain):
-        site = self.sites.get(domain)
-        return site.queue if site else []
-
-    def active_requests(self, domain):
-        site = self.sites.get(domain)
-        return site.active if site else []
-
     def has_capacity(self):
         """Does the downloader have capacity to handle more domains"""
         return len(self.sites) < self.concurrent_domains

scrapy/core/engine.py

@@ -208,7 +208,7 @@ class ExecutionEngine(object):
     def domain_is_idle(self, domain):
         scraping = self._scraping.get(domain)
         pending = self.scheduler.domain_has_pending_requests(domain)
-        downloading = not self.downloader.domain_is_idle(domain)
+        downloading = domain in self.downloader.sites and self.downloader.sites[domain].outstanding() > 0
         haspipe = not self.pipeline.domain_is_idle(domain)
         return not (pending or downloading or haspipe or scraping)
 
@@ -426,11 +426,14 @@ class ExecutionEngine(object):
             ]
         domain_tests = [
             "self.domain_is_idle(domain)",
+            "domain in self.cancelled",
             "self.scheduler.domain_has_pending_requests(domain)",
             "len(self.scheduler.pending_requests[domain])",
-            "self.downloader.outstanding(domain)",
-            "len(self.downloader.request_queue(domain))",
-            "len(self.downloader.active_requests(domain))",
+            "len(self.downloader.sites[domain].queue)",
+            "len(self.downloader.sites[domain].active)",
+            "len(self.downloader.sites[domain].transferring)",
+            "self.downloader.sites[domain].closed",
+            "self.downloader.sites[domain].lastseen",
             "self.pipeline.domain_is_idle(domain)",
             "len(self.pipeline.domaininfo[domain])",
             "len(self._scraping[domain])",