core: allow engine download to be dinamically backouted to the double of the max_concurrent_requests for a domain

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40470

scrapy/trunk/scrapy/core/downloader/manager.py

@@ -35,6 +35,12 @@ class SiteDetails(object):
     def capacity(self):
         return self.max_concurrent_requests - len(self.downloading)
 
+    def outstanding(self):
+        return len(self.active) + len(self.queue)
+
+    def needs_backout(self):
+        return self.outstanding() > (2 * self.max_concurrent_requests)
+
 
 class Downloader(object):
     """Maintain many concurrent downloads and provide an HTTP abstraction
@@ -155,6 +161,10 @@ class Downloader(object):
         else:
             log.msg('Domain %s already closed' % domain, log.TRACE, domain=domain)
 
+    def needs_backout(self, domain):
+        site = self.sites.get(domain)
+        return (site.needs_backout() if site else True)
+
     # Most of the following functions must be reviewed to decide if are really needed
     def domain_is_open(self, domain):
         return domain in self.sites
@@ -169,7 +179,7 @@ class Downloader(object):
         """
         site = self.sites.get(domain)
         if site:
-            return len(site.active) + len(site.queue)
+            return site.outstanding()
 
     def domain_is_idle(self, domain):
         return not self.outstanding(domain)

scrapy/trunk/scrapy/core/engine.py

@@ -38,9 +38,6 @@ class ExecutionEngine(object):
     process for that domain.
     """
 
-    # back off downloading more pages if we exceed this backlog size
-    DOWNLOADER_BACKLOG = 10
-
     def __init__(self):
         self.configured = False
         self.keep_alive = False
@@ -179,9 +176,8 @@ class ExecutionEngine(object):
         if not self.running:
             return
 
-        # outstanding requests in downloader queue
-        outstanding = self.downloader.outstanding(domain)
-        if outstanding > self.DOWNLOADER_BACKLOG:
+        # backout enqueing downloads if domain needs it
+        if self.downloader.needs_backout(domain):
             return
 
         # Next pending request from scheduler