diff --git a/scrapy/trunk/scrapy/core/downloader/manager.py b/scrapy/trunk/scrapy/core/downloader/manager.py index ce8130687..f07c43ab0 100644 --- a/scrapy/trunk/scrapy/core/downloader/manager.py +++ b/scrapy/trunk/scrapy/core/downloader/manager.py @@ -35,6 +35,12 @@ class SiteDetails(object): def capacity(self): return self.max_concurrent_requests - len(self.downloading) + def outstanding(self): + return len(self.active) + len(self.queue) + + def needs_backout(self): + return self.outstanding() > (2 * self.max_concurrent_requests) + class Downloader(object): """Maintain many concurrent downloads and provide an HTTP abstraction @@ -155,6 +161,10 @@ class Downloader(object): else: log.msg('Domain %s already closed' % domain, log.TRACE, domain=domain) + def needs_backout(self, domain): + site = self.sites.get(domain) + return (site.needs_backout() if site else True) + # Most of the following functions must be reviewed to decide if are really needed def domain_is_open(self, domain): return domain in self.sites @@ -169,7 +179,7 @@ class Downloader(object): """ site = self.sites.get(domain) if site: - return len(site.active) + len(site.queue) + return site.outstanding() def domain_is_idle(self, domain): return not self.outstanding(domain) diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py index b8e9faace..8144cedf9 100644 --- a/scrapy/trunk/scrapy/core/engine.py +++ b/scrapy/trunk/scrapy/core/engine.py @@ -38,9 +38,6 @@ class ExecutionEngine(object): process for that domain. """ - # back off downloading more pages if we exceed this backlog size - DOWNLOADER_BACKLOG = 10 - def __init__(self): self.configured = False self.keep_alive = False @@ -179,9 +176,8 @@ class ExecutionEngine(object): if not self.running: return - # outstanding requests in downloader queue - outstanding = self.downloader.outstanding(domain) - if outstanding > self.DOWNLOADER_BACKLOG: + # backout enqueing downloads if domain needs it + if self.downloader.needs_backout(domain): return # Next pending request from scheduler