mirror of https://github.com/scrapy/scrapy.git
added comment downloader backout policy
This commit is contained in:
parent
2ae11e9220
commit
dad05957f3
|
|
@ -42,6 +42,7 @@ class SiteInfo(object):
|
|||
return self.max_concurrent_requests - len(self.transferring)
|
||||
|
||||
def needs_backout(self):
|
||||
# use self.active to include requests in the downloader middleware
|
||||
return len(self.active) > 2 * self.max_concurrent_requests
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue