RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #106 from kalessin/downloader2 

dont discard slot when empty, just save in another dict in order to recycle if needed again
This commit is contained in:
Pablo Hoffman 2012-04-02 14:39:38 -07:00
parent e9184def35 b39cb22d83
commit c27f7eb7e9
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapy/core/downloader/__init__.py
View File

@ -75,7 +75,7 @@ class Downloader(object):
self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
self.inactive_slots = {}
def fetch(self, request, spider):
key, slot = self._get_slot(request, spider)
@ -86,7 +86,7 @@ class Downloader(object):
self.active.remove(request)
slot.active.remove(request)
if not slot.active: # remove empty slots
del self.slots[key]
self.inactive_slots[key] = self.slots.pop(key)
return response
dlfunc = partial(self._enqueue_request, slot=slot)
@ -101,12 +101,15 @@ class Downloader(object):
if self.ip_concurrency:
key = dnscache.get(key, key)
if key not in self.slots:
if self.ip_concurrency:
concurrency = self.ip_concurrency
if key in self.inactive_slots:
self.slots[key] = self.inactive_slots.pop(key)
else:
concurrency = self.domain_concurrency
concurrency, delay = _get_concurrency_delay(concurrency, spider, self.settings)
self.slots[key] = Slot(concurrency, delay, self.settings)
if self.ip_concurrency:
concurrency = self.ip_concurrency
else:
concurrency = self.domain_concurrency
concurrency, delay = _get_concurrency_delay(concurrency, spider, self.settings)
self.slots[key] = Slot(concurrency, delay, self.settings)
return key, self.slots[key]
def _enqueue_request(self, request, spider, slot):