RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

downloadtimeout mid multi spider support removed

This commit is contained in:
nramirezuy 2013-10-08 19:03:26 -02:00
parent 3501c2025e
commit 1d09b6e1f2
2 changed files with 12 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
scrapy/contrib/downloadermiddleware/downloadtimeout.py
View File

@ -3,25 +3,24 @@ Download timeout middleware
See documentation in docs/topics/downloader-middleware.rst
"""
from scrapy.utils.python import WeakKeyCache
from scrapy import signals
class DownloadTimeoutMiddleware(object):
def __init__(self, timeout=180):
self._cache = WeakKeyCache(self._download_timeout)
self._timeout = timeout
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings['DOWNLOAD_TIMEOUT'])
o = cls(crawler.settings['DOWNLOAD_TIMEOUT'])
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
def _download_timeout(self, spider):
if hasattr(spider, 'download_timeout'):
return spider.download_timeout
return self._timeout
def spider_opened(self, spider):
self._timeout = getattr(spider, 'download_timeout', self._timeout)
def process_request(self, request, spider):
timeout = self._cache[spider]
if timeout:
request.meta.setdefault('download_timeout', timeout)
if self._timeout:
request.meta.setdefault('download_timeout', self._timeout)

3
scrapy/tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -17,18 +17,21 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
def test_default_download_timeout(self):
req, spider, mw = self.get_request_spider_mw()
mw.spider_opened(spider)
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta.get('download_timeout'), 180)
def test_spider_has_download_timeout(self):
req, spider, mw = self.get_request_spider_mw()
spider.download_timeout = 2
mw.spider_opened(spider)
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta.get('download_timeout'), 2)
def test_request_has_download_timeout(self):
req, spider, mw = self.get_request_spider_mw()
spider.download_timeout = 2
mw.spider_opened(spider)
req.meta['download_timeout'] = 1
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta.get('download_timeout'), 1)