RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #954 from kalessin/int-download-timeout 

Force to read DOWNLOAD_TIMEOUT as int (for example to pass using environment variable)
This commit is contained in:
Pablo Hoffman 2014-11-26 17:14:18 -02:00
parent dedea72774 7910fa0172
commit c31fb87335
2 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/contrib/downloadermiddleware/downloadtimeout.py
View File

@ -14,7 +14,7 @@ class DownloadTimeoutMiddleware(object):
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.settings['DOWNLOAD_TIMEOUT'])
o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT'))
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o

10
tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -8,8 +8,8 @@ from scrapy.utils.test import get_crawler
class DownloadTimeoutMiddlewareTest(unittest.TestCase):
def get_request_spider_mw(self):
crawler = get_crawler(Spider)
def get_request_spider_mw(self, settings=None):
crawler = get_crawler(Spider, settings)
spider = crawler._create_spider('foo')
request = Request('http://scrapytest.org/')
return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
@ -20,6 +20,12 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase):
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta.get('download_timeout'), 180)
def test_string_download_timeout(self):
req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'})
mw.spider_opened(spider)
assert mw.process_request(req, spider) is None
self.assertEquals(req.meta.get('download_timeout'), 20.1)
def test_spider_has_download_timeout(self):
req, spider, mw = self.get_request_spider_mw()
spider.download_timeout = 2