From 7910fa017243c6f64ecb55b4272894f6eb5d35ff Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Fri, 21 Nov 2014 01:09:32 -0200 Subject: [PATCH] Force to read DOWNLOAD_TIMEOUT as int (for example to pass using environment variable) --- scrapy/contrib/downloadermiddleware/downloadtimeout.py | 2 +- tests/test_downloadermiddleware_downloadtimeout.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py index 612b081ec..18123cfce 100644 --- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py +++ b/scrapy/contrib/downloadermiddleware/downloadtimeout.py @@ -14,7 +14,7 @@ class DownloadTimeoutMiddleware(object): @classmethod def from_crawler(cls, crawler): - o = cls(crawler.settings['DOWNLOAD_TIMEOUT']) + o = cls(crawler.settings.getfloat('DOWNLOAD_TIMEOUT')) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 3e3ff2401..aba06686a 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -8,8 +8,8 @@ from scrapy.utils.test import get_crawler class DownloadTimeoutMiddlewareTest(unittest.TestCase): - def get_request_spider_mw(self): - crawler = get_crawler(Spider) + def get_request_spider_mw(self, settings=None): + crawler = get_crawler(Spider, settings) spider = crawler._create_spider('foo') request = Request('http://scrapytest.org/') return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) @@ -20,6 +20,12 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta.get('download_timeout'), 180) + def test_string_download_timeout(self): + req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'}) + mw.spider_opened(spider) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta.get('download_timeout'), 20.1) + def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2