From 1d09b6e1f2f559d149ef75eb8a50bce41084b4dd Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Tue, 8 Oct 2013 19:03:26 -0200 Subject: [PATCH] downloadtimeout mid multi spider support removed --- .../downloadermiddleware/downloadtimeout.py | 19 +++++++++---------- ...st_downloadermiddleware_downloadtimeout.py | 3 +++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py index ac75186cf..612b081ec 100644 --- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py +++ b/scrapy/contrib/downloadermiddleware/downloadtimeout.py @@ -3,25 +3,24 @@ Download timeout middleware See documentation in docs/topics/downloader-middleware.rst """ -from scrapy.utils.python import WeakKeyCache + +from scrapy import signals class DownloadTimeoutMiddleware(object): def __init__(self, timeout=180): - self._cache = WeakKeyCache(self._download_timeout) self._timeout = timeout @classmethod def from_crawler(cls, crawler): - return cls(crawler.settings['DOWNLOAD_TIMEOUT']) + o = cls(crawler.settings['DOWNLOAD_TIMEOUT']) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o - def _download_timeout(self, spider): - if hasattr(spider, 'download_timeout'): - return spider.download_timeout - return self._timeout + def spider_opened(self, spider): + self._timeout = getattr(spider, 'download_timeout', self._timeout) def process_request(self, request, spider): - timeout = self._cache[spider] - if timeout: - request.meta.setdefault('download_timeout', timeout) + if self._timeout: + request.meta.setdefault('download_timeout', self._timeout) diff --git a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py b/scrapy/tests/test_downloadermiddleware_downloadtimeout.py index 5066becd3..a7e920e78 100644 --- a/scrapy/tests/test_downloadermiddleware_downloadtimeout.py +++ b/scrapy/tests/test_downloadermiddleware_downloadtimeout.py @@ -17,18 +17,21 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): def test_default_download_timeout(self): req, spider, mw = self.get_request_spider_mw() + mw.spider_opened(spider) assert mw.process_request(req, spider) is None self.assertEquals(req.meta.get('download_timeout'), 180) def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 + mw.spider_opened(spider) assert mw.process_request(req, spider) is None self.assertEquals(req.meta.get('download_timeout'), 2) def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 + mw.spider_opened(spider) req.meta['download_timeout'] = 1 assert mw.process_request(req, spider) is None self.assertEquals(req.meta.get('download_timeout'), 1)