RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make RetryMiddleware obey Request.meta 'dont_retry' key when processing exceptions. Closes #259

This commit is contained in:
Pablo Hoffman 2010-10-11 21:28:42 -02:00
parent b4fbc6c5fa
commit f5b188b179
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/contrib/downloadermiddleware/retry.py
View File

@ -51,7 +51,8 @@ class RetryMiddleware(object):
return response
def process_exception(self, request, exception, spider):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
and 'dont_retry' not in request.meta:
return self._retry(request, exception, spider)
def _retry(self, request, reason, spider):

7
scrapy/tests/test_downloadermiddleware_retry.py
View File

@ -35,6 +35,13 @@ class RetryTest(unittest.TestCase):
r = self.mw.process_response(req, rsp, self.spider)
assert r is rsp
def test_dont_retry_exc(self):
req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True})
rsp = Response('http://www.scrapytest.org/503', body='', status=503)
r = self.mw.process_exception(req, DNSLookupError(), self.spider)
assert r is None
def test_503(self):
req = Request('http://www.scrapytest.org/503')
rsp = Response('http://www.scrapytest.org/503', body='', status=503)