From 966bd49c421fdf40a8b21b49c76cd54ded06fe50 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Tue, 14 Mar 2017 16:23:47 +0530 Subject: [PATCH] Update unittest for meta['max_retry_times'] --- tests/test_downloadermiddleware_retry.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index b833cb448..cc3d37075 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -103,6 +103,34 @@ class RetryTest(unittest.TestCase): req = self.mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) + def test_different_retry(self): + + req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 1}) + self._test_retry(req, DNSLookupError('foo')) + req2 = Request('http://www.scrapytest.org/invalid_url') + self._test_retry(req2, DNSLookupError('foo')) + + stats = self.crawler.stats + assert stats.get_value('retry/max_reached') == 2 + assert stats.get_value('retry/count') == 3 + + def _test_retry(self, req, exception): + + req = self.mw.process_exception(req, exception, self.spider) + assert isinstance(req, Request) + + retry_times = req.meta.get('max_retry_times') or self.mw.max_retry_times + + while req.meta['retry_times'] != retry_times: + req = self.mw.process_exception(req, exception, self.spider) + assert isinstance(req, Request) + + self.assertEqual(req.meta['retry_times'], retry_times) + + # discard it + req = self.mw.process_exception(req, exception, self.spider) + self.assertEqual(req, None) + if __name__ == "__main__": unittest.main()