From 743d3261841b221216482250a16114018c768cf6 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 11 Aug 2009 12:39:02 -0300 Subject: [PATCH] returning None from process_response is not allowed, ignore the request using exception instead --- scrapy/contrib/downloadermiddleware/redirect.py | 2 ++ scrapy/tests/test_downloadermiddleware_redirect.py | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index acf109cdc..f6ae43733 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,6 +1,7 @@ from scrapy import log from scrapy.utils.url import urljoin_rfc from scrapy.utils.response import get_meta_refresh +from scrapy.core.exceptions import IgnoreRequest from scrapy.conf import settings @@ -49,5 +50,6 @@ class RedirectMiddleware(object): else: log.msg("Discarding %s: max redirections reached" % request, domain=spider.domain_name, level=log.DEBUG) + raise IgnoreRequest diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index 0db49e485..3838ea9b2 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -2,6 +2,7 @@ import unittest from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware from scrapy.spider import spiders +from scrapy.core.exceptions import IgnoreRequest from scrapy.http import Request, Response, Headers class RedirectMiddlewareTest(unittest.TestCase): @@ -84,9 +85,7 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(req, Request) assert 'redirect_times' in req.meta self.assertEqual(req.meta['redirect_times'], 1) - - req = self.mw.process_response(req, rsp, self.spider) - self.assertEqual(req, None) + self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider) def test_ttl(self): self.mw.max_redirect_times = 100 @@ -95,8 +94,7 @@ class RedirectMiddlewareTest(unittest.TestCase): req = self.mw.process_response(req, rsp, self.spider) assert isinstance(req, Request) - req = self.mw.process_response(req, rsp, self.spider) - self.assertEqual(req, None) + self.assertRaises(IgnoreRequest, self.mw.process_response, req, rsp, self.spider) if __name__ == "__main__": unittest.main()