From 3170dd9fdbce4baf377d0ea4751e57ec97ae6c37 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 21 Apr 2009 01:38:11 +0000 Subject: [PATCH] redirect mw: remove body and content-type/content-length headers on 302 redirects, and added tests. also renamed some tests to more meaningful names --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401074 --- .../contrib/downloadermiddleware/redirect.py | 4 +++- .../tests/test_downloadermiddleware_redirect.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py index 8c7ed9af4..13da6db0f 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -23,7 +23,9 @@ class RedirectMiddleware(object): if status in [302, 303]: redirected_url = urljoin(request.url, response.headers['location']) - redirected = request.replace(url=redirected_url, method='GET', body=None) + redirected = request.replace(url=redirected_url, method='GET', body='') + redirected.headers.pop('Content-Type', None) + redirected.headers.pop('Content-Length', None) return self._redirect(redirected, request, spider, status) if status in [301, 307]: diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py index bea2a7b1e..5d24ae4e3 100644 --- a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py @@ -18,7 +18,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def tearDown(self): dupefilter.close('scrapytest.org') - def test_process_exception(self): + def test_redirect_301(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' req = Request(url) @@ -30,9 +30,11 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, url2) + def test_redirect_302(self): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' - req = Request(url, method='POST') + req = Request(url, method='POST', body='test', + headers={'Content-Type': 'text/plain', 'Content-length': '4'}) hdr = Headers({'Location': [url2]}) rsp = Response(url, headers=hdr) exc = HttpException('302', None, rsp) @@ -41,9 +43,14 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, url2) self.assertEqual(req2.method, 'GET') - assert not req2.body + assert 'Content-Type' not in req2.headers, \ + "Content-Type header must not be present in redirected request" + assert 'Content-Length' not in req2.headers, \ + "Content-Length header must not be present in redirected request" + assert not req2.body, \ + "Redirected body must be empty, not '%s'" % req2.body - def test_process_response(self): + def test_meta_refresh(self): body = """ """