RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

redirect mw: remove body and content-type/content-length headers on 302 redirects, and added tests. also renamed some tests to more meaningful names 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401074
This commit is contained in:
Pablo Hoffman 2009-04-21 01:38:11 +00:00
parent 5861b345c7
commit 3170dd9fdb
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -23,7 +23,9 @@ class RedirectMiddleware(object):
if status in [302, 303]:
redirected_url = urljoin(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url, method='GET', body=None)
redirected = request.replace(url=redirected_url, method='GET', body='')
redirected.headers.pop('Content-Type', None)
redirected.headers.pop('Content-Length', None)
return self._redirect(redirected, request, spider, status)
if status in [301, 307]:

15
scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py
View File

@ -18,7 +18,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
def tearDown(self):
dupefilter.close('scrapytest.org')
def test_process_exception(self):
def test_redirect_301(self):
url = 'http://www.example.com/301'
url2 = 'http://www.example.com/redirected'
req = Request(url)
@ -30,9 +30,11 @@ class RedirectMiddlewareTest(unittest.TestCase):
assert isinstance(req2, Request)
self.assertEqual(req2.url, url2)
def test_redirect_302(self):
url = 'http://www.example.com/302'
url2 = 'http://www.example.com/redirected2'
req = Request(url, method='POST')
req = Request(url, method='POST', body='test',
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
hdr = Headers({'Location': [url2]})
rsp = Response(url, headers=hdr)
exc = HttpException('302', None, rsp)
@ -41,9 +43,14 @@ class RedirectMiddlewareTest(unittest.TestCase):
assert isinstance(req2, Request)
self.assertEqual(req2.url, url2)
self.assertEqual(req2.method, 'GET')
assert not req2.body
assert 'Content-Type' not in req2.headers, \
"Content-Type header must not be present in redirected request"
assert 'Content-Length' not in req2.headers, \
"Content-Length header must not be present in redirected request"
assert not req2.body, \
"Redirected body must be empty, not '%s'" % req2.body
def test_process_response(self):
def test_meta_refresh(self):
body = """<html>
<head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
</html>"""