RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

redirect: 3xx status code based redirection requires Location header to be set

This commit is contained in:
Daniel Grana 2009-05-08 12:01:02 -03:00
parent 05f1e0c12d
commit 3dee4b6728
2 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -21,14 +21,14 @@ class RedirectMiddleware(object):
status = exception.status
response = exception.response
if status in [302, 303]:
if status in [302, 303] and 'Location' in response.headers:
redirected_url = urljoin(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url, method='GET', body='')
redirected.headers.pop('Content-Type', None)
redirected.headers.pop('Content-Length', None)
return self._redirect(redirected, request, spider, status)
if status in [301, 307]:
if status in [301, 307] and 'Location' in response.headers:
redirected_url = urljoin(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, status)

8
scrapy/tests/test_downloadermiddleware_redirect.py
View File

@ -30,6 +30,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
assert isinstance(req2, Request)
self.assertEqual(req2.url, url2)
# response without Location header but with status code is 3XX should be ignored
del rsp.headers['Location']
assert self.mw.process_exception(req, exc, self.spider) is None
def test_redirect_302(self):
url = 'http://www.example.com/302'
url2 = 'http://www.example.com/redirected2'
@ -50,6 +54,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
assert not req2.body, \
"Redirected body must be empty, not '%s'" % req2.body
# response without Location header but with status code is 3XX should be ignored
del rsp.headers['Location']
assert self.mw.process_exception(req, exc, self.spider) is None
def test_meta_refresh(self):
body = """<html>
<head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>