RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Backport][1.0] response_status_message should not fail on non-standard HTTP codes 

utility is used in retry middleware and it was failing to handle non-standard HTTP codes.
Instead of raising exceptions when passing through to_native_str it should return
"Unknown status" message.
This commit is contained in:
pawelmhm 2016-03-12 14:07:20 +01:00 committed by Paul Tremberth
parent 38b09085eb
commit 8efd6d26ad
2 changed files with 21 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/utils/response.py
View File

@ -55,7 +55,7 @@ def response_status_message(status):
"""
# Implicit decode/encode is on purpose to force native strings
# This is properly fixed in Scrapy >=1.1 at revision faf9265
reason = http.RESPONSES.get(int(status)).decode('utf8', errors='replace')
reason = http.RESPONSES.get(int(status), "Unknown Status").decode('utf8', errors='replace')
return '{} {}'.format(status, reason)
def response_httprepr(response):

21
tests/test_utils_response.py
View File

@ -3,7 +3,8 @@ import unittest
from six.moves.urllib.parse import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh
from scrapy.utils.response import (response_httprepr, open_in_browser,
get_meta_refresh, get_base_url, response_status_message)
__doctests__ = ['scrapy.utils.response']
@ -61,5 +62,23 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(get_meta_refresh(r2), (None, None))
self.assertEqual(get_meta_refresh(r3), (None, None))
def test_get_base_url(self):
resp = HtmlResponse("http://www.example.com", body=b"""
<html>
<head><base href="http://www.example.com/img/" target="_blank"></head>
<body>blahablsdfsal&amp;</body>
</html>""")
self.assertEqual(get_base_url(resp), "http://www.example.com/img/")
resp2 = HtmlResponse("http://www.example.com", body=b"""
<html><body>blahablsdfsal&amp;</body></html>""")
self.assertEqual(get_base_url(resp2), "http://www.example.com")
def test_response_status_message(self):
self.assertEqual(response_status_message(200), '200 OK')
self.assertEqual(response_status_message(404), '404 Not Found')
self.assertEqual(response_status_message(573), "573 Unknown Status")
if __name__ == "__main__":
unittest.main()