RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1906 from redapple/backport-1.0-pr1857 

[Backport][1.0] response_status_message should not fail on non-standard HTTP codes
This commit is contained in:
Paul Tremberth 2016-04-07 22:56:09 +02:00
parent 38b09085eb 5bf10cc8ff
commit fd0570e2f8
3 changed files with 23 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/utils/response.py
View File

@ -55,7 +55,7 @@ def response_status_message(status):
"""
# Implicit decode/encode is on purpose to force native strings
# This is properly fixed in Scrapy >=1.1 at revision faf9265
reason = http.RESPONSES.get(int(status)).decode('utf8', errors='replace')
reason = http.RESPONSES.get(int(status), "Unknown Status").decode('utf8', errors='replace')
return '{} {}'.format(status, reason)
def response_httprepr(response):

4
tests/test_linkextractors.py
View File

@ -80,8 +80,8 @@ class LinkExtractorTestCase(unittest.TestCase):
])
self.assertEqual(lx.extract_links(response_latin1), [
Link(url='http://example.com/sample_%F1.html', text=''),
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
Link(url='http://example.com/sample_%C3%B1.html', text=''),
Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
])
def test_matches(self):

21
tests/test_utils_response.py
View File

@ -3,7 +3,8 @@ import unittest
from six.moves.urllib.parse import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh
from scrapy.utils.response import (response_httprepr, open_in_browser,
get_meta_refresh, get_base_url, response_status_message)
__doctests__ = ['scrapy.utils.response']
@ -61,5 +62,23 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertEqual(get_meta_refresh(r2), (None, None))
self.assertEqual(get_meta_refresh(r3), (None, None))
def test_get_base_url(self):
resp = HtmlResponse("http://www.example.com", body=b"""
<html>
<head><base href="http://www.example.com/img/" target="_blank"></head>
<body>blahablsdfsal&amp;</body>
</html>""")
self.assertEqual(get_base_url(resp), "http://www.example.com/img/")
resp2 = HtmlResponse("http://www.example.com", body=b"""
<html><body>blahablsdfsal&amp;</body></html>""")
self.assertEqual(get_base_url(resp2), "http://www.example.com")
def test_response_status_message(self):
self.assertEqual(response_status_message(200), '200 OK')
self.assertEqual(response_status_message(404), '404 Not Found')
self.assertEqual(response_status_message(573), "573 Unknown Status")
if __name__ == "__main__":
unittest.main()