diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 2b7b8544b..5289d4e45 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -55,7 +55,7 @@ def response_status_message(status): """ # Implicit decode/encode is on purpose to force native strings # This is properly fixed in Scrapy >=1.1 at revision faf9265 - reason = http.RESPONSES.get(int(status)).decode('utf8', errors='replace') + reason = http.RESPONSES.get(int(status), "Unknown Status").decode('utf8', errors='replace') return '{} {}'.format(status, reason) def response_httprepr(response): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d78b25f25..b946ef842 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -80,8 +80,8 @@ class LinkExtractorTestCase(unittest.TestCase): ]) self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%F1.html', text=''), - Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')), + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), ]) def test_matches(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 92c92c057..ca768c7d6 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -3,7 +3,8 @@ import unittest from six.moves.urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse -from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh +from scrapy.utils.response import (response_httprepr, open_in_browser, + get_meta_refresh, get_base_url, response_status_message) __doctests__ = ['scrapy.utils.response'] @@ -61,5 +62,23 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(get_meta_refresh(r2), (None, None)) self.assertEqual(get_meta_refresh(r3), (None, None)) + def test_get_base_url(self): + resp = HtmlResponse("http://www.example.com", body=b""" + + + blahablsdfsal& + """) + self.assertEqual(get_base_url(resp), "http://www.example.com/img/") + + resp2 = HtmlResponse("http://www.example.com", body=b""" + blahablsdfsal&""") + self.assertEqual(get_base_url(resp2), "http://www.example.com") + + def test_response_status_message(self): + self.assertEqual(response_status_message(200), '200 OK') + self.assertEqual(response_status_message(404), '404 Not Found') + self.assertEqual(response_status_message(573), "573 Unknown Status") + + if __name__ == "__main__": unittest.main()