From 8efd6d26ad3d0d776411436201a0156e7e5641a3 Mon Sep 17 00:00:00 2001 From: pawelmhm Date: Sat, 12 Mar 2016 14:07:20 +0100 Subject: [PATCH 1/2] [Backport][1.0] response_status_message should not fail on non-standard HTTP codes utility is used in retry middleware and it was failing to handle non-standard HTTP codes. Instead of raising exceptions when passing through to_native_str it should return "Unknown status" message. --- scrapy/utils/response.py | 2 +- tests/test_utils_response.py | 21 ++++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 2b7b8544b..5289d4e45 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -55,7 +55,7 @@ def response_status_message(status): """ # Implicit decode/encode is on purpose to force native strings # This is properly fixed in Scrapy >=1.1 at revision faf9265 - reason = http.RESPONSES.get(int(status)).decode('utf8', errors='replace') + reason = http.RESPONSES.get(int(status), "Unknown Status").decode('utf8', errors='replace') return '{} {}'.format(status, reason) def response_httprepr(response): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 92c92c057..ca768c7d6 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -3,7 +3,8 @@ import unittest from six.moves.urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse -from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh +from scrapy.utils.response import (response_httprepr, open_in_browser, + get_meta_refresh, get_base_url, response_status_message) __doctests__ = ['scrapy.utils.response'] @@ -61,5 +62,23 @@ class ResponseUtilsTest(unittest.TestCase): self.assertEqual(get_meta_refresh(r2), (None, None)) self.assertEqual(get_meta_refresh(r3), (None, None)) + def test_get_base_url(self): + resp = HtmlResponse("http://www.example.com", body=b""" + + + blahablsdfsal& + """) + self.assertEqual(get_base_url(resp), "http://www.example.com/img/") + + resp2 = HtmlResponse("http://www.example.com", body=b""" + blahablsdfsal&""") + self.assertEqual(get_base_url(resp2), "http://www.example.com") + + def test_response_status_message(self): + self.assertEqual(response_status_message(200), '200 OK') + self.assertEqual(response_status_message(404), '404 Not Found') + self.assertEqual(response_status_message(573), "573 Unknown Status") + + if __name__ == "__main__": unittest.main() From 5bf10cc8ffd54e20199a8d58a861541605fde2f1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 7 Apr 2016 22:43:47 +0200 Subject: [PATCH 2/2] Fix tests on URL path encoding for links from latin1 document UTF-8 is to be used for path component. Page encoding only affects query part of URLs. --- tests/test_linkextractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d78b25f25..b946ef842 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -80,8 +80,8 @@ class LinkExtractorTestCase(unittest.TestCase): ]) self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%F1.html', text=''), - Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')), + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), ]) def test_matches(self):