From dba7e39f61cbe2c22d3c9064f32f6e36d74f14b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 3 Aug 2015 10:53:40 -0300 Subject: [PATCH] Do not break cookie parsing on non-utf8 headers --- scrapy/http/cookies.py | 9 ++++++--- tests/test_downloadermiddleware_cookies.py | 18 +++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 740f21d24..e92c3fe73 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -149,11 +149,13 @@ class WrappedRequest(object): return name in self.request.headers def get_header(self, name, default=None): - return to_native_str(self.request.headers.get(name, default)) + return to_native_str(self.request.headers.get(name, default), + errors='replace') def header_items(self): return [ - (to_native_str(k), [to_native_str(x) for x in v]) + (to_native_str(k, errors='replace'), + [to_native_str(x, errors='replace') for x in v]) for k, v in self.request.headers.items() ] @@ -171,6 +173,7 @@ class WrappedResponse(object): # python3 cookiejars calls get_all def get_all(self, name, default=None): - return [to_native_str(v) for v in self.response.headers.getlist(name)] + return [to_native_str(v, errors='replace') + for v in self.response.headers.getlist(name)] # python2 cookiejars calls getheaders getheaders = get_all diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 6174f8c3f..63be0beb8 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -22,20 +22,32 @@ class CookiesMiddlewareTest(TestCase): del self.mw def test_basic(self): - headers = {'Set-Cookie': 'C1=value1; path=/'} req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None assert 'Cookie' not in req.headers + headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers) assert self.mw.process_response(req, res, self.spider) is res - #assert res.cookies - req2 = Request('http://scrapytest.org/sub1/') assert self.mw.process_request(req2, self.spider) is None self.assertEquals(req2.headers.get('Cookie'), b"C1=value1") + def test_do_not_break_on_non_utf8_header(self): + req = Request('http://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + assert 'Cookie' not in req.headers + + headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', + 'Other': b'ignore\xa3me'} + res = Response('http://scrapytest.org/', headers=headers) + assert self.mw.process_response(req, res, self.spider) is res + + req2 = Request('http://scrapytest.org/sub1/') + assert self.mw.process_request(req2, self.spider) is None + self.assertIn('Cookie', req2.headers) + def test_dont_merge_cookies(self): # merge some cookies into jar headers = {'Set-Cookie': 'C1=value1; path=/'}