Do not break cookie parsing on non-utf8 headers

scrapy/http/cookies.py

@@ -149,11 +149,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default))
+        return to_native_str(self.request.headers.get(name, default),
+                             errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k), [to_native_str(x) for x in v])
+            (to_native_str(k, errors='replace'),
+             [to_native_str(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -171,6 +173,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v) for v in self.response.headers.getlist(name)]
+        return [to_native_str(v, errors='replace')
+                for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all

tests/test_downloadermiddleware_cookies.py

@@ -22,20 +22,32 @@ class CookiesMiddlewareTest(TestCase):
         del self.mw
 
     def test_basic(self):
-        headers = {'Set-Cookie': 'C1=value1; path=/'}
         req = Request('http://scrapytest.org/')
         assert self.mw.process_request(req, self.spider) is None
         assert 'Cookie' not in req.headers
 
+        headers = {'Set-Cookie': 'C1=value1; path=/'}
         res = Response('http://scrapytest.org/', headers=headers)
         assert self.mw.process_response(req, res, self.spider) is res
 
-        #assert res.cookies
-
         req2 = Request('http://scrapytest.org/sub1/')
         assert self.mw.process_request(req2, self.spider) is None
         self.assertEquals(req2.headers.get('Cookie'), b"C1=value1")
 
+    def test_do_not_break_on_non_utf8_header(self):
+        req = Request('http://scrapytest.org/')
+        assert self.mw.process_request(req, self.spider) is None
+        assert 'Cookie' not in req.headers
+
+        headers = {'Set-Cookie': b'C1=in\xa3valid; path=/',
+                   'Other': b'ignore\xa3me'}
+        res = Response('http://scrapytest.org/', headers=headers)
+        assert self.mw.process_response(req, res, self.spider) is res
+
+        req2 = Request('http://scrapytest.org/sub1/')
+        assert self.mw.process_request(req2, self.spider) is None
+        self.assertIn('Cookie', req2.headers)
+
     def test_dont_merge_cookies(self):
         # merge some cookies into jar
         headers = {'Set-Cookie': 'C1=value1; path=/'}