From f6ed5edc31e7cc66225c0860e1534a6230511954 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 18 Nov 2016 09:14:54 -0300 Subject: [PATCH] CookiesMiddleware: keep cookies from 'Cookie' request header --- docs/topics/downloader-middleware.rst | 5 + docs/topics/logging.rst | 3 + scrapy/downloadermiddlewares/cookies.py | 78 ++++++++--- tests/test_downloadermiddleware_cookies.py | 145 ++++++++++++++++++--- 4 files changed, 190 insertions(+), 41 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1a87d07b6..323e553e5 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -202,6 +202,11 @@ CookiesMiddleware sends them back on subsequent requests (from that spider), just like web browsers do. + .. caution:: When non-UTF8 encoded byte sequences are passed to a + :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + a warning. Refer to :ref:`topics-logging-advanced-customization` + to customize the logging behaviour. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index e81091651..55065a1a3 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending .. autoclass:: scrapy.logformatter.LogFormatter :members: + +.. _topics-logging-advanced-customization: + Advanced customization ---------------------- diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d57f04bc3..77048f389 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -29,8 +29,7 @@ class CookiesMiddleware: cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - cookies = self._get_request_cookies(jar, request) - for cookie in cookies: + for cookie in self._get_request_cookies(jar, request): jar.set_cookie_if_ok(cookie, request) # set Cookie header @@ -68,28 +67,65 @@ class CookiesMiddleware: msg = "Received cookies from: {}\n{}".format(response, cookies) logger.debug(msg, extra={'spider': spider}) - def _format_cookie(self, cookie): - # build cookie string - cookie_str = '%s=%s' % (cookie['name'], cookie['value']) - - if cookie.get('path', None): - cookie_str += '; Path=%s' % cookie['path'] - if cookie.get('domain', None): - cookie_str += '; Domain=%s' % cookie['domain'] + def _format_cookie(self, cookie, request): + """ + Given a dict consisting of cookie components, return its string representation. + Decode from bytes if necessary. + """ + decoded = {} + for key in ("name", "value", "path", "domain"): + if not cookie.get(key): + if key in ("name", "value"): + msg = "Invalid cookie found in request {}: {} ('{}' is missing)" + logger.warning(msg.format(request, cookie, key)) + return + continue + if isinstance(cookie[key], str): + decoded[key] = cookie[key] + else: + try: + decoded[key] = cookie[key].decode("utf8") + except UnicodeDecodeError: + logger.warning("Non UTF-8 encoded cookie found in request %s: %s", + request, cookie) + decoded[key] = cookie[key].decode("latin1", errors="replace") + cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) + for key, value in decoded.items(): # path, domain + cookie_str += "; {}={}".format(key.capitalize(), value) return cookie_str def _get_request_cookies(self, jar, request): - if isinstance(request.cookies, dict): - cookie_list = [ - {'name': k, 'value': v} - for k, v in request.cookies.items() - ] - else: - cookie_list = request.cookies + """ + Extract cookies from a Request. Values from the `Request.cookies` attribute + take precedence over values from the `Cookie` request header. + """ + def get_cookies_from_header(jar, request): + cookie_header = request.headers.get("Cookie") + if not cookie_header: + return [] + cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";")) + cookie_list_unicode = [] + for cookie_bytes in cookie_gen_bytes: + try: + cookie_unicode = cookie_bytes.decode("utf8") + except UnicodeDecodeError: + logger.warning("Non UTF-8 encoded cookie found in request %s: %s", + request, cookie_bytes) + cookie_unicode = cookie_bytes.decode("latin1", errors="replace") + cookie_list_unicode.append(cookie_unicode) + response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode}) + return jar.make_cookies(response, request) - cookies = [self._format_cookie(x) for x in cookie_list] - headers = {'Set-Cookie': cookies} - response = Response(request.url, headers=headers) + def get_cookies_from_attribute(jar, request): + if not request.cookies: + return [] + elif isinstance(request.cookies, dict): + cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + else: + cookies = request.cookies + formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) + response = Response(request.url, headers={"Set-Cookie": formatted}) + return jar.make_cookies(response, request) - return jar.make_cookies(response, request) + return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index d54434c8f..9ccc2110b 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,20 +1,21 @@ -import re import logging -from unittest import TestCase from testfixtures import LogCapture +from unittest import TestCase +from scrapy.downloadermiddlewares.cookies import CookiesMiddleware +from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from scrapy.exceptions import NotConfigured -from scrapy.downloadermiddlewares.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): - return sorted(re.split(r";\s*", cookies.decode("latin1"))) + return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) def setUp(self): @@ -61,12 +62,13 @@ class CookiesMiddlewareTest(TestCase): def test_setting_enabled_cookies_debug(self): crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture('scrapy.downloadermiddlewares.cookies', - propagate=False, - level=logging.DEBUG) as log: + with LogCapture( + 'scrapy.downloadermiddlewares.cookies', + propagate=False, + level=logging.DEBUG, + ) as log: req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', - headers={'Set-Cookie': 'C1=value1; path=/'}) + res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) mw.process_response(req, res, crawler.spider) req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) @@ -85,12 +87,13 @@ class CookiesMiddlewareTest(TestCase): def test_setting_disabled_cookies_debug(self): crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture('scrapy.downloadermiddlewares.cookies', - propagate=False, - level=logging.DEBUG) as log: + with LogCapture( + 'scrapy.downloadermiddlewares.cookies', + propagate=False, + level=logging.DEBUG, + ) as log: req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', - headers={'Set-Cookie': 'C1=value1; path=/'}) + res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) mw.process_response(req, res, crawler.spider) req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) @@ -102,8 +105,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req, self.spider) is None assert 'Cookie' not in req.headers - headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', - 'Other': b'ignore\xa3me'} + headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'} res = Response('http://scrapytest.org/', headers=headers) assert self.mw.process_response(req, res, self.spider) is res @@ -124,7 +126,10 @@ class CookiesMiddlewareTest(TestCase): assert 'Cookie' not in req.headers # check that returned cookies are not merged back to jar - res = Response('http://scrapytest.org/dontmerge', headers={'Set-Cookie': 'dont=mergeme; path=/'}) + res = Response( + 'http://scrapytest.org/dontmerge', + headers={'Set-Cookie': 'dont=mergeme; path=/'}, + ) assert self.mw.process_response(req, res, self.spider) is res # check that cookies are merged back @@ -179,7 +184,11 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada") def test_cookiejar_key(self): - req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"}) + req = Request( + 'http://scrapytest.org/', + cookies={'galleta': 'salada'}, + meta={'cookiejar': "store1"}, + ) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') @@ -191,7 +200,11 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada') - req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"}) + req3 = Request( + 'http://scrapytest.org/', + cookies={'galleta': 'dulce'}, + meta={'cookiejar': "store2"}, + ) assert self.mw.process_request(req3, self.spider) is None self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce') @@ -229,3 +242,95 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(request, self.spider) is None self.assertIn('Cookie', request.headers) self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + + def test_keep_cookie_from_default_request_headers_middleware(self): + DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') + mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) + # overwrite with values from 'cookies' request argument + req1 = Request('http://example.org', cookies={'default': 'something'}) + assert mw_default_headers.process_request(req1, self.spider) is None + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty') + # keep both + req2 = Request('http://example.com', cookies={'a': 'b'}) + assert mw_default_headers.process_request(req2, self.spider) is None + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + + def test_keep_cookie_header(self): + # keep only cookies from 'Cookie' request header + req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d') + # keep cookies from both 'Cookie' request header and 'cookies' keyword + req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f') + # overwrite values from 'Cookie' request header with 'cookies' keyword + req3 = Request( + 'http://scrapytest.org', + headers={'Cookie': 'a=b; c=d'}, + cookies={'a': 'new', 'e': 'f'}, + ) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f') + + def test_request_cookies_encoding(self): + # 1) UTF8-encoded bytes + req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') + + # 2) Non UTF8-encoded bytes + req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') + + # 3) Unicode string + req3 = Request('http://example.org', cookies={'a': u'á'}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + + def test_request_headers_cookie_encoding(self): + # 1) UTF8-encoded bytes + req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') + + # 2) Non UTF8-encoded bytes + req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') + + # 3) Unicode string + req3 = Request('http://example.org', headers={'Cookie': u'a=á'}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + + def test_invalid_cookies(self): + """ + Invalid cookies are logged as warnings and discarded + """ + with LogCapture( + 'scrapy.downloadermiddlewares.cookies', + propagate=False, + level=logging.INFO, + ) as lc: + cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}] + req1 = Request('http://example.org/1', cookies=cookies1) + assert self.mw.process_request(req1, self.spider) is None + cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] + req2 = Request('http://example.org/2', cookies=cookies2) + assert self.mw.process_request(req2, self.spider) is None + lc.check( + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'value': 'bar'} ('name' is missing)"), + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request :" + " {'name': 'foo'} ('value' is missing)"), + ) + self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') + self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')