mirror of https://github.com/scrapy/scrapy.git
Merge pull request #2400 from elacuesta/keep_cookie_header
[MRG+1] CookiesMiddleware: keep cookies from 'Cookie' request header, fix encoding
This commit is contained in:
Andrey Rahmatullin
GitHub
commit
7bf375098e
|
|
@ -202,6 +202,11 @@ CookiesMiddleware
|
|||
sends them back on subsequent requests (from that spider), just like web
|
||||
browsers do.
|
||||
|
||||
.. caution:: When non-UTF8 encoded byte sequences are passed to a
|
||||
:class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
|
||||
a warning. Refer to :ref:`topics-logging-advanced-customization`
|
||||
to customize the logging behaviour.
|
||||
|
||||
The following settings can be used to configure the cookie middleware:
|
||||
|
||||
* :setting:`COOKIES_ENABLED`
|
||||
|
|
|
|||
|
|
@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending
|
|||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
||||
|
||||
.. _topics-logging-advanced-customization:
|
||||
|
||||
Advanced customization
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -29,8 +29,7 @@ class CookiesMiddleware:
|
|||
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
cookies = self._get_request_cookies(jar, request)
|
||||
for cookie in cookies:
|
||||
for cookie in self._get_request_cookies(jar, request):
|
||||
jar.set_cookie_if_ok(cookie, request)
|
||||
|
||||
# set Cookie header
|
||||
|
|
@ -68,28 +67,65 @@ class CookiesMiddleware:
|
|||
msg = "Received cookies from: {}\n{}".format(response, cookies)
|
||||
logger.debug(msg, extra={'spider': spider})
|
||||
|
||||
def _format_cookie(self, cookie):
|
||||
# build cookie string
|
||||
cookie_str = '%s=%s' % (cookie['name'], cookie['value'])
|
||||
|
||||
if cookie.get('path', None):
|
||||
cookie_str += '; Path=%s' % cookie['path']
|
||||
if cookie.get('domain', None):
|
||||
cookie_str += '; Domain=%s' % cookie['domain']
|
||||
def _format_cookie(self, cookie, request):
|
||||
"""
|
||||
Given a dict consisting of cookie components, return its string representation.
|
||||
Decode from bytes if necessary.
|
||||
"""
|
||||
decoded = {}
|
||||
for key in ("name", "value", "path", "domain"):
|
||||
if not cookie.get(key):
|
||||
if key in ("name", "value"):
|
||||
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
|
||||
logger.warning(msg.format(request, cookie, key))
|
||||
return
|
||||
continue
|
||||
if isinstance(cookie[key], str):
|
||||
decoded[key] = cookie[key]
|
||||
else:
|
||||
try:
|
||||
decoded[key] = cookie[key].decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request, cookie)
|
||||
decoded[key] = cookie[key].decode("latin1", errors="replace")
|
||||
|
||||
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
|
||||
for key, value in decoded.items(): # path, domain
|
||||
cookie_str += "; {}={}".format(key.capitalize(), value)
|
||||
return cookie_str
|
||||
|
||||
def _get_request_cookies(self, jar, request):
|
||||
if isinstance(request.cookies, dict):
|
||||
cookie_list = [
|
||||
{'name': k, 'value': v}
|
||||
for k, v in request.cookies.items()
|
||||
]
|
||||
else:
|
||||
cookie_list = request.cookies
|
||||
"""
|
||||
Extract cookies from a Request. Values from the `Request.cookies` attribute
|
||||
take precedence over values from the `Cookie` request header.
|
||||
"""
|
||||
def get_cookies_from_header(jar, request):
|
||||
cookie_header = request.headers.get("Cookie")
|
||||
if not cookie_header:
|
||||
return []
|
||||
cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
|
||||
cookie_list_unicode = []
|
||||
for cookie_bytes in cookie_gen_bytes:
|
||||
try:
|
||||
cookie_unicode = cookie_bytes.decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request, cookie_bytes)
|
||||
cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
|
||||
cookie_list_unicode.append(cookie_unicode)
|
||||
response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
|
||||
return jar.make_cookies(response, request)
|
||||
|
||||
cookies = [self._format_cookie(x) for x in cookie_list]
|
||||
headers = {'Set-Cookie': cookies}
|
||||
response = Response(request.url, headers=headers)
|
||||
def get_cookies_from_attribute(jar, request):
|
||||
if not request.cookies:
|
||||
return []
|
||||
elif isinstance(request.cookies, dict):
|
||||
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
cookies = request.cookies
|
||||
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
|
||||
response = Response(request.url, headers={"Set-Cookie": formatted})
|
||||
return jar.make_cookies(response, request)
|
||||
|
||||
return jar.make_cookies(response, request)
|
||||
return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
import re
|
||||
import logging
|
||||
from unittest import TestCase
|
||||
from testfixtures import LogCapture
|
||||
from unittest import TestCase
|
||||
|
||||
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
|
||||
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.test import get_crawler
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
|
||||
|
||||
|
||||
class CookiesMiddlewareTest(TestCase):
|
||||
|
||||
def assertCookieValEqual(self, first, second, msg=None):
|
||||
def split_cookies(cookies):
|
||||
return sorted(re.split(r";\s*", cookies.decode("latin1")))
|
||||
return sorted([s.strip() for s in to_bytes(cookies).split(b";")])
|
||||
return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -61,12 +62,13 @@ class CookiesMiddlewareTest(TestCase):
|
|||
def test_setting_enabled_cookies_debug(self):
|
||||
crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True})
|
||||
mw = CookiesMiddleware.from_crawler(crawler)
|
||||
with LogCapture('scrapy.downloadermiddlewares.cookies',
|
||||
propagate=False,
|
||||
level=logging.DEBUG) as log:
|
||||
with LogCapture(
|
||||
'scrapy.downloadermiddlewares.cookies',
|
||||
propagate=False,
|
||||
level=logging.DEBUG,
|
||||
) as log:
|
||||
req = Request('http://scrapytest.org/')
|
||||
res = Response('http://scrapytest.org/',
|
||||
headers={'Set-Cookie': 'C1=value1; path=/'})
|
||||
res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
|
||||
mw.process_response(req, res, crawler.spider)
|
||||
req2 = Request('http://scrapytest.org/sub1/')
|
||||
mw.process_request(req2, crawler.spider)
|
||||
|
|
@ -85,12 +87,13 @@ class CookiesMiddlewareTest(TestCase):
|
|||
def test_setting_disabled_cookies_debug(self):
|
||||
crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False})
|
||||
mw = CookiesMiddleware.from_crawler(crawler)
|
||||
with LogCapture('scrapy.downloadermiddlewares.cookies',
|
||||
propagate=False,
|
||||
level=logging.DEBUG) as log:
|
||||
with LogCapture(
|
||||
'scrapy.downloadermiddlewares.cookies',
|
||||
propagate=False,
|
||||
level=logging.DEBUG,
|
||||
) as log:
|
||||
req = Request('http://scrapytest.org/')
|
||||
res = Response('http://scrapytest.org/',
|
||||
headers={'Set-Cookie': 'C1=value1; path=/'})
|
||||
res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'})
|
||||
mw.process_response(req, res, crawler.spider)
|
||||
req2 = Request('http://scrapytest.org/sub1/')
|
||||
mw.process_request(req2, crawler.spider)
|
||||
|
|
@ -102,8 +105,7 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert self.mw.process_request(req, self.spider) is None
|
||||
assert 'Cookie' not in req.headers
|
||||
|
||||
headers = {'Set-Cookie': b'C1=in\xa3valid; path=/',
|
||||
'Other': b'ignore\xa3me'}
|
||||
headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'}
|
||||
res = Response('http://scrapytest.org/', headers=headers)
|
||||
assert self.mw.process_response(req, res, self.spider) is res
|
||||
|
||||
|
|
@ -124,7 +126,10 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert 'Cookie' not in req.headers
|
||||
|
||||
# check that returned cookies are not merged back to jar
|
||||
res = Response('http://scrapytest.org/dontmerge', headers={'Set-Cookie': 'dont=mergeme; path=/'})
|
||||
res = Response(
|
||||
'http://scrapytest.org/dontmerge',
|
||||
headers={'Set-Cookie': 'dont=mergeme; path=/'},
|
||||
)
|
||||
assert self.mw.process_response(req, res, self.spider) is res
|
||||
|
||||
# check that cookies are merged back
|
||||
|
|
@ -179,7 +184,11 @@ class CookiesMiddlewareTest(TestCase):
|
|||
self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada")
|
||||
|
||||
def test_cookiejar_key(self):
|
||||
req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"})
|
||||
req = Request(
|
||||
'http://scrapytest.org/',
|
||||
cookies={'galleta': 'salada'},
|
||||
meta={'cookiejar': "store1"},
|
||||
)
|
||||
assert self.mw.process_request(req, self.spider) is None
|
||||
self.assertEqual(req.headers.get('Cookie'), b'galleta=salada')
|
||||
|
||||
|
|
@ -191,7 +200,11 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada')
|
||||
|
||||
req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"})
|
||||
req3 = Request(
|
||||
'http://scrapytest.org/',
|
||||
cookies={'galleta': 'dulce'},
|
||||
meta={'cookiejar': "store2"},
|
||||
)
|
||||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce')
|
||||
|
||||
|
|
@ -229,3 +242,95 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert self.mw.process_request(request, self.spider) is None
|
||||
self.assertIn('Cookie', request.headers)
|
||||
self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
|
||||
|
||||
def test_keep_cookie_from_default_request_headers_middleware(self):
|
||||
DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
|
||||
mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
|
||||
# overwrite with values from 'cookies' request argument
|
||||
req1 = Request('http://example.org', cookies={'default': 'something'})
|
||||
assert mw_default_headers.process_request(req1, self.spider) is None
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty')
|
||||
# keep both
|
||||
req2 = Request('http://example.com', cookies={'a': 'b'})
|
||||
assert mw_default_headers.process_request(req2, self.spider) is None
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
|
||||
|
||||
def test_keep_cookie_header(self):
|
||||
# keep only cookies from 'Cookie' request header
|
||||
req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d')
|
||||
# keep cookies from both 'Cookie' request header and 'cookies' keyword
|
||||
req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'})
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f')
|
||||
# overwrite values from 'Cookie' request header with 'cookies' keyword
|
||||
req3 = Request(
|
||||
'http://scrapytest.org',
|
||||
headers={'Cookie': 'a=b; c=d'},
|
||||
cookies={'a': 'new', 'e': 'f'},
|
||||
)
|
||||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f')
|
||||
|
||||
def test_request_cookies_encoding(self):
|
||||
# 1) UTF8-encoded bytes
|
||||
req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')})
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 2) Non UTF8-encoded bytes
|
||||
req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')})
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 3) Unicode string
|
||||
req3 = Request('http://example.org', cookies={'a': u'á'})
|
||||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
def test_request_headers_cookie_encoding(self):
|
||||
# 1) UTF8-encoded bytes
|
||||
req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')})
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 2) Non UTF8-encoded bytes
|
||||
req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')})
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
# 3) Unicode string
|
||||
req3 = Request('http://example.org', headers={'Cookie': u'a=á'})
|
||||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
def test_invalid_cookies(self):
|
||||
"""
|
||||
Invalid cookies are logged as warnings and discarded
|
||||
"""
|
||||
with LogCapture(
|
||||
'scrapy.downloadermiddlewares.cookies',
|
||||
propagate=False,
|
||||
level=logging.INFO,
|
||||
) as lc:
|
||||
cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}]
|
||||
req1 = Request('http://example.org/1', cookies=cookies1)
|
||||
assert self.mw.process_request(req1, self.spider) is None
|
||||
cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}]
|
||||
req2 = Request('http://example.org/2', cookies=cookies2)
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
lc.check(
|
||||
("scrapy.downloadermiddlewares.cookies",
|
||||
"WARNING",
|
||||
"Invalid cookie found in request <GET http://example.org/1>:"
|
||||
" {'value': 'bar'} ('name' is missing)"),
|
||||
("scrapy.downloadermiddlewares.cookies",
|
||||
"WARNING",
|
||||
"Invalid cookie found in request <GET http://example.org/2>:"
|
||||
" {'name': 'foo'} ('value' is missing)"),
|
||||
)
|
||||
self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1')
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2')
|
||||
|
|
|
|||
Loading…
Reference in New Issue