diff --git a/docs/faq.rst b/docs/faq.rst index 5783fdcaf..65159a6c2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -240,3 +240,17 @@ In order to avoid parsing all the entire feed at once in memory, you can use the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use under the cover. + +Does Scrapy manage cookies automatically? +----------------------------------------- + +Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them +back on subsequent requests, like any regular web browser does. + +For more info see :ref:`topics-request-response` and :ref:`cookies-mw`. + +How can I see the cookies being sent and received from Scrapy? +-------------------------------------------------------------- + +Enable the :setting:`COOKIES_DEBUG` setting. + diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 5f1acd63a..8693c13ae 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -158,6 +158,8 @@ middleware, see the :ref:`downloader middleware usage guide For a list of the components enabled by default (and their orders) see the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting. +.. _cookies-mw: + CookiesMiddleware ----------------- @@ -166,7 +168,36 @@ CookiesMiddleware .. class:: CookiesMiddleware - This middleware enables working with sites that need cookies. + This middleware enables working with sites that need cookies. It keeps track + of merging cookies sent by servers, so that they're send in future requests + for that spider, just like a web browser would do. + +The following settings can be used to configure the cookie middleware: + +* :setting:`COOKIES_DEBUG` + +.. setting:: COOKIES_DEBUG + +COOKIES_DEBUG +~~~~~~~~~~~~~ + +Default: ``False`` + +If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie`` +header) and all cookies received in responses (ie. ``Set-Cookie`` header). + +Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: + + 2011-04-06 14:35:10-0300 [diningcity] INFO: Spider opened + 2011-04-06 14:35:10-0300 [diningcity] DEBUG: Sending cookies to: + Cookie: clientlanguage_nl=en_EN + 2011-04-06 14:35:14-0300 [diningcity] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ + Set-Cookie: ip_isocode=US + Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ + 2011-04-06 14:49:50-0300 [diningcity] DEBUG: Crawled (200) (referer: None) + [...] + DefaultHeadersMiddleware ------------------------ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a1092b8e6..baeec8701 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -68,6 +68,8 @@ Request objects request_with_cookies = Request(url="http://www.example.com", cookies={currency: 'USD', country: 'UY'}, meta={'dont_merge_cookies': True}) + + For more info see :ref:`cookies-mw`. :type cookies: dict :param encoding: the encoding of this request (defaults to ``'utf-8'``). diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py index b99c4b338..1d3fbde22 100644 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ b/scrapy/contrib/downloadermiddleware/cookies.py @@ -1,3 +1,4 @@ +import os from collections import defaultdict from scrapy.xlib.pydispatch import dispatcher @@ -28,7 +29,7 @@ class CookiesMiddleware(object): # set Cookie header request.headers.pop('Cookie', None) jar.add_cookie_header(request) - self._debug_cookie(request) + self._debug_cookie(request, spider) def process_response(self, request, response, spider): if 'dont_merge_cookies' in request.meta: @@ -37,31 +38,28 @@ class CookiesMiddleware(object): # extract cookies from Set-Cookie and drop invalid/expired cookies jar = self.jars[spider] jar.extract_cookies(response, request) - self._debug_set_cookie(response) + self._debug_set_cookie(response, spider) return response def spider_closed(self, spider): self.jars.pop(spider, None) - def _debug_cookie(self, request): - """log Cookie header for request""" + def _debug_cookie(self, request, spider): if self.debug: - c = request.headers.get('Cookie') - c = c and [p.split('=')[0] for p in c.split(';')] - log.msg('Cookie: %s for %s' % (c, request.url), level=log.DEBUG) + cl = request.headers.getlist('Cookie') + if cl: + msg = "Sending cookies to: %s" % request + os.linesep + msg += os.linesep.join("Cookie: %s" % c for c in cl) + log.msg(msg, spider=spider, level=log.DEBUG) - def _debug_set_cookie(self, response): - """log Set-Cookies headers but exclude cookie values""" + def _debug_set_cookie(self, response, spider): if self.debug: cl = response.headers.getlist('Set-Cookie') - res = [] - for c in cl: - kv, tail = c.split(';', 1) - k = kv.split('=', 1)[0] - res.append('%s %s' % (k, tail)) - log.msg('Set-Cookie: %s from %s' % (res, response.url)) - + if cl: + msg = "Received cookies from: %s" % response + os.linesep + msg += os.linesep.join("Set-Cookie: %s" % c for c in cl) + log.msg(msg, spider=spider, level=log.DEBUG) def _get_request_cookies(self, jar, request): headers = {'Set-Cookie': ['%s=%s;' % (k, v) for k, v in request.cookies.iteritems()]}