From fe2979edf39ba583032a6af514bb7aad805f47ce Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 9 Feb 2024 13:57:21 +0200 Subject: [PATCH 01/15] cookiejars exposed --- scrapy/downloadermiddlewares/cookies.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 85781efd6..1f920fa2e 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -16,7 +16,7 @@ from typing import ( from tldextract import TLDExtract -from scrapy import Request, Spider +from scrapy import Request, Spider, signals from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured from scrapy.http import Response @@ -51,7 +51,13 @@ class CookiesMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured - return cls(crawler.settings.getbool("COOKIES_DEBUG")) + o = cls(crawler.settings.getbool("COOKIES_DEBUG")) + if crawler.settings.getbool("COOKIES_EXPOSE_JARS"): + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o + + def spider_opened(self, spider: Spider) -> None: + spider.jars = self.jars def _process_cookies( self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request From b8f89608a6a626eb6e98d6deebf737fd2b09f0c3 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 23 Feb 2024 11:22:46 +0100 Subject: [PATCH 02/15] cookiejars exposed --- scrapy/downloadermiddlewares/cookies.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 1f920fa2e..cc70fe366 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -52,12 +52,11 @@ class CookiesMiddleware: if not crawler.settings.getbool("COOKIES_ENABLED"): raise NotConfigured o = cls(crawler.settings.getbool("COOKIES_DEBUG")) - if crawler.settings.getbool("COOKIES_EXPOSE_JARS"): - crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o def spider_opened(self, spider: Spider) -> None: - spider.jars = self.jars + spider.cookie_jars = self.jars def _process_cookies( self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request From 11022ac4d48cbbf244bf0e7763b61089b2a3a7f6 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 7 Jun 2024 14:48:30 +0200 Subject: [PATCH 03/15] cookiejars exposed, docs added --- docs/faq.rst | 2 +- docs/topics/downloader-middleware.rst | 46 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2113b0964..273d4bfcc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -307,7 +307,7 @@ Does Scrapy manage cookies automatically? Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them back on subsequent requests, like any regular web browser does. -For more info see :ref:`topics-request-response` and :ref:`cookies-mw`. +For more info see :ref:`topics-request-response` and :ref:`cookiejars`. How can I see the cookies being sent and received from Scrapy? -------------------------------------------------------------- diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..4940f914a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -293,6 +293,52 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: [...] +.. _cookiejars: + +Direct access to cookiejars from spider +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +On some cases it is required to directly set specific values to existing cookiejar + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerProcess + + + class Quotes(scrapy.Spider): + name = "quotes" + custom_settings = {"DOWNLOAD_DELAY": 1} + + def start_requests(self): + yield scrapy.Request( + url="https://quotes.toscrape.com/login", callback=self.login + ) + + def login(self, response): + self.logger.info(self.cookie_jars[None]) # scrapy.http.cookies.CookieJar object + self.logger.info(self.cookie_jars[None].jar) # http.cookiejar object + + locale_cookie = ( + self.cookie_jars[None]._cookies["quotes.toscrape.com"]["/"].get("session") + ) + locale_cookie.value = locale_cookie.value.upper() + self.logger.info(self.cookie_jars[None].jar) + + + if __name__ == "__main__": + p = CrawlerProcess() + p.crawl(Quotes) + p.start() + +Log output:: + + 2024-02-23 10:51:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2024-02-23 10:51:27 [quotes] INFO: + 2024-02-23 10:51:27 [quotes] INFO: ]> + 2024-02-23 10:51:27 [quotes] INFO: ]> + 2024-02-23 10:51:27 [scrapy.core.engine] INFO: Closing spider (finished) + + DefaultHeadersMiddleware ------------------------ From d8fb9d2e7ecd7117e56f27c1f15bf861fe68c15d Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 7 Jun 2024 15:08:44 +0200 Subject: [PATCH 04/15] cookiejars exposed, docs added --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 273d4bfcc..100e850b0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -307,7 +307,7 @@ Does Scrapy manage cookies automatically? Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them back on subsequent requests, like any regular web browser does. -For more info see :ref:`topics-request-response` and :ref:`cookiejars`. +For more info see :ref:`topics-request-response` , :ref:`cookies-mw` and :ref:`cookiejars`. How can I see the cookies being sent and received from Scrapy? -------------------------------------------------------------- From 53db741532d7a117a999e71860532abc99dc88e5 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 21 Jun 2024 10:10:58 +0200 Subject: [PATCH 05/15] Update docs/faq.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 100e850b0..55a81761c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -307,7 +307,7 @@ Does Scrapy manage cookies automatically? Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them back on subsequent requests, like any regular web browser does. -For more info see :ref:`topics-request-response` , :ref:`cookies-mw` and :ref:`cookiejars`. +For more info see :ref:`topics-request-response`, :ref:`cookies-mw` and :ref:`cookiejars`. How can I see the cookies being sent and received from Scrapy? -------------------------------------------------------------- From 2f457179e8bbe57493fb88197e9b4a5ee9504f5a Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 21 Jun 2024 10:11:30 +0200 Subject: [PATCH 06/15] Update docs/topics/downloader-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/downloader-middleware.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 4940f914a..4ff0cf109 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -297,7 +297,9 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: Direct access to cookiejars from spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -On some cases it is required to directly set specific values to existing cookiejar + +In some cases it is required to directly set specific values in an existing +cookiejar. .. code-block:: python From dd652e425d53d3eb76bf7395b13b2c74bbf86985 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 14 Nov 2024 13:40:33 +0100 Subject: [PATCH 07/15] `get_cookiejar` - method added to spider (if cookies middleware enabled) --- scrapy/downloadermiddlewares/cookies.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 2aa1a8a96..73ff59541 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -37,6 +37,14 @@ def _is_public_domain(domain: str) -> bool: return not parts.domain +def get_cookiejar(self, response_or_request): + try: + cookiejarkey = response_or_request.request.meta.get("cookiejar") + except Exception: + cookiejarkey = response_or_request.meta.get("cookiejar") + return self.cookie_jars[cookiejarkey] + + class CookiesMiddleware: """This middleware enables working with sites that need cookies""" @@ -54,6 +62,8 @@ class CookiesMiddleware: def spider_opened(self, spider: Spider) -> None: spider.cookie_jars = self.jars + attribute_name = "get_cookiejar" + setattr(spider.__class__, attribute_name, get_cookiejar) def _process_cookies( self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request From 85c2d530ed9ff80605499a66d897fb80d5740d05 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 13 Dec 2024 13:48:03 +0100 Subject: [PATCH 08/15] docs updated --- docs/topics/downloader-middleware.rst | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b7661e68f..ac8f8ef07 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -332,6 +332,11 @@ cookiejar. p.crawl(Quotes) p.start() +.. note:: + + If :reqmeta:`cookiejar` didn't specified in request meta, middleware will read this as ``None``, as result of this middleware will create cookiejar object under dict key ``None``. + + Log output:: 2024-02-23 10:51:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) @@ -341,6 +346,39 @@ Log output:: 2024-02-23 10:51:27 [scrapy.core.engine] INFO: Closing spider (finished) +If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method added to spider: + +.. class:: scrapy.Spider + + ... + + .. method:: get_cookiejar(spider, response_or_request) + + Returns ``scrapy.http.cookies.CookieJar`` cookiejar object based on :reqmeta:`cookiejar` value of ``response_or_request`` argument + + :param response_or_request: request or response object where target session/cookiejar object used or planned to use + :type response_or_request: :class:`~scrapy.Request` or :class:`~scrapy.Response` object + + Technically this is short cut to ``request.meta.get('cookiejar')`` or ``response.request.meta.get('cookiejar')`` if ``response_or_request`` is :class:`~scrapy.Response` + + It allows a bit easier access to cookiejar object: + + .. code-block:: python + + ... + self.logger.info(self.get_cookiejar(response)) # scrapy.http.cookies.CookieJar object + self.logger.info(self.get_cookiejar(response).jar) # http.cookiejar object + locale_cookie = ( + self.get_cookiejar(response).jar._cookies["quotes.toscrape.com"]["/"].get("session") + ) + +Known use cases where direct access to cookiejar objects can be useful: + +1. Modifying specific cookie value (as provided on example code sample above). +2. Delete cookiejar. On some cases this can re-initialise session. + +.. warning:: ``http.cookiejar`` object from python standart lib https://docs.python.org/3/library/http.cookies.html doesn't provide option to individually access/modify specific cookie values. It is possible to change it only using it's private attributes as provided on code sample above. + DefaultHeadersMiddleware ------------------------ From 82f0db34d1c2d2444ca5ea95292d15942f918521 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 30 Oct 2025 16:43:27 +0100 Subject: [PATCH 09/15] cookiejar: docs updated ("skip: next") --- docs/topics/downloader-middleware.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 060267bc2..57ed829ab 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -339,6 +339,7 @@ If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method a It allows a bit easier access to cookiejar object: + .. skip: next .. code-block:: python ... From 20913cc7878d2082f548d27ee3d66e9a8079e0b0 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 30 Oct 2025 18:03:58 +0100 Subject: [PATCH 10/15] cookiejar: docs updated ("skip: next") --- docs/topics/downloader-middleware.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 57ed829ab..515108d9d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -277,6 +277,7 @@ Direct access to cookiejars from spider In some cases it is required to directly set specific values in an existing cookiejar. +.. skip: next .. code-block:: python import scrapy From 0628e93fe6fe66451d6945afbcb135598741959e Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 30 Oct 2025 18:25:02 +0100 Subject: [PATCH 11/15] cookiejar: docs updated ("skip: start/end") --- docs/topics/downloader-middleware.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 515108d9d..2a17620bd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -277,7 +277,7 @@ Direct access to cookiejars from spider In some cases it is required to directly set specific values in an existing cookiejar. -.. skip: next +.. skip: start .. code-block:: python import scrapy @@ -309,6 +309,8 @@ cookiejar. p.crawl(Quotes) p.start() +.. skip: end + .. note:: If :reqmeta:`cookiejar` didn't specified in request meta, middleware will read this as ``None``, as result of this middleware will create cookiejar object under dict key ``None``. @@ -340,7 +342,7 @@ If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method a It allows a bit easier access to cookiejar object: - .. skip: next + .. skip: start .. code-block:: python ... @@ -349,6 +351,7 @@ If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method a locale_cookie = ( self.get_cookiejar(response).jar._cookies["quotes.toscrape.com"]["/"].get("session") ) + .. skip: end Known use cases where direct access to cookiejar objects can be useful: From f7ed14d0dee5dfdf009685f2061a1b01c4344bdd Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 30 Oct 2025 18:32:43 +0100 Subject: [PATCH 12/15] cookiejar: docs updated ("skip: start/end") --- docs/topics/downloader-middleware.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2a17620bd..5708a01ca 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -345,7 +345,6 @@ If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method a .. skip: start .. code-block:: python - ... self.logger.info(self.get_cookiejar(response)) # scrapy.http.cookies.CookieJar object self.logger.info(self.get_cookiejar(response).jar) # http.cookiejar object locale_cookie = ( From 28d543bdf6c05cab16a92242e85072463bfb3070 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Thu, 30 Oct 2025 19:59:35 +0100 Subject: [PATCH 13/15] cookiejar: docs updated ("skip: start/end") --- docs/topics/downloader-middleware.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 5708a01ca..579212263 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -342,15 +342,14 @@ If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method a It allows a bit easier access to cookiejar object: - .. skip: start - .. code-block:: python + + .. code-block:: none self.logger.info(self.get_cookiejar(response)) # scrapy.http.cookies.CookieJar object self.logger.info(self.get_cookiejar(response).jar) # http.cookiejar object locale_cookie = ( self.get_cookiejar(response).jar._cookies["quotes.toscrape.com"]["/"].get("session") ) - .. skip: end Known use cases where direct access to cookiejar objects can be useful: From 3d173d2f41b89b21e2b4e00e3d55a5a99ee677e6 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 31 Oct 2025 14:02:03 +0100 Subject: [PATCH 14/15] cookiejar: reference to `http.cookijar.CookieJar` --- docs/topics/downloader-middleware.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 579212263..6df3e1e87 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -274,8 +274,7 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: Direct access to cookiejars from spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In some cases it is required to directly set specific values in an existing -cookiejar. +In some cases it is required to directly set specific values in an existing :class:`scrapy.http.cookies.CookieJar` object where :class:`http.cookies.CookieJar` from python standard library used here) .. skip: start .. code-block:: python @@ -295,7 +294,7 @@ cookiejar. def login(self, response): self.logger.info(self.cookie_jars[None]) # scrapy.http.cookies.CookieJar object - self.logger.info(self.cookie_jars[None].jar) # http.cookiejar object + self.logger.info(self.cookie_jars[None].jar) # http.cookiejar.CookieJar object locale_cookie = ( self.cookie_jars[None]._cookies["quotes.toscrape.com"]["/"].get("session") @@ -346,7 +345,7 @@ If cookie middleware is enabled, ``get_cookiejar(response_or_request)`` method a .. code-block:: none self.logger.info(self.get_cookiejar(response)) # scrapy.http.cookies.CookieJar object - self.logger.info(self.get_cookiejar(response).jar) # http.cookiejar object + self.logger.info(self.get_cookiejar(response).jar) # http.cookiejar.CookieJar object locale_cookie = ( self.get_cookiejar(response).jar._cookies["quotes.toscrape.com"]["/"].get("session") ) From c9bb3a4f4b5dfd9db961a30f231a72da87670b9d Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 31 Oct 2025 14:21:23 +0100 Subject: [PATCH 15/15] cookiejar: typo fixed --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6df3e1e87..d0a5ea63e 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -274,7 +274,7 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: Direct access to cookiejars from spider ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In some cases it is required to directly set specific values in an existing :class:`scrapy.http.cookies.CookieJar` object where :class:`http.cookies.CookieJar` from python standard library used here) +In some cases it is required to directly set specific values in an existing :class:`scrapy.http.cookies.CookieJar` object where :class:`http.cookiejar.CookieJar` from python standard library used here) .. skip: start .. code-block:: python @@ -355,7 +355,7 @@ Known use cases where direct access to cookiejar objects can be useful: 1. Modifying specific cookie value (as provided on example code sample above). 2. Delete cookiejar. On some cases this can re-initialise session. -.. warning:: ``http.cookiejar`` object from python standart lib https://docs.python.org/3/library/http.cookies.html doesn't provide option to individually access/modify specific cookie values. It is possible to change it only using it's private attributes as provided on code sample above. +.. warning:: ``http.cookiejar.CookieJar`` object from python standard lib https://docs.python.org/3/library/http.cookies.html doesn't provide option to individually access/modify specific cookie values. It is possible to change it only using it's private attributes as provided on code sample above. DefaultHeadersMiddleware ------------------------