From 11022ac4d48cbbf244bf0e7763b61089b2a3a7f6 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 7 Jun 2024 14:48:30 +0200 Subject: [PATCH] cookiejars exposed, docs added --- docs/faq.rst | 2 +- docs/topics/downloader-middleware.rst | 46 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 2113b0964..273d4bfcc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -307,7 +307,7 @@ Does Scrapy manage cookies automatically? Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them back on subsequent requests, like any regular web browser does. -For more info see :ref:`topics-request-response` and :ref:`cookies-mw`. +For more info see :ref:`topics-request-response` and :ref:`cookiejars`. How can I see the cookies being sent and received from Scrapy? -------------------------------------------------------------- diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..4940f914a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -293,6 +293,52 @@ Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: [...] +.. _cookiejars: + +Direct access to cookiejars from spider +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +On some cases it is required to directly set specific values to existing cookiejar + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerProcess + + + class Quotes(scrapy.Spider): + name = "quotes" + custom_settings = {"DOWNLOAD_DELAY": 1} + + def start_requests(self): + yield scrapy.Request( + url="https://quotes.toscrape.com/login", callback=self.login + ) + + def login(self, response): + self.logger.info(self.cookie_jars[None]) # scrapy.http.cookies.CookieJar object + self.logger.info(self.cookie_jars[None].jar) # http.cookiejar object + + locale_cookie = ( + self.cookie_jars[None]._cookies["quotes.toscrape.com"]["/"].get("session") + ) + locale_cookie.value = locale_cookie.value.upper() + self.logger.info(self.cookie_jars[None].jar) + + + if __name__ == "__main__": + p = CrawlerProcess() + p.crawl(Quotes) + p.start() + +Log output:: + + 2024-02-23 10:51:27 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + 2024-02-23 10:51:27 [quotes] INFO: + 2024-02-23 10:51:27 [quotes] INFO: ]> + 2024-02-23 10:51:27 [quotes] INFO: ]> + 2024-02-23 10:51:27 [scrapy.core.engine] INFO: Closing spider (finished) + + DefaultHeadersMiddleware ------------------------