From b5e7290f407ae27dbf49d1d5b6d2ab528800b92e Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 12 Aug 2026 16:46:08 +0200 Subject: [PATCH] Add sessions --- docs/conf.py | 1 + docs/index.rst | 6 +- docs/news.rst | 4 +- docs/topics/api.rst | 7 +- docs/topics/cookies.rst | 138 --------------- docs/topics/request-response.rst | 3 +- docs/topics/sessions.rst | 196 +++++++++++++++++++++ docs/topics/settings.rst | 1 + docs/topics/spider-middleware.rst | 8 + docs/topics/stats.rst | 16 ++ scrapy/crawler.py | 8 + scrapy/downloadermiddlewares/cookies.py | 84 +++++++-- scrapy/sessions.py | 99 +++++++++++ scrapy/settings/default_settings.py | 4 + scrapy/spidermiddlewares/sessions.py | 30 ++++ tests/test_downloadermiddleware_cookies.py | 115 +++++++++++- tests/test_sessions.py | 60 +++++++ tests/test_spidermiddleware_sessions.py | 99 +++++++++++ 18 files changed, 718 insertions(+), 161 deletions(-) delete mode 100644 docs/topics/cookies.rst create mode 100644 docs/topics/sessions.rst create mode 100644 scrapy/sessions.py create mode 100644 scrapy/spidermiddlewares/sessions.py create mode 100644 tests/test_sessions.py create mode 100644 tests/test_spidermiddleware_sessions.py diff --git a/docs/conf.py b/docs/conf.py index ad55231bc..69399eefb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -37,6 +37,7 @@ extensions = [ redirects = { "topics/broad-crawls": "optimize.html#broad-crawls", + "topics/cookies": "sessions.html", } templates_path = ["_templates"] diff --git a/docs/index.rst b/docs/index.rst index de06e3488..35cbb6aee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -78,7 +78,7 @@ Basic concepts topics/item-pipeline topics/feed-exports topics/request-response - topics/cookies + topics/sessions topics/link-extractors topics/settings topics/exceptions @@ -110,8 +110,8 @@ Basic concepts :doc:`topics/request-response` Understand the classes used to represent HTTP requests and responses. -:doc:`topics/cookies` - Send and receive cookies. +:doc:`topics/sessions` + Keep cookies and other state across groups of requests. :doc:`topics/link-extractors` Convenient classes to extract links to follow from pages. diff --git a/docs/news.rst b/docs/news.rst index a5cc6c723..9a0778d57 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3260,7 +3260,7 @@ Documentation - Extended documentation for :attr:`.Request.meta`. (:issue:`5565`) -- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:issue:`5936`, +- Fixed the ``dont_merge_cookies`` documentation. (:issue:`5936`, :issue:`6077`) - Added a link to Zyte's export guides to the :ref:`feed exports @@ -9019,7 +9019,7 @@ Scrapy changes: - SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`) - StackTraceDump extension: also dump trackref live references (:commit:`fe2ce93`) - nested items now fully supported in JSON and JSONLines exporters -- added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider +- added ``cookiejar`` Request meta key to support multiple cookie sessions per spider - decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module - dropped support for Python 2.5. See https://www.zyte.com/blog/scrapy-0-15-dropping-support-for-python-2-5/ - dropped support for Twisted 2.5 diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 598edfeb5..07ca6e5e8 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -36,8 +36,9 @@ how you :ref:`configure the downloader middlewares :class:`scrapy.settings.Settings` object. The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`, - :attr:`request_fingerprinter` and :attr:`stats` attributes get their value - when the crawl starts, and raise :exc:`RuntimeError` when read before that. + :attr:`request_fingerprinter`, :attr:`sessions` and :attr:`stats` attributes + get their value when the crawl starts, and raise :exc:`RuntimeError` when + read before that. .. versionchanged:: VERSION Those attributes used to be ``None`` before getting their value. @@ -71,6 +72,8 @@ how you :ref:`configure the downloader middlewares For the API see :class:`~scrapy.signalmanager.SignalManager` class. + .. autoattribute:: sessions + .. attribute:: stats The stats collector of this crawler. diff --git a/docs/topics/cookies.rst b/docs/topics/cookies.rst deleted file mode 100644 index ab5f3dca0..000000000 --- a/docs/topics/cookies.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. _cookies: -.. _cookies-mw: - -======= -Cookies -======= - -Scrapy keeps track of the cookies that websites set and sends them back on -later requests to those websites, just like a web browser does. That is the job -of :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which is -enabled by default. - - -Setting cookies on a request -============================ - -.. invisible-code-block: python - - from scrapy import Request - -Use the ``cookies`` parameter of :class:`~scrapy.Request` to send cookies of -your own, either as a dict: - -.. code-block:: python - - request = Request( - url="https://example.com", - cookies={"currency": "USD", "country": "UY"}, - ) - -Or as a list of dicts, which also lets you set cookie attributes: - -.. code-block:: python - - request = Request( - url="https://example.com", - cookies=[ - { - "name": "currency", - "value": "USD", - "domain": "example.com", - "path": "/currency", - "secure": True, - }, - ], - ) - -Setting attributes is only useful if the cookies are stored for later requests, -i.e. if :reqmeta:`dont_merge_cookies` is not enabled. - -.. caution:: Cookies set through the ``Cookie`` header are not handled by - :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which - drops that header. - -.. caution:: When a cookie name or value is a byte sequence that is not UTF-8 - encoded, the cookie is dropped and a warning is logged. See - :ref:`topics-logging-advanced-customization` to customize the logging - behavior. - - -.. reqmeta:: cookiejar - -Multiple cookie sessions per spider -=================================== - -By default all requests share a single cookie jar (session). To use different -ones, pass an identifier in the :reqmeta:`cookiejar` request meta key: - -.. skip: next -.. code-block:: python - - for i, url in enumerate(urls): - yield Request(url, meta={"cookiejar": i}, callback=self.parse_page) - -The :reqmeta:`cookiejar` meta key is not "sticky", so you need to keep passing -it along on subsequent requests: - -.. code-block:: python - - def parse_page(self, response): - return Request( - "https://example.com/otherpage", - meta={"cookiejar": response.meta["cookiejar"]}, - callback=self.parse_other_page, - ) - - -.. reqmeta:: dont_merge_cookies - -Skipping the cookie jar for a request -===================================== - -Set the :reqmeta:`dont_merge_cookies` request meta key to ``True`` to keep a -request from touching the cookie jar in either direction: no stored cookie is -sent with the request, and no cookie received in the response is stored. The -cookies of the request itself are ignored as well. - - -.. setting:: COOKIES_ENABLED - -COOKIES_ENABLED -=============== - -Default: ``True`` - -Whether to enable :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`. -If disabled, no cookies are sent to web servers. - - -.. setting:: COOKIES_DEBUG - -COOKIES_DEBUG -============= - -Default: ``False`` - -If enabled, Scrapy logs all cookies sent in requests (i.e. the ``Cookie`` -header) and all cookies received in responses (i.e. the ``Set-Cookie`` -header):: - - 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened - 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: - Cookie: clientlanguage_nl=en_EN - 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> - Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ - Set-Cookie: ip_isocode=US - Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ - 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) - [...] - - -CookiesMiddleware -================= - -.. module:: scrapy.downloadermiddlewares.cookies - :synopsis: Cookies Downloader Middleware - -.. autoclass:: CookiesMiddleware diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1d97e39b6..c651b4122 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -827,9 +827,7 @@ Those are: * :reqmeta:`allow_offsite` * :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` -* :reqmeta:`cookiejar` * :reqmeta:`dont_cache` -* :reqmeta:`dont_merge_cookies` * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` @@ -853,6 +851,7 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`session` * :reqmeta:`verbatim_url` .. reqmeta:: bindaddress diff --git a/docs/topics/sessions.rst b/docs/topics/sessions.rst new file mode 100644 index 000000000..1bbf031a4 --- /dev/null +++ b/docs/topics/sessions.rst @@ -0,0 +1,196 @@ +.. _sessions: +.. _cookies: +.. _cookies-mw: + +======== +Sessions +======== + +A session is the state that a group of requests share, starting with their +cookies, which Scrapy stores and sends back on later requests to the same +website, like a web browser does. + +Every request uses the same session unless told otherwise, so a crawl behaves +like a single browser profile. Use more than one to keep parts of a crawl from +sharing state, e.g. to crawl a website through several independent profiles at +the same time. + + +.. _session-choose: + +Choosing the session of a request +================================= + +.. versionadded:: VERSION + +.. invisible-code-block: python + + from scrapy import Request + +.. reqmeta:: session + +Set the :reqmeta:`session` request meta key to a session ID: + +.. skip: next +.. code-block:: python + + for index, url in enumerate(urls): + yield Request(url, meta={"session": index}) + +Any ID works, and the session is created the first time it is used. +:meth:`~scrapy.sessions.Sessions.create` covers the case where you have no ID of +your own to give: it returns a session that is certainly new, which indexing +cannot promise, since the ID you choose may be in use already. + +.. skip: next +.. code-block:: python + + session = self.crawler.sessions.create() + yield Request(url, meta={"session": session.id}) + +The follow-up requests that a spider callback yields stay in the session of the +request that got that callback its response, so a session lasts as long as the +crawl that follows from it without its ID being passed along by hand. Set +:reqmeta:`session` on one of those requests to move it to a different session. + +A request that neither sets :reqmeta:`session` nor inherits one uses the session +whose ID is ``"main"``. Everything a crawl does without asking for a session +happens there, which makes ``session="main"`` the way to send a request back to +it, e.g. from a callback whose own session is a different one. + +Set :reqmeta:`session` to ``None`` for a request to use no session at all: no +stored cookie is sent with it and no cookie received in its response is stored. +The cookies of the request itself are still sent, and its follow-up requests +inherit the lack of a session. + + +.. _session-registry: + +The session registry +==================== + +.. versionadded:: VERSION + +Sessions live in :attr:`Crawler.sessions `, +where you can inspect and modify them: + +.. skip: start +.. code-block:: pycon + + >>> cookies = crawler.sessions["main"].cookies + >>> len(cookies) + 2 + >>> cookies.clear() + +.. skip: end + +When a session stops working, e.g. because the website expired it, call +:meth:`~scrapy.sessions.Sessions.retire` and retry the request with +:func:`~scrapy.downloadermiddlewares.retry.get_retry_request`: the session is +gone, so the retry starts a new one under the same ID. + +.. autoclass:: scrapy.sessions.Sessions + :members: + +.. autoclass:: scrapy.sessions.Session + :members: + + +Sending cookies with a request +============================== + +Use the ``cookies`` parameter of :class:`~scrapy.Request` to send cookies of +your own, either as a dict: + +.. code-block:: python + + request = Request( + url="https://example.com", + cookies={"currency": "USD", "country": "UY"}, + ) + +Or as a list of dicts, which also lets you set cookie attributes: + +.. code-block:: python + + request = Request( + url="https://example.com", + cookies=[ + { + "name": "currency", + "value": "USD", + "domain": "example.com", + "path": "/currency", + "secure": True, + }, + ], + ) + +Setting attributes is only useful if the cookies are stored for later requests, +i.e. if the request has a session. + +.. caution:: Cookies set through the ``Cookie`` header are not handled by + :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which + drops that header. + +.. caution:: When a cookie name or value is a byte sequence that is not UTF-8 + encoded, the cookie is dropped and a warning is logged. See + :ref:`topics-logging-advanced-customization` to customize the logging + behavior. + + +.. setting:: SESSIONS_MAX + +SESSIONS_MAX +============ + +Default: ``1000`` + +Maximum number of sessions to keep in memory. When the limit is reached, the +session that has not been used for the longest time is dropped, losing its +cookies. The first drop is logged as a warning, and :stat:`sessions/dropped` +counts them all. + +Raise it if a crawl needs more sessions alive at the same time, e.g. because it +gives every request a session of its own. + + +.. setting:: COOKIES_ENABLED + +COOKIES_ENABLED +=============== + +Default: ``True`` + +Whether to enable :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`. +If disabled, no cookies are sent to web servers. + + +.. setting:: COOKIES_DEBUG + +COOKIES_DEBUG +============= + +Default: ``False`` + +If enabled, Scrapy logs all cookies sent in requests (i.e. the ``Cookie`` +header) and all cookies received in responses (i.e. the ``Set-Cookie`` +header):: + + 2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened + 2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: + Cookie: clientlanguage_nl=en_EN + 2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html> + Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/ + Set-Cookie: ip_isocode=US + Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/ + 2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) + [...] + + +CookiesMiddleware +================= + +.. module:: scrapy.downloadermiddlewares.cookies + +.. autoclass:: CookiesMiddleware diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 27ef3f7ef..a0da28cca 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2083,6 +2083,7 @@ Default: { "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, + "scrapy.spidermiddlewares.sessions.SessionsSpiderMiddleware": 375, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 1c9ee0c77..f137237a0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -459,6 +459,14 @@ Use ``""`` to override the policy for responses with `no referrer policy `__. +SessionsSpiderMiddleware +------------------------ + +.. module:: scrapy.spidermiddlewares.sessions + +.. autoclass:: SessionsSpiderMiddleware + + StartSpiderMiddleware --------------------- diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 6fb7783d1..5db28b69f 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -722,6 +722,22 @@ one per actual value of the placeholder. :ref:`serialized `, and hence were stored into the memory queue instead. +.. stat:: sessions/created + +``sessions/created`` + Number of :ref:`sessions ` created. + +.. stat:: sessions/dropped + +``sessions/dropped`` + Number of sessions dropped to stay within :setting:`SESSIONS_MAX`. + +.. stat:: sessions/retired + +``sessions/retired`` + Number of sessions deleted with + :meth:`~scrapy.sessions.Sessions.retire`. + .. stat:: spider_exceptions/count ``spider_exceptions/count`` diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 444a5fb67..807d3671f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -17,6 +17,7 @@ from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.extension import ExtensionManager +from scrapy.sessions import Sessions from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader @@ -105,6 +106,11 @@ class Crawler: request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = ( _LateAttribute() ) + sessions: _LateAttribute[Sessions] = _LateAttribute() + """The :class:`~scrapy.sessions.Sessions` registry of this crawler. + + See :ref:`sessions`. + """ stats: _LateAttribute[StatsCollector] = _LateAttribute() def __init__( @@ -137,6 +143,7 @@ class Crawler: self._extensions: ExtensionManager | None = None self._logformatter: LogFormatter | None = None self._request_fingerprinter: RequestFingerprinterProtocol | None = None + self._sessions: Sessions | None = None self._stats: StatsCollector | None = None def _update_root_log_handler(self) -> None: @@ -154,6 +161,7 @@ class Crawler: "max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN" ) self.stats = load_object(self.settings["STATS_CLASS"])(self) + self.sessions = Sessions(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = build_from_crawler(lf_cls, self) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 34d071fd5..05e25160f 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,14 +1,16 @@ from __future__ import annotations import logging -from collections import defaultdict -from typing import TYPE_CHECKING, Any +import warnings +from typing import TYPE_CHECKING, Any, ClassVar from tldextract import TLDExtract -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Response from scrapy.http.cookies import CookieJar +from scrapy.sessions import _MAIN_ID +from scrapy.utils.datatypes import LocalCache from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -24,6 +26,7 @@ if TYPE_CHECKING: from scrapy import Request, Spider from scrapy.crawler import Crawler from scrapy.http.request import VerboseCookie + from scrapy.sessions import Session logger = logging.getLogger(__name__) @@ -43,9 +46,15 @@ class CookiesMiddleware: crawler: Crawler + _DEPRECATED_KEYS: ClassVar[dict[str, str]] = { + "cookiejar": "Use the session request meta key instead.", + "dont_merge_cookies": "Set the session request meta key to None instead.", + } + def __init__(self, debug: bool = False): - self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar) self.debug: bool = debug + # Session ID of every cookiejar meta key seen so far, for self.jars. + self._session_ids: LocalCache[Any, str] = LocalCache() @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -53,8 +62,53 @@ class CookiesMiddleware: raise NotConfigured o = cls(crawler.settings.getbool("COOKIES_DEBUG")) o.crawler = crawler + o._session_ids.limit = crawler.settings.getint("SESSIONS_MAX") return o + @property + def jars(self) -> dict[Any, CookieJar]: + """Cookie jars of the :reqmeta:`cookiejar` request meta keys seen so + far.""" + warnings.warn( + "CookiesMiddleware.jars is deprecated, use Crawler.sessions instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + sessions = self.crawler.sessions + return {key: sessions[id_].cookies for key, id_ in self._session_ids.items()} + + def _session(self, request: Request) -> Session | None: + """Return the session of *request*, or ``None`` if it has none.""" + has_session = "session" in request.meta + for key in self._DEPRECATED_KEYS: + if key in request.meta: + self._warn_deprecated_key(key, ignored=has_session) + if has_session: + session_id = request.meta["session"] + return None if session_id is None else self.crawler.sessions[session_id] + if request.meta.get("dont_merge_cookies", False): + return None + jar_key = request.meta.get("cookiejar") + jar_id = _MAIN_ID if jar_key is None else f"cookiejar:{jar_key!r}" + self._session_ids[jar_key] = jar_id + return self.crawler.sessions[jar_id] + + def _warn_deprecated_key(self, key: str, *, ignored: bool) -> None: + if ignored: + message = ( + f"The {key} request meta key is deprecated, and it is being " + f"ignored because the session request meta key is set on the " + f"same request. Remove {key}." + ) + else: + message = ( + f"The {key} request meta key is deprecated. " + f"{self._DEPRECATED_KEYS[key]} Note that, unlike {key}, session " + f"is inherited by the follow-up requests that a spider callback " + f"yields." + ) + warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=3) + def _process_cookies( self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request ) -> None: @@ -77,30 +131,34 @@ class CookiesMiddleware: def process_request( self, request: Request, spider: Spider | None = None ) -> Request | Response | None: - if request.meta.get("dont_merge_cookies", False): + session = self._session(request) + if session is None: + # The cookies of the request are its own, so they are sent even with + # no session to merge them into; a jar that no one keeps turns them + # into a Cookie header. dont_merge_cookies drops them instead. + if request.cookies and not request.meta.get("dont_merge_cookies", False): + self._set_cookie_header(CookieJar(), request) return None + self._set_cookie_header(session.cookies, request) + return None - cookiejarkey = request.meta.get("cookiejar") - jar = self.jars[cookiejarkey] + def _set_cookie_header(self, jar: CookieJar, request: Request) -> None: cookies = self._get_request_cookies(jar, request) self._process_cookies(cookies, jar=jar, request=request) - - # set Cookie header request.headers.pop("Cookie", None) jar.add_cookie_header(request) self._debug_cookie(request) - return None @_warn_spider_arg def process_response( self, request: Request, response: Response, spider: Spider | None = None ) -> Request | Response: - if request.meta.get("dont_merge_cookies", False): + session = self._session(request) + if session is None: return response # extract cookies from Set-Cookie and drop invalid/expired cookies - cookiejarkey = request.meta.get("cookiejar") - jar = self.jars[cookiejarkey] + jar = session.cookies cookies = jar.make_cookies(response, request) self._process_cookies(cookies, jar=jar, request=request) diff --git a/scrapy/sessions.py b/scrapy/sessions.py new file mode 100644 index 000000000..e67b5bed3 --- /dev/null +++ b/scrapy/sessions.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import logging +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +from scrapy.http.cookies import CookieJar + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +logger = logging.getLogger(__name__) + +_MAIN_ID = "main" + + +@dataclass +class Session: + """State that requests bound to the same session share. + + .. versionadded:: VERSION + """ + + id: str + """Session ID, i.e. the value of the :reqmeta:`session` request meta key of + the requests bound to this session.""" + + cookies: CookieJar = field(default_factory=CookieJar) + """Cookies of the session.""" + + meta: dict[Any, Any] = field(default_factory=dict) + """Free-form data about the session.""" + + +class Sessions: + """Registry of the :class:`Session` objects of a crawler, available as + :attr:`Crawler.sessions `. + + .. versionadded:: VERSION + + Indexing by session ID returns the matching session, creating it if it does + not exist, e.g. ``crawler.sessions["main"]``. IDs are strings; a value of a + different type is used as its :func:`str` form. + + It holds at most :setting:`SESSIONS_MAX` sessions; when full, the session + that has not been used for the longest time is dropped. + """ + + def __init__(self, crawler: Crawler): + self._crawler = crawler + self._max = crawler.settings.getint("SESSIONS_MAX") + self._sessions: OrderedDict[str, Session] = OrderedDict() + self._logged_drop = False + + def __getitem__(self, session_id: Any) -> Session: + session_id = str(session_id) + if (session := self._sessions.get(session_id)) is not None: + self._sessions.move_to_end(session_id) + return session + while self._sessions and len(self._sessions) >= self._max: + self._drop() + session = self._sessions[session_id] = Session(session_id) + self._crawler.stats.inc_value("sessions/created") + return session + + def __contains__(self, session_id: Any) -> bool: + return str(session_id) in self._sessions + + def _drop(self) -> None: + session_id, _ = self._sessions.popitem(last=False) + self._crawler.stats.inc_value("sessions/dropped") + if not self._logged_drop: + self._logged_drop = True + logger.warning( + f"Dropped session {session_id!r}, and its state, to stay within " + f"SESSIONS_MAX ({self._max}). Raise SESSIONS_MAX if your " + f"sessions are being dropped while still in use - no more " + f"dropped sessions will be logged.", + extra={"spider": self._crawler.spider}, + ) + + def create(self) -> Session: + """Create a session with a unique ID and return it. + + Use it when you need a session that is certainly new and have no ID of + your own to give it. + """ + return self[uuid4().hex] + + def retire(self, session_id: Any) -> None: + """Delete the session with the given ID, if it exists. + + Requests bound to it get a new, empty session. + """ + if self._sessions.pop(str(session_id), None) is not None: + self._crawler.stats.inc_value("sessions/retired") diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7b30b849..73bf31c41 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -193,6 +193,7 @@ __all__ = [ "SCHEDULER_START_DISK_QUEUE", "SCHEDULER_START_MEMORY_QUEUE", "SCRAPER_SLOT_MAX_ACTIVE_SIZE", + "SESSIONS_MAX", "SPIDER_CONTRACTS", "SPIDER_CONTRACTS_BASE", "SPIDER_LOADER_CLASS", @@ -537,6 +538,8 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SESSIONS_MAX = 1000 + SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { "scrapy.contracts.default.UrlContract": 1, @@ -554,6 +557,7 @@ SPIDER_MIDDLEWARES_BASE = { # Engine side "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25, "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50, + "scrapy.spidermiddlewares.sessions.SessionsSpiderMiddleware": 375, "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, diff --git a/scrapy/spidermiddlewares/sessions.py b/scrapy/spidermiddlewares/sessions.py new file mode 100644 index 000000000..938ac8939 --- /dev/null +++ b/scrapy/spidermiddlewares/sessions.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware + +if TYPE_CHECKING: + from scrapy.http import Request, Response + + +class SessionsSpiderMiddleware(BaseSpiderMiddleware): + """Bind requests from a spider callback to the :ref:`session ` of + the request whose response reached that callback. + + .. versionadded:: VERSION + + Requests that set the :reqmeta:`session` request meta key themselves keep + their own session. + """ + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if ( + response is not None + and "session" not in request.meta + and "session" in response.meta + ): + request.meta["session"] = response.meta["session"] + return request diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 6c4c80eae..864558120 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -7,7 +7,7 @@ import pytest from scrapy.downloadermiddlewares.cookies import CookiesMiddleware from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import CookiesT, VerboseCookie from scrapy.utils.misc import build_from_crawler @@ -190,6 +190,7 @@ class TestCookiesMiddleware: assert self.mw.process_request(req2) is None assert "Cookie" in req2.headers + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_dont_merge_cookies(self): # merge some cookies into jar headers = {"Set-Cookie": "C1=value1; path=/"} @@ -280,6 +281,7 @@ class TestCookiesMiddleware: req2.headers.get("Cookie"), b"C1=value1; galleta=salada" ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_cookiejar_key(self): req = Request( "http://scrapytest.org/", @@ -340,6 +342,117 @@ class TestCookiesMiddleware: assert self.mw.process_request(req6) is None assert req6.headers.get("Cookie") is None + def _set_cookie(self, value: str, **meta: Any) -> None: + req = Request("http://scrapytest.org/", meta=meta) + assert self.mw.process_request(req) is None + res = Response("http://scrapytest.org/", headers={"Set-Cookie": value}) + assert self.mw.process_response(req, res) is res + + def _get_cookie(self, **meta: Any) -> bytes | None: + req = Request("http://scrapytest.org/", meta=meta) + assert self.mw.process_request(req) is None + return req.headers.get("Cookie") + + def test_session_key(self) -> None: + self._set_cookie("C1=value1; path=/", session="store1") + self._set_cookie("C2=value2; path=/", session="store2") + assert self._get_cookie(session="store1") == b"C1=value1" + assert self._get_cookie(session="store2") == b"C2=value2" + assert self._get_cookie() is None + assert self._get_cookie(session="main") is None + + def test_session_none(self) -> None: + self._set_cookie("C1=value1; path=/") + assert self._get_cookie(session=None) is None + self._set_cookie("C2=value2; path=/", session=None) + assert self._get_cookie() == b"C1=value1" + + def test_session_none_request_cookies(self) -> None: + req = Request( + "http://scrapytest.org/", + cookies={"galleta": "salada"}, + meta={"session": None}, + ) + assert self.mw.process_request(req) is None + assert req.headers.get("Cookie") == b"galleta=salada" + + res = Response( + "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"} + ) + assert self.mw.process_response(req, res) is res + assert self._get_cookie() is None + assert self._get_cookie(session=None) is None + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_dont_merge_cookies_request_cookies(self) -> None: + req = Request( + "http://scrapytest.org/", + cookies={"galleta": "salada"}, + meta={"dont_merge_cookies": True}, + ) + assert self.mw.process_request(req) is None + assert "Cookie" not in req.headers + + def test_session_none_cookie_header(self) -> None: + req = Request( + "http://scrapytest.org/", + headers={"Cookie": "galleta=salada"}, + meta={"session": None}, + ) + assert self.mw.process_request(req) is None + assert req.headers.get("Cookie") == b"galleta=salada" + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_session_over_cookiejar(self) -> None: + self._set_cookie("C1=value1; path=/", session="store1") + assert self._get_cookie(session="store1", cookiejar="store2") == b"C1=value1" + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_cookiejar_key_types(self) -> None: + self._set_cookie("C1=value1; path=/", cookiejar=1) + assert self._get_cookie(cookiejar="1") is None + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_cookiejar_sessions(self) -> None: + self._set_cookie("C1=value1; path=/", cookiejar="store1") + assert "cookiejar:'store1'" in self.mw.crawler.sessions + assert self._get_cookie(session="cookiejar:'store1'") == b"C1=value1" + + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") + def test_jars_deprecated(self) -> None: + self._set_cookie("C1=value1; path=/") + self._set_cookie("C2=value2; path=/", cookiejar="store1") + with pytest.warns(ScrapyDeprecationWarning, match="CookiesMiddleware.jars"): + jars = self.mw.jars + assert sorted(str(key) for key in jars) == ["None", "store1"] + assert [cookie.name for cookie in jars["store1"]] == ["C2"] + + def test_deprecated_keys_ignored(self) -> None: + for key in ("cookiejar", "dont_merge_cookies"): + req = Request( + "http://scrapytest.org/", meta={key: "store1", "session": "store2"} + ) + with pytest.warns( + ScrapyDeprecationWarning, match=f"{key} request meta key is deprecated" + ) as warnings: + assert self.mw.process_request(req) is None + assert "is being ignored" in str(warnings[0].message) + + def test_cookiejar_deprecated(self) -> None: + req = Request("http://scrapytest.org/", meta={"cookiejar": "store1"}) + with pytest.warns( + ScrapyDeprecationWarning, match="cookiejar request meta key is deprecated" + ): + assert self.mw.process_request(req) is None + + def test_dont_merge_cookies_deprecated(self) -> None: + req = Request("http://scrapytest.org/", meta={"dont_merge_cookies": True}) + with pytest.warns( + ScrapyDeprecationWarning, + match="dont_merge_cookies request meta key is deprecated", + ): + assert self.mw.process_request(req) is None + def test_local_domain(self): request = Request("http://example-host/", cookies={"currencyCookie": "USD"}) assert self.mw.process_request(request) is None diff --git a/tests/test_sessions.py b/tests/test_sessions.py new file mode 100644 index 000000000..3a921fcc9 --- /dev/null +++ b/tests/test_sessions.py @@ -0,0 +1,60 @@ +import pytest + +from scrapy.sessions import Session +from scrapy.utils.test import get_crawler + + +def test_get() -> None: + sessions = get_crawler().sessions + session = sessions["foo"] + assert isinstance(session, Session) + assert session.id == "foo" + assert sessions["foo"] is session + assert "foo" in sessions + assert "bar" not in sessions + + +def test_non_string_id() -> None: + sessions = get_crawler().sessions + assert sessions[1].id == "1" + assert sessions[1] is sessions["1"] + assert 1 in sessions + sessions.retire(1) + assert "1" not in sessions + + +def test_create() -> None: + sessions = get_crawler().sessions + session = sessions.create() + assert sessions[session.id] is session + assert sessions.create().id != session.id + + +def test_retire() -> None: + crawler = get_crawler() + sessions = crawler.sessions + sessions["foo"].meta["a"] = 1 + sessions.retire("foo") + assert "foo" not in sessions + assert sessions["foo"].meta == {} + sessions.retire("bar") # unknown IDs are ignored + assert crawler.stats.get_value("sessions/retired") == 1 + + +def test_max(caplog: pytest.LogCaptureFixture) -> None: + crawler = get_crawler(settings_dict={"SESSIONS_MAX": 2}) + sessions = crawler.sessions + a = sessions["a"] + sessions["b"] + assert sessions["a"] is a + caplog.clear() + sessions["c"] + assert "b" not in sessions + assert "a" in sessions + assert crawler.stats.get_value("sessions/created") == 3 + assert crawler.stats.get_value("sessions/dropped") == 1 + assert "Dropped session 'b'" in caplog.text + caplog.clear() + sessions["d"] + assert crawler.stats.get_value("sessions/dropped") == 2 + assert not caplog.text diff --git a/tests/test_spidermiddleware_sessions.py b/tests/test_spidermiddleware_sessions.py new file mode 100644 index 000000000..ae2e29f5e --- /dev/null +++ b/tests/test_spidermiddleware_sessions.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy.http import Request, Response +from scrapy.spidermiddlewares.sessions import SessionsSpiderMiddleware +from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import AsyncIterator + + from tests.mockserver.http import MockServer + +UNSET = object() + + +@pytest.fixture +def mw() -> SessionsSpiderMiddleware: + return build_from_crawler(SessionsSpiderMiddleware, get_crawler(Spider)) + + +def process(mw: SessionsSpiderMiddleware, source_meta: dict[str, Any] | None) -> Any: + response = None + if source_meta is not None: + response = Response("https://example.com") + response.request = Request("https://example.com", meta=source_meta) + request = Request("https://example.com/next") + assert list(mw.process_spider_output(response, [request])) == [request] + return request.meta.get("session", UNSET) + + +def test_inherit(mw: SessionsSpiderMiddleware) -> None: + assert process(mw, {"session": "store1"}) == "store1" + + +def test_inherit_none(mw: SessionsSpiderMiddleware) -> None: + assert process(mw, {"session": None}) is None + + +def test_unset_source(mw: SessionsSpiderMiddleware) -> None: + assert process(mw, {}) is UNSET + + +def test_start_request(mw: SessionsSpiderMiddleware) -> None: + assert process(mw, None) is UNSET + + +def test_own_session_wins(mw: SessionsSpiderMiddleware) -> None: + response = Response("https://example.com") + response.request = Request("https://example.com", meta={"session": "store1"}) + request = Request("https://example.com/next", meta={"session": "store2"}) + list(mw.process_spider_output(response, [request])) + assert request.meta["session"] == "store2" + + +def test_items_pass_through(mw: SessionsSpiderMiddleware) -> None: + response = Response("https://example.com") + response.request = Request("https://example.com", meta={"session": "store1"}) + item = {"a": 1} + assert list(mw.process_spider_output(response, [item])) == [item] + + +class _CookieSpider(Spider): + name = "sessions" + + def __init__(self, mockserver: MockServer, **kwargs: Any): + super().__init__(**kwargs) + self.mockserver = mockserver + self.sent: list[list[str]] = [] + + async def start(self) -> AsyncIterator[Request]: + yield Request(self.mockserver.url("/set-cookie?a=1"), meta={"session": "s1"}) + yield Request(self.mockserver.url("/set-cookie?b=2")) + + def parse(self, response: Response) -> Any: + yield Request( + self.mockserver.url("/echo"), + callback=self.parse_echo, + dont_filter=True, + ) + + def parse_echo(self, response: Response) -> None: + self.sent.append(json.loads(response.text)["headers"].get("Cookie", [])) + + +@coroutine_test +async def test_crawl(mockserver: MockServer) -> None: + crawler = get_crawler(_CookieSpider) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _CookieSpider) + assert sorted(crawler.spider.sent) == [["a=1"], ["b=2"]] + assert "main" in crawler.sessions + assert "s1" in crawler.sessions