mirror of https://github.com/scrapy/scrapy.git
Merge 2a70758ba5 into e28e56aa61
This commit is contained in:
Adrian
GitHub
commit
9040473763
|
|
@ -37,6 +37,7 @@ extensions = [
|
|||
|
||||
redirects = {
|
||||
"topics/broad-crawls": "optimize.html#broad-crawls",
|
||||
"topics/cookies": "sessions.html",
|
||||
}
|
||||
|
||||
templates_path = ["_templates"]
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ Basic concepts
|
|||
topics/item-pipeline
|
||||
topics/feed-exports
|
||||
topics/request-response
|
||||
topics/cookies
|
||||
topics/sessions
|
||||
topics/link-extractors
|
||||
topics/settings
|
||||
topics/exceptions
|
||||
|
|
@ -110,8 +110,8 @@ Basic concepts
|
|||
:doc:`topics/request-response`
|
||||
Understand the classes used to represent HTTP requests and responses.
|
||||
|
||||
:doc:`topics/cookies`
|
||||
Send and receive cookies.
|
||||
:doc:`topics/sessions`
|
||||
Keep cookies and other state across groups of requests.
|
||||
|
||||
:doc:`topics/link-extractors`
|
||||
Convenient classes to extract links to follow from pages.
|
||||
|
|
|
|||
|
|
@ -3260,7 +3260,7 @@ Documentation
|
|||
- Extended documentation for :attr:`.Request.meta`.
|
||||
(:gh:`5565`)
|
||||
|
||||
- Fixed the :reqmeta:`dont_merge_cookies` documentation. (:gh:`5936`,
|
||||
- Fixed the ``dont_merge_cookies`` documentation. (:gh:`5936`,
|
||||
:gh:`6077`)
|
||||
|
||||
- Added a link to Zyte's export guides to the :ref:`feed exports
|
||||
|
|
@ -9019,7 +9019,7 @@ Scrapy changes:
|
|||
- SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`)
|
||||
- StackTraceDump extension: also dump trackref live references (:commit:`fe2ce93`)
|
||||
- nested items now fully supported in JSON and JSONLines exporters
|
||||
- added :reqmeta:`cookiejar` Request meta key to support multiple cookie sessions per spider
|
||||
- added ``cookiejar`` Request meta key to support multiple cookie sessions per spider
|
||||
- decoupled encoding detection code to `w3lib.encoding`_, and ported Scrapy code to use that module
|
||||
- dropped support for Python 2.5. See https://www.zyte.com/blog/scrapy-0-15-dropping-support-for-python-2-5/
|
||||
- dropped support for Twisted 2.5
|
||||
|
|
|
|||
|
|
@ -36,8 +36,9 @@ how you :ref:`configure the downloader middlewares
|
|||
:class:`scrapy.settings.Settings` object.
|
||||
|
||||
The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`,
|
||||
:attr:`request_fingerprinter` and :attr:`stats` attributes get their value
|
||||
when the crawl starts, and raise :exc:`RuntimeError` when read before that.
|
||||
:attr:`request_fingerprinter`, :attr:`sessions` and :attr:`stats` attributes
|
||||
get their value when the crawl starts, and raise :exc:`RuntimeError` when
|
||||
read before that.
|
||||
|
||||
.. versionchanged:: VERSION
|
||||
Those attributes used to be ``None`` before getting their value.
|
||||
|
|
@ -71,6 +72,8 @@ how you :ref:`configure the downloader middlewares
|
|||
|
||||
For the API see :class:`~scrapy.signalmanager.SignalManager` class.
|
||||
|
||||
.. autoattribute:: sessions
|
||||
|
||||
.. attribute:: stats
|
||||
|
||||
The stats collector of this crawler.
|
||||
|
|
|
|||
|
|
@ -1,138 +0,0 @@
|
|||
.. _cookies:
|
||||
.. _cookies-mw:
|
||||
|
||||
=======
|
||||
Cookies
|
||||
=======
|
||||
|
||||
Scrapy keeps track of the cookies that websites set and sends them back on
|
||||
later requests to those websites, just like a web browser does. That is the job
|
||||
of :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which is
|
||||
enabled by default.
|
||||
|
||||
|
||||
Setting cookies on a request
|
||||
============================
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from scrapy import Request
|
||||
|
||||
Use the ``cookies`` parameter of :class:`~scrapy.Request` to send cookies of
|
||||
your own, either as a dict:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
request = Request(
|
||||
url="https://example.com",
|
||||
cookies={"currency": "USD", "country": "UY"},
|
||||
)
|
||||
|
||||
Or as a list of dicts, which also lets you set cookie attributes:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
request = Request(
|
||||
url="https://example.com",
|
||||
cookies=[
|
||||
{
|
||||
"name": "currency",
|
||||
"value": "USD",
|
||||
"domain": "example.com",
|
||||
"path": "/currency",
|
||||
"secure": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
Setting attributes is only useful if the cookies are stored for later requests,
|
||||
i.e. if :reqmeta:`dont_merge_cookies` is not enabled.
|
||||
|
||||
.. caution:: Cookies set through the ``Cookie`` header are not handled by
|
||||
:class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which
|
||||
drops that header.
|
||||
|
||||
.. caution:: When a cookie name or value is a byte sequence that is not UTF-8
|
||||
encoded, the cookie is dropped and a warning is logged. See
|
||||
:ref:`topics-logging-advanced-customization` to customize the logging
|
||||
behavior.
|
||||
|
||||
|
||||
.. reqmeta:: cookiejar
|
||||
|
||||
Multiple cookie sessions per spider
|
||||
===================================
|
||||
|
||||
By default all requests share a single cookie jar (session). To use different
|
||||
ones, pass an identifier in the :reqmeta:`cookiejar` request meta key:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
for i, url in enumerate(urls):
|
||||
yield Request(url, meta={"cookiejar": i}, callback=self.parse_page)
|
||||
|
||||
The :reqmeta:`cookiejar` meta key is not "sticky", so you need to keep passing
|
||||
it along on subsequent requests:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def parse_page(self, response):
|
||||
return Request(
|
||||
"https://example.com/otherpage",
|
||||
meta={"cookiejar": response.meta["cookiejar"]},
|
||||
callback=self.parse_other_page,
|
||||
)
|
||||
|
||||
|
||||
.. reqmeta:: dont_merge_cookies
|
||||
|
||||
Skipping the cookie jar for a request
|
||||
=====================================
|
||||
|
||||
Set the :reqmeta:`dont_merge_cookies` request meta key to ``True`` to keep a
|
||||
request from touching the cookie jar in either direction: no stored cookie is
|
||||
sent with the request, and no cookie received in the response is stored. The
|
||||
cookies of the request itself are ignored as well.
|
||||
|
||||
|
||||
.. setting:: COOKIES_ENABLED
|
||||
|
||||
COOKIES_ENABLED
|
||||
===============
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether to enable :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`.
|
||||
If disabled, no cookies are sent to web servers.
|
||||
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
COOKIES_DEBUG
|
||||
=============
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If enabled, Scrapy logs all cookies sent in requests (i.e. the ``Cookie``
|
||||
header) and all cookies received in responses (i.e. the ``Set-Cookie``
|
||||
header)::
|
||||
|
||||
2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened
|
||||
2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
|
||||
Cookie: clientlanguage_nl=en_EN
|
||||
2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
|
||||
Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
|
||||
Set-Cookie: ip_isocode=US
|
||||
Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
|
||||
2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
|
||||
[...]
|
||||
|
||||
|
||||
CookiesMiddleware
|
||||
=================
|
||||
|
||||
.. module:: scrapy.downloadermiddlewares.cookies
|
||||
:synopsis: Cookies Downloader Middleware
|
||||
|
||||
.. autoclass:: CookiesMiddleware
|
||||
|
|
@ -827,9 +827,7 @@ Those are:
|
|||
* :reqmeta:`allow_offsite`
|
||||
* :reqmeta:`autothrottle_dont_adjust_delay`
|
||||
* :reqmeta:`bindaddress`
|
||||
* :reqmeta:`cookiejar`
|
||||
* :reqmeta:`dont_cache`
|
||||
* :reqmeta:`dont_merge_cookies`
|
||||
* :reqmeta:`dont_obey_robotstxt`
|
||||
* :reqmeta:`dont_redirect`
|
||||
* :reqmeta:`dont_retry`
|
||||
|
|
@ -853,6 +851,7 @@ Those are:
|
|||
* :reqmeta:`redirect_reasons`
|
||||
* :reqmeta:`redirect_urls`
|
||||
* :reqmeta:`referrer_policy`
|
||||
* :reqmeta:`session`
|
||||
* :reqmeta:`verbatim_url`
|
||||
|
||||
.. reqmeta:: bindaddress
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
.. _sessions:
|
||||
.. _cookies:
|
||||
.. _cookies-mw:
|
||||
|
||||
========
|
||||
Sessions
|
||||
========
|
||||
|
||||
A session is the state that a group of requests share, starting with their
|
||||
cookies, which Scrapy stores and sends back on later requests to the same
|
||||
website, like a web browser does.
|
||||
|
||||
Every request uses the same session unless told otherwise, so a crawl behaves
|
||||
like a single browser profile. Use more than one to keep parts of a crawl from
|
||||
sharing state, e.g. to crawl a website through several independent profiles at
|
||||
the same time.
|
||||
|
||||
|
||||
.. _session-choose:
|
||||
|
||||
Choosing the session of a request
|
||||
=================================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
.. invisible-code-block: python
|
||||
|
||||
from scrapy import Request
|
||||
|
||||
.. reqmeta:: session
|
||||
|
||||
Set the :reqmeta:`session` request meta key to a session ID:
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
for index, url in enumerate(urls):
|
||||
yield Request(url, meta={"session": index})
|
||||
|
||||
Any ID works, and the session is created the first time it is used.
|
||||
:meth:`~scrapy.sessions.Sessions.create` covers the case where you have no ID of
|
||||
your own to give: it returns a session that is certainly new, which indexing
|
||||
cannot promise, since the ID you choose may be in use already.
|
||||
|
||||
.. skip: next
|
||||
.. code-block:: python
|
||||
|
||||
session = self.crawler.sessions.create()
|
||||
yield Request(url, meta={"session": session.id})
|
||||
|
||||
The follow-up requests that a spider callback yields stay in the session of the
|
||||
request that got that callback its response, so a session lasts as long as the
|
||||
crawl that follows from it without its ID being passed along by hand. Set
|
||||
:reqmeta:`session` on one of those requests to move it to a different session.
|
||||
|
||||
A request that neither sets :reqmeta:`session` nor inherits one uses the session
|
||||
whose ID is ``"main"``. Everything a crawl does without asking for a session
|
||||
happens there, which makes ``session="main"`` the way to send a request back to
|
||||
it, e.g. from a callback whose own session is a different one.
|
||||
|
||||
Set :reqmeta:`session` to ``None`` for a request to use no session at all: no
|
||||
stored cookie is sent with it and no cookie received in its response is stored.
|
||||
The cookies of the request itself are still sent, and its follow-up requests
|
||||
inherit the lack of a session.
|
||||
|
||||
|
||||
.. _session-registry:
|
||||
|
||||
The session registry
|
||||
====================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Sessions live in :attr:`Crawler.sessions <scrapy.crawler.Crawler.sessions>`,
|
||||
where you can inspect and modify them:
|
||||
|
||||
.. skip: start
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> cookies = crawler.sessions["main"].cookies
|
||||
>>> len(cookies)
|
||||
2
|
||||
>>> cookies.clear()
|
||||
|
||||
.. skip: end
|
||||
|
||||
When a session stops working, e.g. because the website expired it, call
|
||||
:meth:`~scrapy.sessions.Sessions.retire` and retry the request with
|
||||
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request`: the session is
|
||||
gone, so the retry starts a new one under the same ID.
|
||||
|
||||
.. autoclass:: scrapy.sessions.Sessions
|
||||
:members:
|
||||
|
||||
.. autoclass:: scrapy.sessions.Session
|
||||
:members:
|
||||
|
||||
|
||||
Sending cookies with a request
|
||||
==============================
|
||||
|
||||
Use the ``cookies`` parameter of :class:`~scrapy.Request` to send cookies of
|
||||
your own, either as a dict:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
request = Request(
|
||||
url="https://example.com",
|
||||
cookies={"currency": "USD", "country": "UY"},
|
||||
)
|
||||
|
||||
Or as a list of dicts, which also lets you set cookie attributes:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
request = Request(
|
||||
url="https://example.com",
|
||||
cookies=[
|
||||
{
|
||||
"name": "currency",
|
||||
"value": "USD",
|
||||
"domain": "example.com",
|
||||
"path": "/currency",
|
||||
"secure": True,
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
Setting attributes is only useful if the cookies are stored for later requests,
|
||||
i.e. if the request has a session.
|
||||
|
||||
.. caution:: Cookies set through the ``Cookie`` header are not handled by
|
||||
:class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which
|
||||
drops that header.
|
||||
|
||||
.. caution:: When a cookie name or value is a byte sequence that is not UTF-8
|
||||
encoded, the cookie is dropped and a warning is logged. See
|
||||
:ref:`topics-logging-advanced-customization` to customize the logging
|
||||
behavior.
|
||||
|
||||
|
||||
.. setting:: SESSIONS_MAX
|
||||
|
||||
SESSIONS_MAX
|
||||
============
|
||||
|
||||
Default: ``1000``
|
||||
|
||||
Maximum number of sessions to keep in memory. When the limit is reached, the
|
||||
session that has not been used for the longest time is dropped, losing its
|
||||
cookies. The first drop is logged as a warning, and :stat:`sessions/dropped`
|
||||
counts them all.
|
||||
|
||||
Raise it if a crawl needs more sessions alive at the same time, e.g. because it
|
||||
gives every request a session of its own.
|
||||
|
||||
|
||||
.. setting:: COOKIES_ENABLED
|
||||
|
||||
COOKIES_ENABLED
|
||||
===============
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether to enable :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`.
|
||||
If disabled, no cookies are sent to web servers.
|
||||
|
||||
|
||||
.. setting:: COOKIES_DEBUG
|
||||
|
||||
COOKIES_DEBUG
|
||||
=============
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If enabled, Scrapy logs all cookies sent in requests (i.e. the ``Cookie``
|
||||
header) and all cookies received in responses (i.e. the ``Set-Cookie``
|
||||
header)::
|
||||
|
||||
2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened
|
||||
2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
|
||||
Cookie: clientlanguage_nl=en_EN
|
||||
2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
|
||||
Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
|
||||
Set-Cookie: ip_isocode=US
|
||||
Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
|
||||
2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
|
||||
[...]
|
||||
|
||||
|
||||
CookiesMiddleware
|
||||
=================
|
||||
|
||||
.. module:: scrapy.downloadermiddlewares.cookies
|
||||
|
||||
.. autoclass:: CookiesMiddleware
|
||||
|
|
@ -2119,6 +2119,7 @@ Default:
|
|||
{
|
||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
||||
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
||||
"scrapy.spidermiddlewares.sessions.SessionsSpiderMiddleware": 375,
|
||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
|
||||
|
|
|
|||
|
|
@ -459,6 +459,14 @@ Use ``""`` to override the policy for responses with `no referrer policy
|
|||
<https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string>`__.
|
||||
|
||||
|
||||
SessionsSpiderMiddleware
|
||||
------------------------
|
||||
|
||||
.. module:: scrapy.spidermiddlewares.sessions
|
||||
|
||||
.. autoclass:: SessionsSpiderMiddleware
|
||||
|
||||
|
||||
StartSpiderMiddleware
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -722,6 +722,22 @@ one per actual value of the placeholder.
|
|||
:ref:`serialized <request-serialization>`, and hence were stored into the
|
||||
memory queue instead.
|
||||
|
||||
.. stat:: sessions/created
|
||||
|
||||
``sessions/created``
|
||||
Number of :ref:`sessions <sessions>` created.
|
||||
|
||||
.. stat:: sessions/dropped
|
||||
|
||||
``sessions/dropped``
|
||||
Number of sessions dropped to stay within :setting:`SESSIONS_MAX`.
|
||||
|
||||
.. stat:: sessions/retired
|
||||
|
||||
``sessions/retired``
|
||||
Number of sessions deleted with
|
||||
:meth:`~scrapy.sessions.Sessions.retire`.
|
||||
|
||||
.. stat:: spider_exceptions/count
|
||||
|
||||
``spider_exceptions/count``
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from scrapy.addons import AddonManager
|
|||
from scrapy.core.engine import ExecutionEngine
|
||||
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
|
||||
from scrapy.extension import ExtensionManager
|
||||
from scrapy.sessions import Sessions
|
||||
from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
|
||||
from scrapy.signalmanager import SignalManager
|
||||
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
|
||||
|
|
@ -105,6 +106,11 @@ class Crawler:
|
|||
request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = (
|
||||
_LateAttribute()
|
||||
)
|
||||
sessions: _LateAttribute[Sessions] = _LateAttribute()
|
||||
"""The :class:`~scrapy.sessions.Sessions` registry of this crawler.
|
||||
|
||||
See :ref:`sessions`.
|
||||
"""
|
||||
stats: _LateAttribute[StatsCollector] = _LateAttribute()
|
||||
|
||||
def __init__(
|
||||
|
|
@ -137,6 +143,7 @@ class Crawler:
|
|||
self._extensions: ExtensionManager | None = None
|
||||
self._logformatter: LogFormatter | None = None
|
||||
self._request_fingerprinter: RequestFingerprinterProtocol | None = None
|
||||
self._sessions: Sessions | None = None
|
||||
self._stats: StatsCollector | None = None
|
||||
|
||||
def _update_root_log_handler(self) -> None:
|
||||
|
|
@ -154,6 +161,7 @@ class Crawler:
|
|||
"max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"
|
||||
)
|
||||
self.stats = load_object(self.settings["STATS_CLASS"])(self)
|
||||
self.sessions = Sessions(self)
|
||||
|
||||
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
|
||||
self.logformatter = build_from_crawler(lf_cls, self)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,16 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from tldextract import TLDExtract
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.sessions import _MAIN_ID
|
||||
from scrapy.utils.datatypes import LocalCache
|
||||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_unicode
|
||||
|
|
@ -24,6 +26,7 @@ if TYPE_CHECKING:
|
|||
from scrapy import Request, Spider
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http.request import VerboseCookie
|
||||
from scrapy.sessions import Session
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -43,9 +46,15 @@ class CookiesMiddleware:
|
|||
|
||||
crawler: Crawler
|
||||
|
||||
_DEPRECATED_KEYS: ClassVar[dict[str, str]] = {
|
||||
"cookiejar": "Use the session request meta key instead.",
|
||||
"dont_merge_cookies": "Set the session request meta key to None instead.",
|
||||
}
|
||||
|
||||
def __init__(self, debug: bool = False):
|
||||
self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar)
|
||||
self.debug: bool = debug
|
||||
# Session ID of every cookiejar meta key seen so far, for self.jars.
|
||||
self._session_ids: LocalCache[Any, str] = LocalCache()
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
|
|
@ -53,8 +62,53 @@ class CookiesMiddleware:
|
|||
raise NotConfigured
|
||||
o = cls(crawler.settings.getbool("COOKIES_DEBUG"))
|
||||
o.crawler = crawler
|
||||
o._session_ids.limit = crawler.settings.getint("SESSIONS_MAX")
|
||||
return o
|
||||
|
||||
@property
|
||||
def jars(self) -> dict[Any, CookieJar]:
|
||||
"""Cookie jars of the :reqmeta:`cookiejar` request meta keys seen so
|
||||
far."""
|
||||
warnings.warn(
|
||||
"CookiesMiddleware.jars is deprecated, use Crawler.sessions instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
sessions = self.crawler.sessions
|
||||
return {key: sessions[id_].cookies for key, id_ in self._session_ids.items()}
|
||||
|
||||
def _session(self, request: Request) -> Session | None:
|
||||
"""Return the session of *request*, or ``None`` if it has none."""
|
||||
has_session = "session" in request.meta
|
||||
for key in self._DEPRECATED_KEYS:
|
||||
if key in request.meta:
|
||||
self._warn_deprecated_key(key, ignored=has_session)
|
||||
if has_session:
|
||||
session_id = request.meta["session"]
|
||||
return None if session_id is None else self.crawler.sessions[session_id]
|
||||
if request.meta.get("dont_merge_cookies", False):
|
||||
return None
|
||||
jar_key = request.meta.get("cookiejar")
|
||||
jar_id = _MAIN_ID if jar_key is None else f"cookiejar:{jar_key!r}"
|
||||
self._session_ids[jar_key] = jar_id
|
||||
return self.crawler.sessions[jar_id]
|
||||
|
||||
def _warn_deprecated_key(self, key: str, *, ignored: bool) -> None:
|
||||
if ignored:
|
||||
message = (
|
||||
f"The {key} request meta key is deprecated, and it is being "
|
||||
f"ignored because the session request meta key is set on the "
|
||||
f"same request. Remove {key}."
|
||||
)
|
||||
else:
|
||||
message = (
|
||||
f"The {key} request meta key is deprecated. "
|
||||
f"{self._DEPRECATED_KEYS[key]} Note that, unlike {key}, session "
|
||||
f"is inherited by the follow-up requests that a spider callback "
|
||||
f"yields."
|
||||
)
|
||||
warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=3)
|
||||
|
||||
def _process_cookies(
|
||||
self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
|
||||
) -> None:
|
||||
|
|
@ -77,30 +131,34 @@ class CookiesMiddleware:
|
|||
def process_request(
|
||||
self, request: Request, spider: Spider | None = None
|
||||
) -> Request | Response | None:
|
||||
if request.meta.get("dont_merge_cookies", False):
|
||||
session = self._session(request)
|
||||
if session is None:
|
||||
# The cookies of the request are its own, so they are sent even with
|
||||
# no session to merge them into; a jar that no one keeps turns them
|
||||
# into a Cookie header. dont_merge_cookies drops them instead.
|
||||
if request.cookies and not request.meta.get("dont_merge_cookies", False):
|
||||
self._set_cookie_header(CookieJar(), request)
|
||||
return None
|
||||
self._set_cookie_header(session.cookies, request)
|
||||
return None
|
||||
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
def _set_cookie_header(self, jar: CookieJar, request: Request) -> None:
|
||||
cookies = self._get_request_cookies(jar, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
# set Cookie header
|
||||
request.headers.pop("Cookie", None)
|
||||
jar.add_cookie_header(request)
|
||||
self._debug_cookie(request)
|
||||
return None
|
||||
|
||||
@_warn_spider_arg
|
||||
def process_response(
|
||||
self, request: Request, response: Response, spider: Spider | None = None
|
||||
) -> Request | Response:
|
||||
if request.meta.get("dont_merge_cookies", False):
|
||||
session = self._session(request)
|
||||
if session is None:
|
||||
return response
|
||||
|
||||
# extract cookies from Set-Cookie and drop invalid/expired cookies
|
||||
cookiejarkey = request.meta.get("cookiejar")
|
||||
jar = self.jars[cookiejarkey]
|
||||
jar = session.cookies
|
||||
cookies = jar.make_cookies(response, request)
|
||||
self._process_cookies(cookies, jar=jar, request=request)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from uuid import uuid4
|
||||
|
||||
from scrapy.http.cookies import CookieJar
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAIN_ID = "main"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Session:
|
||||
"""State that requests bound to the same session share.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
"""
|
||||
|
||||
id: str
|
||||
"""Session ID, i.e. the value of the :reqmeta:`session` request meta key of
|
||||
the requests bound to this session."""
|
||||
|
||||
cookies: CookieJar = field(default_factory=CookieJar)
|
||||
"""Cookies of the session."""
|
||||
|
||||
meta: dict[Any, Any] = field(default_factory=dict)
|
||||
"""Free-form data about the session."""
|
||||
|
||||
|
||||
class Sessions:
|
||||
"""Registry of the :class:`Session` objects of a crawler, available as
|
||||
:attr:`Crawler.sessions <scrapy.crawler.Crawler.sessions>`.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Indexing by session ID returns the matching session, creating it if it does
|
||||
not exist, e.g. ``crawler.sessions["main"]``. IDs are strings; a value of a
|
||||
different type is used as its :func:`str` form.
|
||||
|
||||
It holds at most :setting:`SESSIONS_MAX` sessions; when full, the session
|
||||
that has not been used for the longest time is dropped.
|
||||
"""
|
||||
|
||||
def __init__(self, crawler: Crawler):
|
||||
self._crawler = crawler
|
||||
self._max = crawler.settings.getint("SESSIONS_MAX")
|
||||
self._sessions: OrderedDict[str, Session] = OrderedDict()
|
||||
self._logged_drop = False
|
||||
|
||||
def __getitem__(self, session_id: Any) -> Session:
|
||||
session_id = str(session_id)
|
||||
if (session := self._sessions.get(session_id)) is not None:
|
||||
self._sessions.move_to_end(session_id)
|
||||
return session
|
||||
while self._sessions and len(self._sessions) >= self._max:
|
||||
self._drop()
|
||||
session = self._sessions[session_id] = Session(session_id)
|
||||
self._crawler.stats.inc_value("sessions/created")
|
||||
return session
|
||||
|
||||
def __contains__(self, session_id: Any) -> bool:
|
||||
return str(session_id) in self._sessions
|
||||
|
||||
def _drop(self) -> None:
|
||||
session_id, _ = self._sessions.popitem(last=False)
|
||||
self._crawler.stats.inc_value("sessions/dropped")
|
||||
if not self._logged_drop:
|
||||
self._logged_drop = True
|
||||
logger.warning(
|
||||
f"Dropped session {session_id!r}, and its state, to stay within "
|
||||
f"SESSIONS_MAX ({self._max}). Raise SESSIONS_MAX if your "
|
||||
f"sessions are being dropped while still in use - no more "
|
||||
f"dropped sessions will be logged.",
|
||||
extra={"spider": self._crawler.spider},
|
||||
)
|
||||
|
||||
def create(self) -> Session:
|
||||
"""Create a session with a unique ID and return it.
|
||||
|
||||
Use it when you need a session that is certainly new and have no ID of
|
||||
your own to give it.
|
||||
"""
|
||||
return self[uuid4().hex]
|
||||
|
||||
def retire(self, session_id: Any) -> None:
|
||||
"""Delete the session with the given ID, if it exists.
|
||||
|
||||
Requests bound to it get a new, empty session.
|
||||
"""
|
||||
if self._sessions.pop(str(session_id), None) is not None:
|
||||
self._crawler.stats.inc_value("sessions/retired")
|
||||
|
|
@ -194,6 +194,7 @@ __all__ = [
|
|||
"SCHEDULER_START_DISK_QUEUE",
|
||||
"SCHEDULER_START_MEMORY_QUEUE",
|
||||
"SCRAPER_SLOT_MAX_ACTIVE_SIZE",
|
||||
"SESSIONS_MAX",
|
||||
"SPIDER_CONTRACTS",
|
||||
"SPIDER_CONTRACTS_BASE",
|
||||
"SPIDER_LOADER_CLASS",
|
||||
|
|
@ -540,6 +541,8 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
|
|||
|
||||
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
|
||||
|
||||
SESSIONS_MAX = 1000
|
||||
|
||||
SPIDER_CONTRACTS = {}
|
||||
SPIDER_CONTRACTS_BASE = {
|
||||
"scrapy.contracts.default.UrlContract": 1,
|
||||
|
|
@ -557,6 +560,7 @@ SPIDER_MIDDLEWARES_BASE = {
|
|||
# Engine side
|
||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
||||
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
||||
"scrapy.spidermiddlewares.sessions.SessionsSpiderMiddleware": 375,
|
||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
|
||||
class SessionsSpiderMiddleware(BaseSpiderMiddleware):
|
||||
"""Bind requests from a spider callback to the :ref:`session <sessions>` of
|
||||
the request whose response reached that callback.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
Requests that set the :reqmeta:`session` request meta key themselves keep
|
||||
their own session.
|
||||
"""
|
||||
|
||||
def get_processed_request(
|
||||
self, request: Request, response: Response | None
|
||||
) -> Request | None:
|
||||
if (
|
||||
response is not None
|
||||
and "session" not in request.meta
|
||||
and "session" in response.meta
|
||||
):
|
||||
request.meta["session"] = response.meta["session"]
|
||||
return request
|
||||
|
|
@ -7,7 +7,7 @@ import pytest
|
|||
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
|
||||
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.http.request import CookiesT, VerboseCookie
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
|
|
@ -190,6 +190,7 @@ class TestCookiesMiddleware:
|
|||
assert self.mw.process_request(req2) is None
|
||||
assert "Cookie" in req2.headers
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_dont_merge_cookies(self):
|
||||
# merge some cookies into jar
|
||||
headers = {"Set-Cookie": "C1=value1; path=/"}
|
||||
|
|
@ -280,6 +281,7 @@ class TestCookiesMiddleware:
|
|||
req2.headers.get("Cookie"), b"C1=value1; galleta=salada"
|
||||
)
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_cookiejar_key(self):
|
||||
req = Request(
|
||||
"http://scrapytest.org/",
|
||||
|
|
@ -340,6 +342,117 @@ class TestCookiesMiddleware:
|
|||
assert self.mw.process_request(req6) is None
|
||||
assert req6.headers.get("Cookie") is None
|
||||
|
||||
def _set_cookie(self, value: str, **meta: Any) -> None:
|
||||
req = Request("http://scrapytest.org/", meta=meta)
|
||||
assert self.mw.process_request(req) is None
|
||||
res = Response("http://scrapytest.org/", headers={"Set-Cookie": value})
|
||||
assert self.mw.process_response(req, res) is res
|
||||
|
||||
def _get_cookie(self, **meta: Any) -> bytes | None:
|
||||
req = Request("http://scrapytest.org/", meta=meta)
|
||||
assert self.mw.process_request(req) is None
|
||||
return req.headers.get("Cookie")
|
||||
|
||||
def test_session_key(self) -> None:
|
||||
self._set_cookie("C1=value1; path=/", session="store1")
|
||||
self._set_cookie("C2=value2; path=/", session="store2")
|
||||
assert self._get_cookie(session="store1") == b"C1=value1"
|
||||
assert self._get_cookie(session="store2") == b"C2=value2"
|
||||
assert self._get_cookie() is None
|
||||
assert self._get_cookie(session="main") is None
|
||||
|
||||
def test_session_none(self) -> None:
|
||||
self._set_cookie("C1=value1; path=/")
|
||||
assert self._get_cookie(session=None) is None
|
||||
self._set_cookie("C2=value2; path=/", session=None)
|
||||
assert self._get_cookie() == b"C1=value1"
|
||||
|
||||
def test_session_none_request_cookies(self) -> None:
|
||||
req = Request(
|
||||
"http://scrapytest.org/",
|
||||
cookies={"galleta": "salada"},
|
||||
meta={"session": None},
|
||||
)
|
||||
assert self.mw.process_request(req) is None
|
||||
assert req.headers.get("Cookie") == b"galleta=salada"
|
||||
|
||||
res = Response(
|
||||
"http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
|
||||
)
|
||||
assert self.mw.process_response(req, res) is res
|
||||
assert self._get_cookie() is None
|
||||
assert self._get_cookie(session=None) is None
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_dont_merge_cookies_request_cookies(self) -> None:
|
||||
req = Request(
|
||||
"http://scrapytest.org/",
|
||||
cookies={"galleta": "salada"},
|
||||
meta={"dont_merge_cookies": True},
|
||||
)
|
||||
assert self.mw.process_request(req) is None
|
||||
assert "Cookie" not in req.headers
|
||||
|
||||
def test_session_none_cookie_header(self) -> None:
|
||||
req = Request(
|
||||
"http://scrapytest.org/",
|
||||
headers={"Cookie": "galleta=salada"},
|
||||
meta={"session": None},
|
||||
)
|
||||
assert self.mw.process_request(req) is None
|
||||
assert req.headers.get("Cookie") == b"galleta=salada"
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_session_over_cookiejar(self) -> None:
|
||||
self._set_cookie("C1=value1; path=/", session="store1")
|
||||
assert self._get_cookie(session="store1", cookiejar="store2") == b"C1=value1"
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_cookiejar_key_types(self) -> None:
|
||||
self._set_cookie("C1=value1; path=/", cookiejar=1)
|
||||
assert self._get_cookie(cookiejar="1") is None
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_cookiejar_sessions(self) -> None:
|
||||
self._set_cookie("C1=value1; path=/", cookiejar="store1")
|
||||
assert "cookiejar:'store1'" in self.mw.crawler.sessions
|
||||
assert self._get_cookie(session="cookiejar:'store1'") == b"C1=value1"
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_jars_deprecated(self) -> None:
|
||||
self._set_cookie("C1=value1; path=/")
|
||||
self._set_cookie("C2=value2; path=/", cookiejar="store1")
|
||||
with pytest.warns(ScrapyDeprecationWarning, match="CookiesMiddleware.jars"):
|
||||
jars = self.mw.jars
|
||||
assert sorted(str(key) for key in jars) == ["None", "store1"]
|
||||
assert [cookie.name for cookie in jars["store1"]] == ["C2"]
|
||||
|
||||
def test_deprecated_keys_ignored(self) -> None:
|
||||
for key in ("cookiejar", "dont_merge_cookies"):
|
||||
req = Request(
|
||||
"http://scrapytest.org/", meta={key: "store1", "session": "store2"}
|
||||
)
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match=f"{key} request meta key is deprecated"
|
||||
) as warnings:
|
||||
assert self.mw.process_request(req) is None
|
||||
assert "is being ignored" in str(warnings[0].message)
|
||||
|
||||
def test_cookiejar_deprecated(self) -> None:
|
||||
req = Request("http://scrapytest.org/", meta={"cookiejar": "store1"})
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match="cookiejar request meta key is deprecated"
|
||||
):
|
||||
assert self.mw.process_request(req) is None
|
||||
|
||||
def test_dont_merge_cookies_deprecated(self) -> None:
|
||||
req = Request("http://scrapytest.org/", meta={"dont_merge_cookies": True})
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="dont_merge_cookies request meta key is deprecated",
|
||||
):
|
||||
assert self.mw.process_request(req) is None
|
||||
|
||||
def test_local_domain(self):
|
||||
request = Request("http://example-host/", cookies={"currencyCookie": "USD"})
|
||||
assert self.mw.process_request(request) is None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
import pytest
|
||||
|
||||
from scrapy.sessions import Session
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
def test_get() -> None:
|
||||
sessions = get_crawler().sessions
|
||||
session = sessions["foo"]
|
||||
assert isinstance(session, Session)
|
||||
assert session.id == "foo"
|
||||
assert sessions["foo"] is session
|
||||
assert "foo" in sessions
|
||||
assert "bar" not in sessions
|
||||
|
||||
|
||||
def test_non_string_id() -> None:
|
||||
sessions = get_crawler().sessions
|
||||
assert sessions[1].id == "1"
|
||||
assert sessions[1] is sessions["1"]
|
||||
assert 1 in sessions
|
||||
sessions.retire(1)
|
||||
assert "1" not in sessions
|
||||
|
||||
|
||||
def test_create() -> None:
|
||||
sessions = get_crawler().sessions
|
||||
session = sessions.create()
|
||||
assert sessions[session.id] is session
|
||||
assert sessions.create().id != session.id
|
||||
|
||||
|
||||
def test_retire() -> None:
|
||||
crawler = get_crawler()
|
||||
sessions = crawler.sessions
|
||||
sessions["foo"].meta["a"] = 1
|
||||
sessions.retire("foo")
|
||||
assert "foo" not in sessions
|
||||
assert sessions["foo"].meta == {}
|
||||
sessions.retire("bar") # unknown IDs are ignored
|
||||
assert crawler.stats.get_value("sessions/retired") == 1
|
||||
|
||||
|
||||
def test_max(caplog: pytest.LogCaptureFixture) -> None:
|
||||
crawler = get_crawler(settings_dict={"SESSIONS_MAX": 2})
|
||||
sessions = crawler.sessions
|
||||
a = sessions["a"]
|
||||
sessions["b"]
|
||||
assert sessions["a"] is a
|
||||
caplog.clear()
|
||||
sessions["c"]
|
||||
assert "b" not in sessions
|
||||
assert "a" in sessions
|
||||
assert crawler.stats.get_value("sessions/created") == 3
|
||||
assert crawler.stats.get_value("sessions/dropped") == 1
|
||||
assert "Dropped session 'b'" in caplog.text
|
||||
caplog.clear()
|
||||
sessions["d"]
|
||||
assert crawler.stats.get_value("sessions/dropped") == 2
|
||||
assert not caplog.text
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.spidermiddlewares.sessions import SessionsSpiderMiddleware
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
UNSET = object()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mw() -> SessionsSpiderMiddleware:
|
||||
return build_from_crawler(SessionsSpiderMiddleware, get_crawler(Spider))
|
||||
|
||||
|
||||
def process(mw: SessionsSpiderMiddleware, source_meta: dict[str, Any] | None) -> Any:
|
||||
response = None
|
||||
if source_meta is not None:
|
||||
response = Response("https://example.com")
|
||||
response.request = Request("https://example.com", meta=source_meta)
|
||||
request = Request("https://example.com/next")
|
||||
assert list(mw.process_spider_output(response, [request])) == [request]
|
||||
return request.meta.get("session", UNSET)
|
||||
|
||||
|
||||
def test_inherit(mw: SessionsSpiderMiddleware) -> None:
|
||||
assert process(mw, {"session": "store1"}) == "store1"
|
||||
|
||||
|
||||
def test_inherit_none(mw: SessionsSpiderMiddleware) -> None:
|
||||
assert process(mw, {"session": None}) is None
|
||||
|
||||
|
||||
def test_unset_source(mw: SessionsSpiderMiddleware) -> None:
|
||||
assert process(mw, {}) is UNSET
|
||||
|
||||
|
||||
def test_start_request(mw: SessionsSpiderMiddleware) -> None:
|
||||
assert process(mw, None) is UNSET
|
||||
|
||||
|
||||
def test_own_session_wins(mw: SessionsSpiderMiddleware) -> None:
|
||||
response = Response("https://example.com")
|
||||
response.request = Request("https://example.com", meta={"session": "store1"})
|
||||
request = Request("https://example.com/next", meta={"session": "store2"})
|
||||
list(mw.process_spider_output(response, [request]))
|
||||
assert request.meta["session"] == "store2"
|
||||
|
||||
|
||||
def test_items_pass_through(mw: SessionsSpiderMiddleware) -> None:
|
||||
response = Response("https://example.com")
|
||||
response.request = Request("https://example.com", meta={"session": "store1"})
|
||||
item = {"a": 1}
|
||||
assert list(mw.process_spider_output(response, [item])) == [item]
|
||||
|
||||
|
||||
class _CookieSpider(Spider):
|
||||
name = "sessions"
|
||||
|
||||
def __init__(self, mockserver: MockServer, **kwargs: Any):
|
||||
super().__init__(**kwargs)
|
||||
self.mockserver = mockserver
|
||||
self.sent: list[list[str]] = []
|
||||
|
||||
async def start(self) -> AsyncIterator[Request]:
|
||||
yield Request(self.mockserver.url("/set-cookie?a=1"), meta={"session": "s1"})
|
||||
yield Request(self.mockserver.url("/set-cookie?b=2"))
|
||||
|
||||
def parse(self, response: Response) -> Any:
|
||||
yield Request(
|
||||
self.mockserver.url("/echo"),
|
||||
callback=self.parse_echo,
|
||||
dont_filter=True,
|
||||
)
|
||||
|
||||
def parse_echo(self, response: Response) -> None:
|
||||
self.sent.append(json.loads(response.text)["headers"].get("Cookie", []))
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_crawl(mockserver: MockServer) -> None:
|
||||
crawler = get_crawler(_CookieSpider)
|
||||
await crawler.crawl_async(mockserver=mockserver)
|
||||
assert isinstance(crawler.spider, _CookieSpider)
|
||||
assert sorted(crawler.spider.sent) == [["a=1"], ["b=2"]]
|
||||
assert "main" in crawler.sessions
|
||||
assert "s1" in crawler.sessions
|
||||
Loading…
Reference in New Issue