diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a5f799444..9110e5fa2 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -208,7 +208,9 @@ Request objects To pass data from one spider callback to another, consider using :attr:`cb_kwargs` instead. However, request metadata may be the right choice in certain scenarios, such as to maintain some debugging data - across all follow-up requests (e.g. the source URL). + across all follow-up requests (e.g. the source URL). To copy some + metadata keys automatically into follow-up requests, consider using the + :setting:`STICKY_META_KEYS` setting. A common use of request metadata is to define request-specific parameters for Scrapy components (extensions, middlewares, etc.). For diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a592f8bc5..dec914f67 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2042,6 +2042,7 @@ Default: "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, + "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000, } A dict containing the spider middlewares enabled by default in Scrapy, and @@ -2087,6 +2088,59 @@ finishes. For more info see: :ref:`topics-stats`. +.. setting:: STICKY_META_KEYS + +STICKY_META_KEYS +---------------- + +Default: ``[]`` (empty list) + +The :attr:`Request.meta ` keys to copy automatically +from a response into the follow-up requests yielded by its callback, handled by +:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`. + +Metadata keys already set on a follow-up request are not overwritten. + +For example, the following spider:: + + class MySpider(Spider): + name = "myspider" + + async def start(self): + start_url = "https://toscrape.com/" + yield Request(start_url, meta={"start_url": start_url}) + + def parse(self, response): + for a in response.css("a"): + yield response.follow( + a, + meta={"start_url": response.meta["start_url"]}, + ) + yield { + "url": response.url, + "start_url": response.meta["start_url"], + } + +can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting:: + + class MySpider(Spider): + name = "myspider" + custom_settings = { + "STICKY_META_KEYS": ["start_url"], + } + + async def start(self): + start_url = "https://toscrape.com/" + yield Request(start_url, meta={"start_url": start_url}) + + def parse(self, response): + for a in response.css("a"): + yield response.follow(a) + yield { + "url": response.url, + "start_url": response.meta["start_url"], + } + .. setting:: TELNETCONSOLE_ENABLED TELNETCONSOLE_ENABLED diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 870adbc8c..3500275fd 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -452,6 +452,12 @@ StartSpiderMiddleware .. autoclass:: StartSpiderMiddleware +StickyMetaParamsMiddleware +-------------------------- + +.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware + + UrlLengthMiddleware ------------------- diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3a7dfd018..b92ccc533 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -203,6 +203,7 @@ __all__ = [ "STATSMAILER_RCPTS", "STATS_CLASS", "STATS_DUMP", + "STICKY_META_KEYS", "TELNETCONSOLE_ENABLED", "TELNETCONSOLE_HOST", "TELNETCONSOLE_PASSWORD", @@ -556,6 +557,7 @@ SPIDER_MIDDLEWARES_BASE = { "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, + "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000, # Spider side } @@ -566,6 +568,8 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] +STICKY_META_KEYS = [] + TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py new file mode 100644 index 000000000..72d7c5752 --- /dev/null +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from scrapy.exceptions import NotConfigured +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + + +class StickyMetaParamsMiddleware(BaseSpiderMiddleware): + """Copy a configurable list of :attr:`Request.meta ` + keys from a response into the follow-up requests of its callback. + + The keys to copy are read from the :setting:`STICKY_META_KEYS` setting. + Keys already present in a follow-up request are not overwritten. + """ + + def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]): + super().__init__(crawler) + self.sticky_meta_keys: list[str] = sticky_meta_keys + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS") + if not sticky_meta_keys: + raise NotConfigured + return cls(crawler, sticky_meta_keys) + + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + return request + for key in self.sticky_meta_keys: + if key in response.meta: + request.meta.setdefault(key, response.meta[key]) + return request diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py new file mode 100644 index 000000000..d66c1faf8 --- /dev/null +++ b/tests/test_spidermiddleware_stickymeta.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from scrapy import Request, Spider +from scrapy.exceptions import NotConfigured +from scrapy.http import Response +from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +TEST_URL = "http://www.example.com" + + +def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware: + crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys}) + return build_from_crawler(StickyMetaParamsMiddleware, crawler) + + +async def _run_all_paths( + mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any] +) -> list[list[Any]]: + """Run the spider output through every processing path of the middleware.""" + return [ + list(mw.process_spider_output(response, spider_output)), + await collect_asyncgen( + mw.process_spider_output_async(response, as_async_generator(spider_output)) + ), + ] + + +def test_not_configured() -> None: + crawler = get_crawler(Spider) + with pytest.raises(NotConfigured): + build_from_crawler(StickyMetaParamsMiddleware, crawler) + + +def test_comma_separated_string_setting() -> None: + mw = _make_mw("param1,param2") + assert mw.sticky_meta_keys == ["param1", "param2"] + + +@coroutine_test +async def test_sticky_params() -> None: + mw = _make_mw(["param2"]) + request = Request( + TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"} + ) + response = Response(TEST_URL, request=request) + spider_output = [Request(TEST_URL), {"name": "dummy"}] + for processed in await _run_all_paths(mw, response, spider_output): + assert processed[0].meta == {"param2": "Stickied!"} + assert processed[1] == {"name": "dummy"} + + +@coroutine_test +async def test_sticky_param_does_not_override_manually_configured_param() -> None: + mw = _make_mw(["param", "param2"]) + request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"}) + response = Response(TEST_URL, request=request) + spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})] + for processed in await _run_all_paths(mw, response, spider_output): + assert processed[0].meta == { + "param": "Override stickied", + "param2": "Stickied!", + } + + +@coroutine_test +async def test_start_requests_have_no_response() -> None: + """Start seeds are processed with ``response=None`` and are left untouched.""" + mw = _make_mw(["param"]) + start_request = Request(TEST_URL, meta={"param": "value"}) + processed = await collect_asyncgen( + mw.process_start(as_async_generator([start_request])) + ) + assert processed[0].meta == {"param": "value"}