Merge e806885232 into d8ba1571e7

docs/topics/request-response.rst

@@ -208,7 +208,9 @@ Request objects
         To pass data from one spider callback to another, consider using
         :attr:`cb_kwargs` instead. However, request metadata may be the right
         choice in certain scenarios, such as to maintain some debugging data
-        across all follow-up requests (e.g. the source URL).
+        across all follow-up requests (e.g. the source URL). To copy some
+        metadata keys automatically into follow-up requests, consider using the
+        :setting:`STICKY_META_KEYS` setting.
 
         A common use of request metadata is to define request-specific
         parameters for Scrapy components (extensions, middlewares, etc.). For

docs/topics/settings.rst

@@ -2042,6 +2042,7 @@ Default:
         "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
         "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
         "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
+        "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000,
     }
 
 A dict containing the spider middlewares enabled by default in Scrapy, and
@@ -2087,6 +2088,59 @@ finishes.
 
 For more info see: :ref:`topics-stats`.
 
+.. setting:: STICKY_META_KEYS
+
+STICKY_META_KEYS
+----------------
+
+Default: ``[]`` (empty list)
+
+The :attr:`Request.meta <scrapy.http.Request.meta>` keys to copy automatically
+from a response into the follow-up requests yielded by its callback, handled by
+:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`.
+
+Metadata keys already set on a follow-up request are not overwritten.
+
+For example, the following spider::
+
+    class MySpider(Spider):
+        name = "myspider"
+
+        async def start(self):
+            start_url = "https://toscrape.com/"
+            yield Request(start_url, meta={"start_url": start_url})
+
+        def parse(self, response):
+            for a in response.css("a"):
+                yield response.follow(
+                    a,
+                    meta={"start_url": response.meta["start_url"]},
+                )
+            yield {
+                "url": response.url,
+                "start_url": response.meta["start_url"],
+            }
+
+can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting::
+
+    class MySpider(Spider):
+        name = "myspider"
+        custom_settings = {
+            "STICKY_META_KEYS": ["start_url"],
+        }
+
+        async def start(self):
+            start_url = "https://toscrape.com/"
+            yield Request(start_url, meta={"start_url": start_url})
+
+        def parse(self, response):
+            for a in response.css("a"):
+                yield response.follow(a)
+            yield {
+                "url": response.url,
+                "start_url": response.meta["start_url"],
+            }
+
 .. setting:: TELNETCONSOLE_ENABLED
 
 TELNETCONSOLE_ENABLED

docs/topics/spider-middleware.rst

@@ -452,6 +452,12 @@ StartSpiderMiddleware
 .. autoclass:: StartSpiderMiddleware
 
 
+StickyMetaParamsMiddleware
+--------------------------
+
+.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware
+
+
 UrlLengthMiddleware
 -------------------
 

scrapy/settings/default_settings.py

@@ -203,6 +203,7 @@ __all__ = [
     "STATSMAILER_RCPTS",
     "STATS_CLASS",
     "STATS_DUMP",
+    "STICKY_META_KEYS",
     "TELNETCONSOLE_ENABLED",
     "TELNETCONSOLE_HOST",
     "TELNETCONSOLE_PASSWORD",
@@ -556,6 +557,7 @@ SPIDER_MIDDLEWARES_BASE = {
     "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
     "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
     "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
+    "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000,
     # Spider side
 }
 
@@ -566,6 +568,8 @@ STATS_DUMP = True
 
 STATSMAILER_RCPTS = []
 
+STICKY_META_KEYS = []
+
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_HOST = "127.0.0.1"
 TELNETCONSOLE_PORT = [6023, 6073]

scrapy/spidermiddlewares/stickymeta.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from scrapy.exceptions import NotConfigured
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+    from scrapy.crawler import Crawler
+    from scrapy.http import Request, Response
+
+
+class StickyMetaParamsMiddleware(BaseSpiderMiddleware):
+    """Copy a configurable list of :attr:`Request.meta <scrapy.http.Request.meta>`
+    keys from a response into the follow-up requests of its callback.
+
+    The keys to copy are read from the :setting:`STICKY_META_KEYS` setting.
+    Keys already present in a follow-up request are not overwritten.
+    """
+
+    def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]):
+        super().__init__(crawler)
+        self.sticky_meta_keys: list[str] = sticky_meta_keys
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS")
+        if not sticky_meta_keys:
+            raise NotConfigured
+        return cls(crawler, sticky_meta_keys)
+
+    def get_processed_request(
+        self, request: Request, response: Response | None
+    ) -> Request | None:
+        if response is None:
+            return request
+        for key in self.sticky_meta_keys:
+            if key in response.meta:
+                request.meta.setdefault(key, response.meta[key])
+        return request

tests/test_spidermiddleware_stickymeta.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from scrapy import Request, Spider
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+TEST_URL = "http://www.example.com"
+
+
+def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware:
+    crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys})
+    return build_from_crawler(StickyMetaParamsMiddleware, crawler)
+
+
+async def _run_all_paths(
+    mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any]
+) -> list[list[Any]]:
+    """Run the spider output through every processing path of the middleware."""
+    return [
+        list(mw.process_spider_output(response, spider_output)),
+        await collect_asyncgen(
+            mw.process_spider_output_async(response, as_async_generator(spider_output))
+        ),
+    ]
+
+
+def test_not_configured() -> None:
+    crawler = get_crawler(Spider)
+    with pytest.raises(NotConfigured):
+        build_from_crawler(StickyMetaParamsMiddleware, crawler)
+
+
+def test_comma_separated_string_setting() -> None:
+    mw = _make_mw("param1,param2")
+    assert mw.sticky_meta_keys == ["param1", "param2"]
+
+
+@coroutine_test
+async def test_sticky_params() -> None:
+    mw = _make_mw(["param2"])
+    request = Request(
+        TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"}
+    )
+    response = Response(TEST_URL, request=request)
+    spider_output = [Request(TEST_URL), {"name": "dummy"}]
+    for processed in await _run_all_paths(mw, response, spider_output):
+        assert processed[0].meta == {"param2": "Stickied!"}
+        assert processed[1] == {"name": "dummy"}
+
+
+@coroutine_test
+async def test_sticky_param_does_not_override_manually_configured_param() -> None:
+    mw = _make_mw(["param", "param2"])
+    request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"})
+    response = Response(TEST_URL, request=request)
+    spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})]
+    for processed in await _run_all_paths(mw, response, spider_output):
+        assert processed[0].meta == {
+            "param": "Override stickied",
+            "param2": "Stickied!",
+        }
+
+
+@coroutine_test
+async def test_start_requests_have_no_response() -> None:
+    """Start seeds are processed with ``response=None`` and are left untouched."""
+    mw = _make_mw(["param"])
+    start_request = Request(TEST_URL, meta={"param": "value"})
+    processed = await collect_asyncgen(
+        mw.process_start(as_async_generator([start_request]))
+    )
+    assert processed[0].meta == {"param": "value"}