diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f4454cd4c..7073cbf98 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2085,8 +2085,51 @@ STICKY_META_KEYS Default: ``[]`` (empty list) -Configure which key/values should be stickied in :attr:`Request.meta `. See -:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` for more info. +The :attr:`Request.meta ` keys to copy automatically +from a response into the follow-up requests yielded by its callback, handled by +:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`. + +Metadata keys already set on a follow-up request are not overwritten. + +For example, the following spider:: + + class MySpider(Spider): + name = "myspider" + + async def start(self): + start_url = "https://toscrape.com/" + yield Request(start_url, meta={"start_url": start_url}) + + def parse(self, response): + for a in response.css("a"): + yield response.follow( + a, + meta={"start_url": response.meta["start_url"]}, + ) + yield { + "url": response.url, + "start_url": response.meta["start_url"], + } + +can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting:: + + class MySpider(Spider): + name = "myspider" + custom_settings = { + "STICKY_META_KEYS": ["start_url"], + } + + async def start(self): + start_url = "https://toscrape.com/" + yield Request(start_url, meta={"start_url": start_url}) + + def parse(self, response): + for a in response.css("a"): + yield response.follow(a) + yield { + "url": response.url, + "start_url": response.meta["start_url"], + } .. setting:: TELNETCONSOLE_ENABLED diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8f5038b90..b30ccd335 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -452,6 +452,19 @@ StartSpiderMiddleware .. autoclass:: StartSpiderMiddleware +StickyMetaParamsMiddleware +-------------------------- + +.. module:: scrapy.spidermiddlewares.stickymeta + :synopsis: Sticky Meta Params Spider Middleware + +.. autoclass:: StickyMetaParamsMiddleware + + Allows copying some request metadata into follow-up requests automatically. + + See the :setting:`STICKY_META_KEYS` setting. + + UrlLengthMiddleware ------------------- @@ -466,87 +479,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - - -StickyMetaParamsMiddleware --------------------------- - -.. module:: scrapy.spidermiddlewares.stickymeta - :synopsis: Spider Middleware that forwards metadata through requests - -.. class:: StickyMetaParamsMiddleware - - When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` - parameters between ``Requests`` and ``Responses``. - - The :class:`StickyMetaParamsMiddleware` can be configured through the following - settings (see the settings documentation for more info): - - * :setting:`STICKY_META_KEYS` - The :ref:`Request.meta ` keys that you want - to automatically forward to next requests. - - See the difference between :ref:`topics-spiders` with and without the middleware: - - Without the middleware:: - - class DontStickySpider(Spider): - name = 'dont_sticky' - start_urls = ['https://www.example.com'] - - def parse(self, response): - for param in range(5): - yield Request( - 'https://www.example.com/next', - meta={'param': param}, - callback=self.parse_2 - ) - - def parse_2(self, response): - # Get important information from response - info = response.xpath('//info/text()').get('info') - # We need to get the param from meta and forward it - param = response.meta['param'] - yield Request( - 'https://www.example.com/next', - meta={'info': info, 'param': param}, - callback=self.parse_3 - ) - - def parse_3(self, response): - # Yield item - yield { - 'param': response.meta['param'], - 'info': response.meta['info'] - } - - With the middleware:: - - class StickySpider(Spider): - name = 'sticky' - start_urls = ['https://www.example.com'] - custom_settings = {'STICKY_META_KEYS': ['param']} # Will always forward the meta param "param" - - def parse(self, response): - for param in range(5): - yield Request( - 'https://www.example.com/next', - meta={'param': param}, - callback=self.parse_2 - ) - - def parse_2(self, response): - # Get important information from response - info = response.xpath('//info/text()').get('info') - # We don't need to get the "param" value from meta and resend it. - yield Request( - 'https://www.example.com/next', - meta={'info': info}, - callback=self.parse_3 - ) - - def parse_3(self, response): - # Yield item - yield { - 'param': response.meta['param'], - 'info': response.meta['info'] - } diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index 8c18e9c98..f18d8e3af 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -1,31 +1,48 @@ +""" +Sticky Meta Params Spider Middleware + +See documentation in docs/topics/spider-middleware.rst +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.exceptions import NotConfigured -from scrapy.http import Request +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Request, Response -class StickyMetaParamsMiddleware: - """Forward a configurable list of meta keys through subsequent requests""" +class StickyMetaParamsMiddleware(BaseSpiderMiddleware): + """Copy a configurable list of :attr:`Request.meta ` + keys from a response into the follow-up requests of its callback. + + The keys to copy are read from the :setting:`STICKY_META_KEYS` setting. + Keys already present in a follow-up request are not overwritten. + """ + + def __init__(self, sticky_meta_keys: list[str]): # pylint: disable=super-init-not-called + self.sticky_meta_keys: list[str] = sticky_meta_keys @classmethod - def from_crawler(cls, crawler): - keys_to_sticky = crawler.settings.getlist('STICKY_META_KEYS') - if not keys_to_sticky: + def from_crawler(cls, crawler: Crawler) -> Self: + sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS") + if not sticky_meta_keys: raise NotConfigured - return cls(keys_to_sticky) + return cls(sticky_meta_keys) - def __init__(self, keys_to_sticky): - self.keys_to_sticky = keys_to_sticky - - def process_spider_output(self, response, result, spider): - sticky_meta = { - k: response.meta[k] - for k in self.keys_to_sticky - if k in response.meta - } - for r in result: - if not isinstance(r, Request): - yield r - continue - for k, v in sticky_meta.items(): - if k not in r.meta: - r.meta[k] = v - yield r + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + return request + for key in self.sticky_meta_keys: + if key in response.meta and key not in request.meta: + request.meta[key] = response.meta[key] + return request diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index a6a774d7c..c8dd972a2 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -1,81 +1,81 @@ -from unittest import TestCase +from __future__ import annotations + +from typing import Any import pytest +from scrapy import Request, Spider from scrapy.exceptions import NotConfigured -from scrapy.http import Request, Response -from scrapy.item import Field, Item +from scrapy.http import Response from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware -from scrapy.spiders import Spider +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +TEST_URL = "http://www.example.com" -class MockItem(Item): - name = Field() +def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware: + crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys}) + return StickyMetaParamsMiddleware.from_crawler(crawler) -class TestStickyMetaParamsMiddleware(TestCase): +async def _run_all_paths( + mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any] +) -> list[list[Any]]: + """Run the spider output through every processing path of the middleware.""" + return [ + list(mw.process_spider_output(response, spider_output)), + await collect_asyncgen( + mw.process_spider_output_async(response, as_async_generator(spider_output)) + ), + ] - def setUp(self): - self.test_url = 'http://www.example.com' - def create_middleware(self, crawler): - return StickyMetaParamsMiddleware.from_crawler(crawler) +def test_not_configured() -> None: + crawler = get_crawler(Spider) + with pytest.raises(NotConfigured): + build_from_crawler(StickyMetaParamsMiddleware, crawler) - def test_middleware_not_enabled(self): - crawler = get_crawler(Spider) - with pytest.raises(NotConfigured): - self.create_middleware(crawler) - def test_sticky_params(self): - crawler = get_crawler(Spider, {'STICKY_META_KEYS': ['param2']}) - middleware = self.create_middleware(crawler) - request = Request( - self.test_url, - meta={ - 'param': 'Will not be stickied', - 'param2': 'Stickied!' - }) - response = Response(self.test_url, request=request) - result = [ - Request(self.test_url), - MockItem(name='dummy') - ] - results = middleware.process_spider_output(response, result, None) - for result in results: - if isinstance(result, Request): - self.assertEqual(result.meta, {'param2': 'Stickied!'}) +def test_comma_separated_string_setting() -> None: + mw = _make_mw("param1,param2") + assert mw.sticky_meta_keys == ["param1", "param2"] - def test_comma_separated_string_setting(self): - crawler = get_crawler(Spider, {'STICKY_META_KEYS': 'param1,param2'}) - middleware = self.create_middleware(crawler) - assert type(middleware.keys_to_sticky) is list - assert middleware.keys_to_sticky == ['param1', 'param2'] - def test_sticky_param_does_not_override_manually_configured_param(self): - crawler = get_crawler( - Spider, - {'STICKY_META_KEYS': ['param', 'param2']} - ) - middleware = self.create_middleware(crawler) - request = Request( - self.test_url, - meta={ - 'param': 'Stickied!', - 'param2': 'Stickied!' - }) - response = Response(self.test_url, request=request) - result = [ - Request(self.test_url, meta={'param': 'Override stickied'}), - MockItem(name='dummy') - ] - results = middleware.process_spider_output(response, result, None) - for result in results: - if isinstance(result, Request): - self.assertEqual( - result.meta, - { - 'param': 'Override stickied', - 'param2': 'Stickied!' - } - ) +@coroutine_test +async def test_sticky_params() -> None: + mw = _make_mw(["param2"]) + request = Request( + TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"} + ) + response = Response(TEST_URL, request=request) + spider_output = [Request(TEST_URL), {"name": "dummy"}] + for processed in await _run_all_paths(mw, response, spider_output): + assert processed[0].meta == {"param2": "Stickied!"} + assert processed[1] == {"name": "dummy"} + + +@coroutine_test +async def test_sticky_param_does_not_override_manually_configured_param() -> None: + mw = _make_mw(["param", "param2"]) + request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"}) + response = Response(TEST_URL, request=request) + spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})] + for processed in await _run_all_paths(mw, response, spider_output): + assert processed[0].meta == { + "param": "Override stickied", + "param2": "Stickied!", + } + + +@coroutine_test +async def test_start_requests_have_no_response() -> None: + """Start seeds are processed with ``response=None`` and are left untouched.""" + mw = _make_mw(["param"]) + start_request = Request(TEST_URL, meta={"param": "value"}) + processed = await collect_asyncgen( + mw.process_start(as_async_generator([start_request])) + ) + assert processed[0].meta == {"param": "value"}