From e80688523274ccedda384018264009f0427b0fd6 Mon Sep 17 00:00:00 2001 From: Adnan Awan Date: Tue, 14 Jul 2026 20:18:18 +0500 Subject: [PATCH] Address PR review comments on StickyMetaParamsMiddleware Drop the redundant autoclass description and unused module directive in the docs, remove the stale module-level docstring, fix __init__ to properly call super().__init__(crawler) instead of skipping it, simplify the meta copy with setdefault, and use build_from_crawler in the test helper for consistency. --- docs/topics/spider-middleware.rst | 9 +-------- scrapy/spidermiddlewares/stickymeta.py | 15 +++++---------- tests/test_spidermiddleware_stickymeta.py | 2 +- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index b30ccd335..c7d0c1645 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -455,14 +455,7 @@ StartSpiderMiddleware StickyMetaParamsMiddleware -------------------------- -.. module:: scrapy.spidermiddlewares.stickymeta - :synopsis: Sticky Meta Params Spider Middleware - -.. autoclass:: StickyMetaParamsMiddleware - - Allows copying some request metadata into follow-up requests automatically. - - See the :setting:`STICKY_META_KEYS` setting. +.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware UrlLengthMiddleware diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index f18d8e3af..72d7c5752 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -1,9 +1,3 @@ -""" -Sticky Meta Params Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING @@ -27,7 +21,8 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware): Keys already present in a follow-up request are not overwritten. """ - def __init__(self, sticky_meta_keys: list[str]): # pylint: disable=super-init-not-called + def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]): + super().__init__(crawler) self.sticky_meta_keys: list[str] = sticky_meta_keys @classmethod @@ -35,7 +30,7 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware): sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS") if not sticky_meta_keys: raise NotConfigured - return cls(sticky_meta_keys) + return cls(crawler, sticky_meta_keys) def get_processed_request( self, request: Request, response: Response | None @@ -43,6 +38,6 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware): if response is None: return request for key in self.sticky_meta_keys: - if key in response.meta and key not in request.meta: - request.meta[key] = response.meta[key] + if key in response.meta: + request.meta.setdefault(key, response.meta[key]) return request diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index c8dd972a2..d66c1faf8 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -18,7 +18,7 @@ TEST_URL = "http://www.example.com" def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware: crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys}) - return StickyMetaParamsMiddleware.from_crawler(crawler) + return build_from_crawler(StickyMetaParamsMiddleware, crawler) async def _run_all_paths(