mirror of https://github.com/scrapy/scrapy.git
Address PR review comments on StickyMetaParamsMiddleware
Drop the redundant autoclass description and unused module directive in the docs, remove the stale module-level docstring, fix __init__ to properly call super().__init__(crawler) instead of skipping it, simplify the meta copy with setdefault, and use build_from_crawler in the test helper for consistency.
This commit is contained in:
parent
a2556ce75c
commit
e806885232
|
|
@ -455,14 +455,7 @@ StartSpiderMiddleware
|
|||
StickyMetaParamsMiddleware
|
||||
--------------------------
|
||||
|
||||
.. module:: scrapy.spidermiddlewares.stickymeta
|
||||
:synopsis: Sticky Meta Params Spider Middleware
|
||||
|
||||
.. autoclass:: StickyMetaParamsMiddleware
|
||||
|
||||
Allows copying some request metadata into follow-up requests automatically.
|
||||
|
||||
See the :setting:`STICKY_META_KEYS` setting.
|
||||
.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware
|
||||
|
||||
|
||||
UrlLengthMiddleware
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
"""
|
||||
Sticky Meta Params Spider Middleware
|
||||
|
||||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
|
@ -27,7 +21,8 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware):
|
|||
Keys already present in a follow-up request are not overwritten.
|
||||
"""
|
||||
|
||||
def __init__(self, sticky_meta_keys: list[str]): # pylint: disable=super-init-not-called
|
||||
def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]):
|
||||
super().__init__(crawler)
|
||||
self.sticky_meta_keys: list[str] = sticky_meta_keys
|
||||
|
||||
@classmethod
|
||||
|
|
@ -35,7 +30,7 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware):
|
|||
sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS")
|
||||
if not sticky_meta_keys:
|
||||
raise NotConfigured
|
||||
return cls(sticky_meta_keys)
|
||||
return cls(crawler, sticky_meta_keys)
|
||||
|
||||
def get_processed_request(
|
||||
self, request: Request, response: Response | None
|
||||
|
|
@ -43,6 +38,6 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware):
|
|||
if response is None:
|
||||
return request
|
||||
for key in self.sticky_meta_keys:
|
||||
if key in response.meta and key not in request.meta:
|
||||
request.meta[key] = response.meta[key]
|
||||
if key in response.meta:
|
||||
request.meta.setdefault(key, response.meta[key])
|
||||
return request
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ TEST_URL = "http://www.example.com"
|
|||
|
||||
def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware:
|
||||
crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys})
|
||||
return StickyMetaParamsMiddleware.from_crawler(crawler)
|
||||
return build_from_crawler(StickyMetaParamsMiddleware, crawler)
|
||||
|
||||
|
||||
async def _run_all_paths(
|
||||
|
|
|
|||
Loading…
Reference in New Issue