mirror of https://github.com/scrapy/scrapy.git
Modernize StickyMetaParamsMiddleware and address PR #4141 review
- Subclass BaseSpiderMiddleware and implement get_processed_request(), gaining async process_spider_output support and start-seed handling. - Add type hints and `from __future__ import annotations`. - Rename keys_to_sticky -> sticky_meta_keys (public API consistency). - Define STICKY_META_KEYS default and add it to __all__. - Rework docs: autoclass entry (alphabetical, before UrlLengthMiddleware), move the before/after example into the STICKY_META_KEYS setting docs. - Rewrite tests to cover sync + async output paths and start seeds. Co-authored-by: Luiz Francisco Rodrigues da Silva <luizfrdasilva@gmail.com> Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Adnan Awan
parent
38fa7fe5d0
commit
ed1acc295a
|
|
@ -2085,8 +2085,51 @@ STICKY_META_KEYS
|
|||
|
||||
Default: ``[]`` (empty list)
|
||||
|
||||
Configure which key/values should be stickied in :attr:`Request.meta <scrapy.http.Request.meta>`. See
|
||||
:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` for more info.
|
||||
The :attr:`Request.meta <scrapy.http.Request.meta>` keys to copy automatically
|
||||
from a response into the follow-up requests yielded by its callback, handled by
|
||||
:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`.
|
||||
|
||||
Metadata keys already set on a follow-up request are not overwritten.
|
||||
|
||||
For example, the following spider::
|
||||
|
||||
class MySpider(Spider):
|
||||
name = "myspider"
|
||||
|
||||
async def start(self):
|
||||
start_url = "https://toscrape.com/"
|
||||
yield Request(start_url, meta={"start_url": start_url})
|
||||
|
||||
def parse(self, response):
|
||||
for a in response.css("a"):
|
||||
yield response.follow(
|
||||
a,
|
||||
meta={"start_url": response.meta["start_url"]},
|
||||
)
|
||||
yield {
|
||||
"url": response.url,
|
||||
"start_url": response.meta["start_url"],
|
||||
}
|
||||
|
||||
can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting::
|
||||
|
||||
class MySpider(Spider):
|
||||
name = "myspider"
|
||||
custom_settings = {
|
||||
"STICKY_META_KEYS": ["start_url"],
|
||||
}
|
||||
|
||||
async def start(self):
|
||||
start_url = "https://toscrape.com/"
|
||||
yield Request(start_url, meta={"start_url": start_url})
|
||||
|
||||
def parse(self, response):
|
||||
for a in response.css("a"):
|
||||
yield response.follow(a)
|
||||
yield {
|
||||
"url": response.url,
|
||||
"start_url": response.meta["start_url"],
|
||||
}
|
||||
|
||||
.. setting:: TELNETCONSOLE_ENABLED
|
||||
|
||||
|
|
|
|||
|
|
@ -452,6 +452,19 @@ StartSpiderMiddleware
|
|||
.. autoclass:: StartSpiderMiddleware
|
||||
|
||||
|
||||
StickyMetaParamsMiddleware
|
||||
--------------------------
|
||||
|
||||
.. module:: scrapy.spidermiddlewares.stickymeta
|
||||
:synopsis: Sticky Meta Params Spider Middleware
|
||||
|
||||
.. autoclass:: StickyMetaParamsMiddleware
|
||||
|
||||
Allows copying some request metadata into follow-up requests automatically.
|
||||
|
||||
See the :setting:`STICKY_META_KEYS` setting.
|
||||
|
||||
|
||||
UrlLengthMiddleware
|
||||
-------------------
|
||||
|
||||
|
|
@ -466,87 +479,3 @@ UrlLengthMiddleware
|
|||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
|
||||
|
||||
|
||||
StickyMetaParamsMiddleware
|
||||
--------------------------
|
||||
|
||||
.. module:: scrapy.spidermiddlewares.stickymeta
|
||||
:synopsis: Spider Middleware that forwards metadata through requests
|
||||
|
||||
.. class:: StickyMetaParamsMiddleware
|
||||
|
||||
When enabled and configured, this middleware forwards the desired :ref:`Request.meta <topics-request-meta>`
|
||||
parameters between ``Requests`` and ``Responses``.
|
||||
|
||||
The :class:`StickyMetaParamsMiddleware` can be configured through the following
|
||||
settings (see the settings documentation for more info):
|
||||
|
||||
* :setting:`STICKY_META_KEYS` - The :ref:`Request.meta <topics-request-meta>` keys that you want
|
||||
to automatically forward to next requests.
|
||||
|
||||
See the difference between :ref:`topics-spiders` with and without the middleware:
|
||||
|
||||
Without the middleware::
|
||||
|
||||
class DontStickySpider(Spider):
|
||||
name = 'dont_sticky'
|
||||
start_urls = ['https://www.example.com']
|
||||
|
||||
def parse(self, response):
|
||||
for param in range(5):
|
||||
yield Request(
|
||||
'https://www.example.com/next',
|
||||
meta={'param': param},
|
||||
callback=self.parse_2
|
||||
)
|
||||
|
||||
def parse_2(self, response):
|
||||
# Get important information from response
|
||||
info = response.xpath('//info/text()').get('info')
|
||||
# We need to get the param from meta and forward it
|
||||
param = response.meta['param']
|
||||
yield Request(
|
||||
'https://www.example.com/next',
|
||||
meta={'info': info, 'param': param},
|
||||
callback=self.parse_3
|
||||
)
|
||||
|
||||
def parse_3(self, response):
|
||||
# Yield item
|
||||
yield {
|
||||
'param': response.meta['param'],
|
||||
'info': response.meta['info']
|
||||
}
|
||||
|
||||
With the middleware::
|
||||
|
||||
class StickySpider(Spider):
|
||||
name = 'sticky'
|
||||
start_urls = ['https://www.example.com']
|
||||
custom_settings = {'STICKY_META_KEYS': ['param']} # Will always forward the meta param "param"
|
||||
|
||||
def parse(self, response):
|
||||
for param in range(5):
|
||||
yield Request(
|
||||
'https://www.example.com/next',
|
||||
meta={'param': param},
|
||||
callback=self.parse_2
|
||||
)
|
||||
|
||||
def parse_2(self, response):
|
||||
# Get important information from response
|
||||
info = response.xpath('//info/text()').get('info')
|
||||
# We don't need to get the "param" value from meta and resend it.
|
||||
yield Request(
|
||||
'https://www.example.com/next',
|
||||
meta={'info': info},
|
||||
callback=self.parse_3
|
||||
)
|
||||
|
||||
def parse_3(self, response):
|
||||
# Yield item
|
||||
yield {
|
||||
'param': response.meta['param'],
|
||||
'info': response.meta['info']
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,31 +1,48 @@
|
|||
"""
|
||||
Sticky Meta Params Spider Middleware
|
||||
|
||||
See documentation in docs/topics/spider-middleware.rst
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Request
|
||||
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
|
||||
class StickyMetaParamsMiddleware:
|
||||
"""Forward a configurable list of meta keys through subsequent requests"""
|
||||
class StickyMetaParamsMiddleware(BaseSpiderMiddleware):
|
||||
"""Copy a configurable list of :attr:`Request.meta <scrapy.http.Request.meta>`
|
||||
keys from a response into the follow-up requests of its callback.
|
||||
|
||||
The keys to copy are read from the :setting:`STICKY_META_KEYS` setting.
|
||||
Keys already present in a follow-up request are not overwritten.
|
||||
"""
|
||||
|
||||
def __init__(self, sticky_meta_keys: list[str]): # pylint: disable=super-init-not-called
|
||||
self.sticky_meta_keys: list[str] = sticky_meta_keys
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
keys_to_sticky = crawler.settings.getlist('STICKY_META_KEYS')
|
||||
if not keys_to_sticky:
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS")
|
||||
if not sticky_meta_keys:
|
||||
raise NotConfigured
|
||||
return cls(keys_to_sticky)
|
||||
return cls(sticky_meta_keys)
|
||||
|
||||
def __init__(self, keys_to_sticky):
|
||||
self.keys_to_sticky = keys_to_sticky
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
sticky_meta = {
|
||||
k: response.meta[k]
|
||||
for k in self.keys_to_sticky
|
||||
if k in response.meta
|
||||
}
|
||||
for r in result:
|
||||
if not isinstance(r, Request):
|
||||
yield r
|
||||
continue
|
||||
for k, v in sticky_meta.items():
|
||||
if k not in r.meta:
|
||||
r.meta[k] = v
|
||||
yield r
|
||||
def get_processed_request(
|
||||
self, request: Request, response: Response | None
|
||||
) -> Request | None:
|
||||
if response is None:
|
||||
return request
|
||||
for key in self.sticky_meta_keys:
|
||||
if key in response.meta and key not in request.meta:
|
||||
request.meta[key] = response.meta[key]
|
||||
return request
|
||||
|
|
|
|||
|
|
@ -1,81 +1,81 @@
|
|||
from unittest import TestCase
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.item import Field, Item
|
||||
from scrapy.http import Response
|
||||
from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
TEST_URL = "http://www.example.com"
|
||||
|
||||
|
||||
class MockItem(Item):
|
||||
name = Field()
|
||||
def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware:
|
||||
crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys})
|
||||
return StickyMetaParamsMiddleware.from_crawler(crawler)
|
||||
|
||||
|
||||
class TestStickyMetaParamsMiddleware(TestCase):
|
||||
async def _run_all_paths(
|
||||
mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any]
|
||||
) -> list[list[Any]]:
|
||||
"""Run the spider output through every processing path of the middleware."""
|
||||
return [
|
||||
list(mw.process_spider_output(response, spider_output)),
|
||||
await collect_asyncgen(
|
||||
mw.process_spider_output_async(response, as_async_generator(spider_output))
|
||||
),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
self.test_url = 'http://www.example.com'
|
||||
|
||||
def create_middleware(self, crawler):
|
||||
return StickyMetaParamsMiddleware.from_crawler(crawler)
|
||||
def test_not_configured() -> None:
|
||||
crawler = get_crawler(Spider)
|
||||
with pytest.raises(NotConfigured):
|
||||
build_from_crawler(StickyMetaParamsMiddleware, crawler)
|
||||
|
||||
def test_middleware_not_enabled(self):
|
||||
crawler = get_crawler(Spider)
|
||||
with pytest.raises(NotConfigured):
|
||||
self.create_middleware(crawler)
|
||||
|
||||
def test_sticky_params(self):
|
||||
crawler = get_crawler(Spider, {'STICKY_META_KEYS': ['param2']})
|
||||
middleware = self.create_middleware(crawler)
|
||||
request = Request(
|
||||
self.test_url,
|
||||
meta={
|
||||
'param': 'Will not be stickied',
|
||||
'param2': 'Stickied!'
|
||||
})
|
||||
response = Response(self.test_url, request=request)
|
||||
result = [
|
||||
Request(self.test_url),
|
||||
MockItem(name='dummy')
|
||||
]
|
||||
results = middleware.process_spider_output(response, result, None)
|
||||
for result in results:
|
||||
if isinstance(result, Request):
|
||||
self.assertEqual(result.meta, {'param2': 'Stickied!'})
|
||||
def test_comma_separated_string_setting() -> None:
|
||||
mw = _make_mw("param1,param2")
|
||||
assert mw.sticky_meta_keys == ["param1", "param2"]
|
||||
|
||||
def test_comma_separated_string_setting(self):
|
||||
crawler = get_crawler(Spider, {'STICKY_META_KEYS': 'param1,param2'})
|
||||
middleware = self.create_middleware(crawler)
|
||||
assert type(middleware.keys_to_sticky) is list
|
||||
assert middleware.keys_to_sticky == ['param1', 'param2']
|
||||
|
||||
def test_sticky_param_does_not_override_manually_configured_param(self):
|
||||
crawler = get_crawler(
|
||||
Spider,
|
||||
{'STICKY_META_KEYS': ['param', 'param2']}
|
||||
)
|
||||
middleware = self.create_middleware(crawler)
|
||||
request = Request(
|
||||
self.test_url,
|
||||
meta={
|
||||
'param': 'Stickied!',
|
||||
'param2': 'Stickied!'
|
||||
})
|
||||
response = Response(self.test_url, request=request)
|
||||
result = [
|
||||
Request(self.test_url, meta={'param': 'Override stickied'}),
|
||||
MockItem(name='dummy')
|
||||
]
|
||||
results = middleware.process_spider_output(response, result, None)
|
||||
for result in results:
|
||||
if isinstance(result, Request):
|
||||
self.assertEqual(
|
||||
result.meta,
|
||||
{
|
||||
'param': 'Override stickied',
|
||||
'param2': 'Stickied!'
|
||||
}
|
||||
)
|
||||
@coroutine_test
|
||||
async def test_sticky_params() -> None:
|
||||
mw = _make_mw(["param2"])
|
||||
request = Request(
|
||||
TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"}
|
||||
)
|
||||
response = Response(TEST_URL, request=request)
|
||||
spider_output = [Request(TEST_URL), {"name": "dummy"}]
|
||||
for processed in await _run_all_paths(mw, response, spider_output):
|
||||
assert processed[0].meta == {"param2": "Stickied!"}
|
||||
assert processed[1] == {"name": "dummy"}
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_sticky_param_does_not_override_manually_configured_param() -> None:
|
||||
mw = _make_mw(["param", "param2"])
|
||||
request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"})
|
||||
response = Response(TEST_URL, request=request)
|
||||
spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})]
|
||||
for processed in await _run_all_paths(mw, response, spider_output):
|
||||
assert processed[0].meta == {
|
||||
"param": "Override stickied",
|
||||
"param2": "Stickied!",
|
||||
}
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_start_requests_have_no_response() -> None:
|
||||
"""Start seeds are processed with ``response=None`` and are left untouched."""
|
||||
mw = _make_mw(["param"])
|
||||
start_request = Request(TEST_URL, meta={"param": "value"})
|
||||
processed = await collect_asyncgen(
|
||||
mw.process_start(as_async_generator([start_request]))
|
||||
)
|
||||
assert processed[0].meta == {"param": "value"}
|
||||
|
|
|
|||
Loading…
Reference in New Issue