From 63a7301c2d3d63a91423bc76d5b67eac0595775b Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 8 Nov 2019 10:59:04 -0300 Subject: [PATCH 01/17] Add StickyMetaParamsMiddleware Adds a spider middleware that sticky meta parameters, forwarding desired keys through requests and responses. --- docs/topics/spider-middleware.rst | 83 +++++++++++++++++++++++ docs/topics/spiders.rst | 7 ++ scrapy/spidermiddlewares/stickymeta.py | 27 ++++++++ scrapy/spiders/__init__.py | 1 + tests/test_spidermiddleware_stickymeta.py | 54 +++++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 scrapy/spidermiddlewares/stickymeta.py create mode 100644 tests/test_spidermiddleware_stickymeta.py diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0e8210130..863d70135 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -451,3 +451,86 @@ UrlLengthMiddleware * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + +StickyMetaParamsMiddleware +-------------------------- + +.. module:: scrapy.spidermiddlewares.stickymeta + :synopsis: Spider Middleware that forwards meta parameters through requests + +.. class:: StickyMetaParamsMiddleware + + When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` + parameters between ``Requests`` and ``Responses``. + + The :class:`StickyMetaParamsMiddleware` can be configure though the ``sticky_meta_keys`` + spider attribute. + + * :attr:`~scrapy.spiders.Spider.sticky_meta_keys` - List of keys to be stickied. + + See the difference between :ref:`topics-spiders` with and without the middleware: + + Without the middleware:: + + class DontStickySpider(Spider): + name = 'dont_sticky' + start_urls = ['https://www.example.com'] + + def parse(self, response): + for param in range(5): + yield Request( + 'https://www.example.com/next', + meta={'param': param}, + callback=self.parse_2 + ) + + def parse_2(self, response): + # Get important information from response + info = response.xpath('//info/text()').get('info') + # We need to get the param from meta and forward it + param = response.meta['param'] + yield Request( + 'https://www.example.com/next', + meta={'info': info, 'param': param}, + callback=self.parse_3 + ) + + def parse_3(self, response): + # Yield item + yield { + 'param': response.meta['param'], + 'info': response.meta['info'] + } + + With the middleware:: + + class StickySpider(Spider): + name = 'sticky' + start_urls = ['https://www.example.com'] + sticky_meta_keys = ['param'] # Will always forward the meta param "param" + + def parse(self, response): + for param in range(5): + yield Request( + 'https://www.example.com/next', + meta={'param': param}, + callback=self.parse_2 + ) + + def parse_2(self, response): + # Get important information from response + info = response.xpath('//info/text()').get('info') + # We don't need to get the "param" value from meta and resend it. + yield Request( + 'https://www.example.com/next', + meta={'info': info}, + callback=self.parse_3 + ) + + def parse_3(self, response): + # Yield item + yield { + 'param': response.meta['param'], + 'info': response.meta['info'] + } + diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d65a43afd..181f1608d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -196,6 +196,13 @@ scrapy.Spider Called when the spider closes. This method provides a shortcut to signals.connect() for the :signal:`spider_closed` signal. + .. attribute:: sticky_meta_keys + + Keys to be stickied in ``meta`` dictionary. + + For more information about stickied meta parameters see: + :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` + Let's see an example:: import scrapy diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py new file mode 100644 index 000000000..7b5524d46 --- /dev/null +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -0,0 +1,27 @@ +from scrapy.exceptions import NotConfigured +from scrapy.http import Request + + +class StickyMetaParamsMiddleware(object): + """Forward a configurable list of meta keys through subsequent requests""" + + @classmethod + def from_crawler(cls, crawler): + keys_to_sticky = getattr(crawler.spider, 'sticky_meta_keys', []) + if not keys_to_sticky: + raise NotConfigured + return cls(keys_to_sticky) + + def __init__(self, keys_to_sticky): + self.keys_to_sticky = keys_to_sticky + + def process_spider_output(self, response, result, spider): + sticky_meta = {k: response.meta[k] for k in self.keys_to_sticky if k in response.meta} + for r in result: + if not isinstance(r, Request): + yield r + continue + for k, v in sticky_meta.items(): + if k not in r.meta: + r.meta[k] = v + yield r diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 94095bc27..62362dcea 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -21,6 +21,7 @@ class Spider(object_ref): name = None custom_settings = None + sticky_meta_keys = [] def __init__(self, name=None, **kwargs): if name is not None: diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py new file mode 100644 index 000000000..b5baec1dd --- /dev/null +++ b/tests/test_spidermiddleware_stickymeta.py @@ -0,0 +1,54 @@ +import pytest + +from unittest import TestCase + +from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware +from scrapy.exceptions import NotConfigured +from scrapy.http import Response, Request +from scrapy.item import Item, Field +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +class MockItem(Item): + name = Field() + + +class TestStickyMetaParamsMiddleware(TestCase): + + def setUp(self): + self.test_url = 'http://www.example.com' + + def create_middleware(self, crawler): + return StickyMetaParamsMiddleware.from_crawler(crawler) + + def _get_crawler(self, spider): + crawler = get_crawler(Spider) + crawler.spider = spider + return crawler + + def test_middleware_not_enabled(self): + spider = Spider('dummy') + crawler = self._get_crawler(spider) + with pytest.raises(NotConfigured): + self.create_middleware(crawler) + + def test_sticky_params(self): + spider = Spider('dummy') + spider.sticky_meta_keys = ['param2'] + crawler = self._get_crawler(spider) + middleware = self.create_middleware(crawler) + request = Request( + self.test_url, + meta={ + 'param': 'Will not be stickied', + 'param2': 'Stickied!' + }) + response = Response(self.test_url, request=request) + result = [ + Request(self.test_url), + MockItem(name='dummy') # Add a item just to increase the test coverage + ] + for result in middleware.process_spider_output(response, result, spider): + if isinstance(result, Request): + self.assertEqual(result.meta, {'param2': 'Stickied!'}) From 42fcdbd98c29bff6ba579cc9365cccd453f7ea37 Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 8 Nov 2019 11:44:30 -0300 Subject: [PATCH 02/17] Add StickyMetaParamsMiddleware to SPIDER_MIDDLEWARES_BASE --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9c22999cb..fdf2b6960 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -263,6 +263,7 @@ SPIDER_MIDDLEWARES_BASE = { # Engine side 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, + 'scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware': 600, 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, From 0f532b03ce841e88d9a1ed543f7fe169c1d702b6 Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 8 Nov 2019 15:09:58 -0300 Subject: [PATCH 03/17] Fix linter errors --- scrapy/spidermiddlewares/stickymeta.py | 6 +++++- tests/test_spidermiddleware_stickymeta.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index 7b5524d46..b4493f9ba 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -16,7 +16,11 @@ class StickyMetaParamsMiddleware(object): self.keys_to_sticky = keys_to_sticky def process_spider_output(self, response, result, spider): - sticky_meta = {k: response.meta[k] for k in self.keys_to_sticky if k in response.meta} + sticky_meta = { + k: response.meta[k] + for k in self.keys_to_sticky + if k in response.meta + } for r in result: if not isinstance(r, Request): yield r diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index b5baec1dd..64ee3796b 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -47,8 +47,9 @@ class TestStickyMetaParamsMiddleware(TestCase): response = Response(self.test_url, request=request) result = [ Request(self.test_url), - MockItem(name='dummy') # Add a item just to increase the test coverage + MockItem(name='dummy') ] - for result in middleware.process_spider_output(response, result, spider): + results = middleware.process_spider_output(response, result, spider) + for result in results: if isinstance(result, Request): self.assertEqual(result.meta, {'param2': 'Stickied!'}) From 1dea50ba0ddf2ee24af21b0ef7c1f489d40e8cac Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 6 Dec 2019 12:09:31 -0300 Subject: [PATCH 04/17] Update docs after suggestions --- docs/topics/request-response.rst | 3 +++ docs/topics/spider-middleware.rst | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4cf367d96..2baf1f022 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -172,6 +172,9 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. + Values are only acessible on the next :class:`Response`, to forward a key/value + through subsequent responses see :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`. + .. attribute:: Request.cb_kwargs A dictionary that contains arbitrary metadata for this request. Its contents diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 863d70135..16a1de79d 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -463,11 +463,9 @@ StickyMetaParamsMiddleware When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` parameters between ``Requests`` and ``Responses``. - The :class:`StickyMetaParamsMiddleware` can be configure though the ``sticky_meta_keys`` + The :class:`StickyMetaParamsMiddleware` can be configured through the :attr:`~scrapy.spiders.Spider.sticky_meta_keys` spider attribute. - * :attr:`~scrapy.spiders.Spider.sticky_meta_keys` - List of keys to be stickied. - See the difference between :ref:`topics-spiders` with and without the middleware: Without the middleware:: From 1432d76f03f859c353a7687c042a4c1e902149ce Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 6 Dec 2019 12:10:12 -0300 Subject: [PATCH 05/17] Fix import order in test file --- tests/test_spidermiddleware_stickymeta.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index 64ee3796b..bc9074645 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -1,11 +1,11 @@ -import pytest - from unittest import TestCase -from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware +import pytest + from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request -from scrapy.item import Item, Field +from scrapy.http import Request, Response +from scrapy.item import Field, Item +from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware from scrapy.spiders import Spider from scrapy.utils.test import get_crawler From f89b3dff93aace50e92b06f834396ec860d0f359 Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 6 Dec 2019 13:20:24 -0300 Subject: [PATCH 06/17] Configure StickyMetaParamsMiddleware from settings Configure middleware through settings instead of a spider attribute. --- docs/topics/settings.rst | 10 ++++++++++ docs/topics/spider-middleware.rst | 5 ++--- docs/topics/spiders.rst | 7 ------- scrapy/spidermiddlewares/stickymeta.py | 2 +- scrapy/spiders/__init__.py | 1 - tests/test_spidermiddleware_stickymeta.py | 14 +++----------- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a1d15a760..f6ae11117 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1383,6 +1383,16 @@ Default: ``[]`` (empty list) Send Scrapy stats after spiders finish scraping. See :class:`~scrapy.extensions.statsmailer.StatsMailer` for more info. +.. setting:: STICKY_META_KEYS + +STICKY_META_KEYS +---------------- + +Default: ``[]`` (empty list) + +Configure which key/values should be stickied in :attr:`Request.meta`. See +:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` for more info. + .. setting:: TELNETCONSOLE_ENABLED TELNETCONSOLE_ENABLED diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 16a1de79d..dd313d0c7 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -463,8 +463,7 @@ StickyMetaParamsMiddleware When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` parameters between ``Requests`` and ``Responses``. - The :class:`StickyMetaParamsMiddleware` can be configured through the :attr:`~scrapy.spiders.Spider.sticky_meta_keys` - spider attribute. + The :class:`StickyMetaParamsMiddleware` can be configured through the settings :setting:`STICKY_META_KEYS`. See the difference between :ref:`topics-spiders` with and without the middleware: @@ -505,7 +504,7 @@ StickyMetaParamsMiddleware class StickySpider(Spider): name = 'sticky' start_urls = ['https://www.example.com'] - sticky_meta_keys = ['param'] # Will always forward the meta param "param" + custom_settings = {'STICKY_META_KEYS': ['param']} # Will always forward the meta param "param" def parse(self, response): for param in range(5): diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 181f1608d..d65a43afd 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -196,13 +196,6 @@ scrapy.Spider Called when the spider closes. This method provides a shortcut to signals.connect() for the :signal:`spider_closed` signal. - .. attribute:: sticky_meta_keys - - Keys to be stickied in ``meta`` dictionary. - - For more information about stickied meta parameters see: - :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` - Let's see an example:: import scrapy diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index b4493f9ba..3ebce594b 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -7,7 +7,7 @@ class StickyMetaParamsMiddleware(object): @classmethod def from_crawler(cls, crawler): - keys_to_sticky = getattr(crawler.spider, 'sticky_meta_keys', []) + keys_to_sticky = crawler.settings.get('STICKY_META_KEYS', []) if not keys_to_sticky: raise NotConfigured return cls(keys_to_sticky) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 4c0fef388..9429f6cb2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -20,7 +20,6 @@ class Spider(object_ref): name = None custom_settings = None - sticky_meta_keys = [] def __init__(self, name=None, **kwargs): if name is not None: diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index bc9074645..c1ab21373 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -22,21 +22,13 @@ class TestStickyMetaParamsMiddleware(TestCase): def create_middleware(self, crawler): return StickyMetaParamsMiddleware.from_crawler(crawler) - def _get_crawler(self, spider): - crawler = get_crawler(Spider) - crawler.spider = spider - return crawler - def test_middleware_not_enabled(self): - spider = Spider('dummy') - crawler = self._get_crawler(spider) + crawler = get_crawler(Spider) with pytest.raises(NotConfigured): self.create_middleware(crawler) def test_sticky_params(self): - spider = Spider('dummy') - spider.sticky_meta_keys = ['param2'] - crawler = self._get_crawler(spider) + crawler = get_crawler(Spider, {'STICKY_META_KEYS': ['param2']}) middleware = self.create_middleware(crawler) request = Request( self.test_url, @@ -49,7 +41,7 @@ class TestStickyMetaParamsMiddleware(TestCase): Request(self.test_url), MockItem(name='dummy') ] - results = middleware.process_spider_output(response, result, spider) + results = middleware.process_spider_output(response, result, None) for result in results: if isinstance(result, Request): self.assertEqual(result.meta, {'param2': 'Stickied!'}) From f8de153725c273cb2d5411d45c9cd7b4d4f18c97 Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 6 Dec 2019 14:49:31 -0300 Subject: [PATCH 07/17] Use getlist to obtain STICKY_META_KEYS setting --- scrapy/spidermiddlewares/stickymeta.py | 2 +- tests/test_spidermiddleware_stickymeta.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index 3ebce594b..121da9f84 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -7,7 +7,7 @@ class StickyMetaParamsMiddleware(object): @classmethod def from_crawler(cls, crawler): - keys_to_sticky = crawler.settings.get('STICKY_META_KEYS', []) + keys_to_sticky = crawler.settings.getlist('STICKY_META_KEYS') if not keys_to_sticky: raise NotConfigured return cls(keys_to_sticky) diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index c1ab21373..b3955af39 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -45,3 +45,9 @@ class TestStickyMetaParamsMiddleware(TestCase): for result in results: if isinstance(result, Request): self.assertEqual(result.meta, {'param2': 'Stickied!'}) + + def test_comma_separated_string_setting(self): + crawler = get_crawler(Spider, {'STICKY_META_KEYS': 'param1,param2'}) + middleware = self.create_middleware(crawler) + assert type(middleware.keys_to_sticky) is list + assert middleware.keys_to_sticky == ['param1', 'param2'] From 195f43b0543757c3255ff4b2aae7095863fcc8b3 Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 6 Dec 2019 18:58:43 -0300 Subject: [PATCH 08/17] Test that a user configured param is not overriden by a stickied value --- tests/test_spidermiddleware_stickymeta.py | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index b3955af39..a6a774d7c 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -51,3 +51,31 @@ class TestStickyMetaParamsMiddleware(TestCase): middleware = self.create_middleware(crawler) assert type(middleware.keys_to_sticky) is list assert middleware.keys_to_sticky == ['param1', 'param2'] + + def test_sticky_param_does_not_override_manually_configured_param(self): + crawler = get_crawler( + Spider, + {'STICKY_META_KEYS': ['param', 'param2']} + ) + middleware = self.create_middleware(crawler) + request = Request( + self.test_url, + meta={ + 'param': 'Stickied!', + 'param2': 'Stickied!' + }) + response = Response(self.test_url, request=request) + result = [ + Request(self.test_url, meta={'param': 'Override stickied'}), + MockItem(name='dummy') + ] + results = middleware.process_spider_output(response, result, None) + for result in results: + if isinstance(result, Request): + self.assertEqual( + result.meta, + { + 'param': 'Override stickied', + 'param2': 'Stickied!' + } + ) From 509865d10b474f61b0ba3362698f35893ebca711 Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 6 Dec 2019 19:12:41 -0300 Subject: [PATCH 09/17] Improve documentation --- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 2 +- docs/topics/spider-middleware.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 2baf1f022..1676602be 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -172,7 +172,7 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. - Values are only acessible on the next :class:`Response`, to forward a key/value + Values are only acessible on the next :class:`~scrapy.http.Response`, to forward a key/value through subsequent responses see :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`. .. attribute:: Request.cb_kwargs diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f6ae11117..a5843f79b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1390,7 +1390,7 @@ STICKY_META_KEYS Default: ``[]`` (empty list) -Configure which key/values should be stickied in :attr:`Request.meta`. See +Configure which key/values should be stickied in :attr:`Request.meta `. See :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` for more info. .. setting:: TELNETCONSOLE_ENABLED diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index dd313d0c7..3a109f2a5 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -463,7 +463,7 @@ StickyMetaParamsMiddleware When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` parameters between ``Requests`` and ``Responses``. - The :class:`StickyMetaParamsMiddleware` can be configured through the settings :setting:`STICKY_META_KEYS`. + The :class:`StickyMetaParamsMiddleware` can be configured through the :setting:`STICKY_META_KEYS` setting. See the difference between :ref:`topics-spiders` with and without the middleware: From 190ea5f58bebe5c5160f3601b6567146a8ba58c0 Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Date: Fri, 17 Mar 2023 16:57:23 -0300 Subject: [PATCH 10/17] Update scrapy/spidermiddlewares/stickymeta.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/spidermiddlewares/stickymeta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index 121da9f84..8c18e9c98 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -2,7 +2,7 @@ from scrapy.exceptions import NotConfigured from scrapy.http import Request -class StickyMetaParamsMiddleware(object): +class StickyMetaParamsMiddleware: """Forward a configurable list of meta keys through subsequent requests""" @classmethod From bffcb7e0907dfa3145c464b23c0ecaba711c3a9d Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:03:49 -0300 Subject: [PATCH 11/17] Update scrapy/settings/default_settings.py --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a40139718..0792f6efd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -275,10 +275,10 @@ SPIDER_MIDDLEWARES_BASE = { # Engine side 'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50, 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500, - 'scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware': 600, 'scrapy.spidermiddlewares.referer.RefererMiddleware': 700, 'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800, 'scrapy.spidermiddlewares.depth.DepthMiddleware': 900, + 'scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware': 1000, # Spider side } From b1de67520792563849a379edf1704deb5a64cbb8 Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:05:19 -0300 Subject: [PATCH 12/17] Update docs/topics/spider-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 444b4fba3..871186235 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -446,7 +446,7 @@ StickyMetaParamsMiddleware -------------------------- .. module:: scrapy.spidermiddlewares.stickymeta - :synopsis: Spider Middleware that forwards meta parameters through requests + :synopsis: Spider Middleware that forwards metadata through requests .. class:: StickyMetaParamsMiddleware From 786dc0fa6e2967e2cc2a25245de90b6d370c6b43 Mon Sep 17 00:00:00 2001 From: Felipe Boff Nunes <51033921+felipeboffnunes@users.noreply.github.com> Date: Fri, 17 Mar 2023 17:06:33 -0300 Subject: [PATCH 13/17] Update docs/topics/request-response.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 25b2deffe..215ebac79 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -190,8 +190,8 @@ Request objects cloned using the ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.meta`` attribute. - Values are only acessible on the next :class:`~scrapy.http.Response`, to forward a key/value - through subsequent responses see :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`. + To keep some data across multiple spider callbacks, consider using the + :setting`STICKY_META_KEYS` setting. .. attribute:: Request.cb_kwargs From ed1acc295a1978dfdbd33694b8a28f2ee52481ad Mon Sep 17 00:00:00 2001 From: Adnan Awan Date: Wed, 1 Jul 2026 18:11:03 +0500 Subject: [PATCH 14/17] Modernize StickyMetaParamsMiddleware and address PR #4141 review - Subclass BaseSpiderMiddleware and implement get_processed_request(), gaining async process_spider_output support and start-seed handling. - Add type hints and `from __future__ import annotations`. - Rename keys_to_sticky -> sticky_meta_keys (public API consistency). - Define STICKY_META_KEYS default and add it to __all__. - Rework docs: autoclass entry (alphabetical, before UrlLengthMiddleware), move the before/after example into the STICKY_META_KEYS setting docs. - Rewrite tests to cover sync + async output paths and start seeds. Co-authored-by: Luiz Francisco Rodrigues da Silva Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/topics/settings.rst | 47 +++++++- docs/topics/spider-middleware.rst | 97 +++------------- scrapy/spidermiddlewares/stickymeta.py | 65 +++++++---- tests/test_spidermiddleware_stickymeta.py | 130 +++++++++++----------- 4 files changed, 164 insertions(+), 175 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f4454cd4c..7073cbf98 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2085,8 +2085,51 @@ STICKY_META_KEYS Default: ``[]`` (empty list) -Configure which key/values should be stickied in :attr:`Request.meta `. See -:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` for more info. +The :attr:`Request.meta ` keys to copy automatically +from a response into the follow-up requests yielded by its callback, handled by +:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`. + +Metadata keys already set on a follow-up request are not overwritten. + +For example, the following spider:: + + class MySpider(Spider): + name = "myspider" + + async def start(self): + start_url = "https://toscrape.com/" + yield Request(start_url, meta={"start_url": start_url}) + + def parse(self, response): + for a in response.css("a"): + yield response.follow( + a, + meta={"start_url": response.meta["start_url"]}, + ) + yield { + "url": response.url, + "start_url": response.meta["start_url"], + } + +can be rewritten as follows using the :setting:`STICKY_META_KEYS` setting:: + + class MySpider(Spider): + name = "myspider" + custom_settings = { + "STICKY_META_KEYS": ["start_url"], + } + + async def start(self): + start_url = "https://toscrape.com/" + yield Request(start_url, meta={"start_url": start_url}) + + def parse(self, response): + for a in response.css("a"): + yield response.follow(a) + yield { + "url": response.url, + "start_url": response.meta["start_url"], + } .. setting:: TELNETCONSOLE_ENABLED diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8f5038b90..b30ccd335 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -452,6 +452,19 @@ StartSpiderMiddleware .. autoclass:: StartSpiderMiddleware +StickyMetaParamsMiddleware +-------------------------- + +.. module:: scrapy.spidermiddlewares.stickymeta + :synopsis: Sticky Meta Params Spider Middleware + +.. autoclass:: StickyMetaParamsMiddleware + + Allows copying some request metadata into follow-up requests automatically. + + See the :setting:`STICKY_META_KEYS` setting. + + UrlLengthMiddleware ------------------- @@ -466,87 +479,3 @@ UrlLengthMiddleware settings (see the settings documentation for more info): * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. - - -StickyMetaParamsMiddleware --------------------------- - -.. module:: scrapy.spidermiddlewares.stickymeta - :synopsis: Spider Middleware that forwards metadata through requests - -.. class:: StickyMetaParamsMiddleware - - When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` - parameters between ``Requests`` and ``Responses``. - - The :class:`StickyMetaParamsMiddleware` can be configured through the following - settings (see the settings documentation for more info): - - * :setting:`STICKY_META_KEYS` - The :ref:`Request.meta ` keys that you want - to automatically forward to next requests. - - See the difference between :ref:`topics-spiders` with and without the middleware: - - Without the middleware:: - - class DontStickySpider(Spider): - name = 'dont_sticky' - start_urls = ['https://www.example.com'] - - def parse(self, response): - for param in range(5): - yield Request( - 'https://www.example.com/next', - meta={'param': param}, - callback=self.parse_2 - ) - - def parse_2(self, response): - # Get important information from response - info = response.xpath('//info/text()').get('info') - # We need to get the param from meta and forward it - param = response.meta['param'] - yield Request( - 'https://www.example.com/next', - meta={'info': info, 'param': param}, - callback=self.parse_3 - ) - - def parse_3(self, response): - # Yield item - yield { - 'param': response.meta['param'], - 'info': response.meta['info'] - } - - With the middleware:: - - class StickySpider(Spider): - name = 'sticky' - start_urls = ['https://www.example.com'] - custom_settings = {'STICKY_META_KEYS': ['param']} # Will always forward the meta param "param" - - def parse(self, response): - for param in range(5): - yield Request( - 'https://www.example.com/next', - meta={'param': param}, - callback=self.parse_2 - ) - - def parse_2(self, response): - # Get important information from response - info = response.xpath('//info/text()').get('info') - # We don't need to get the "param" value from meta and resend it. - yield Request( - 'https://www.example.com/next', - meta={'info': info}, - callback=self.parse_3 - ) - - def parse_3(self, response): - # Yield item - yield { - 'param': response.meta['param'], - 'info': response.meta['info'] - } diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index 8c18e9c98..f18d8e3af 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -1,31 +1,48 @@ +""" +Sticky Meta Params Spider Middleware + +See documentation in docs/topics/spider-middleware.rst +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + from scrapy.exceptions import NotConfigured -from scrapy.http import Request +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Request, Response -class StickyMetaParamsMiddleware: - """Forward a configurable list of meta keys through subsequent requests""" +class StickyMetaParamsMiddleware(BaseSpiderMiddleware): + """Copy a configurable list of :attr:`Request.meta ` + keys from a response into the follow-up requests of its callback. + + The keys to copy are read from the :setting:`STICKY_META_KEYS` setting. + Keys already present in a follow-up request are not overwritten. + """ + + def __init__(self, sticky_meta_keys: list[str]): # pylint: disable=super-init-not-called + self.sticky_meta_keys: list[str] = sticky_meta_keys @classmethod - def from_crawler(cls, crawler): - keys_to_sticky = crawler.settings.getlist('STICKY_META_KEYS') - if not keys_to_sticky: + def from_crawler(cls, crawler: Crawler) -> Self: + sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS") + if not sticky_meta_keys: raise NotConfigured - return cls(keys_to_sticky) + return cls(sticky_meta_keys) - def __init__(self, keys_to_sticky): - self.keys_to_sticky = keys_to_sticky - - def process_spider_output(self, response, result, spider): - sticky_meta = { - k: response.meta[k] - for k in self.keys_to_sticky - if k in response.meta - } - for r in result: - if not isinstance(r, Request): - yield r - continue - for k, v in sticky_meta.items(): - if k not in r.meta: - r.meta[k] = v - yield r + def get_processed_request( + self, request: Request, response: Response | None + ) -> Request | None: + if response is None: + return request + for key in self.sticky_meta_keys: + if key in response.meta and key not in request.meta: + request.meta[key] = response.meta[key] + return request diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index a6a774d7c..c8dd972a2 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -1,81 +1,81 @@ -from unittest import TestCase +from __future__ import annotations + +from typing import Any import pytest +from scrapy import Request, Spider from scrapy.exceptions import NotConfigured -from scrapy.http import Request, Response -from scrapy.item import Field, Item +from scrapy.http import Response from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware -from scrapy.spiders import Spider +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +TEST_URL = "http://www.example.com" -class MockItem(Item): - name = Field() +def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware: + crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys}) + return StickyMetaParamsMiddleware.from_crawler(crawler) -class TestStickyMetaParamsMiddleware(TestCase): +async def _run_all_paths( + mw: StickyMetaParamsMiddleware, response: Response, spider_output: list[Any] +) -> list[list[Any]]: + """Run the spider output through every processing path of the middleware.""" + return [ + list(mw.process_spider_output(response, spider_output)), + await collect_asyncgen( + mw.process_spider_output_async(response, as_async_generator(spider_output)) + ), + ] - def setUp(self): - self.test_url = 'http://www.example.com' - def create_middleware(self, crawler): - return StickyMetaParamsMiddleware.from_crawler(crawler) +def test_not_configured() -> None: + crawler = get_crawler(Spider) + with pytest.raises(NotConfigured): + build_from_crawler(StickyMetaParamsMiddleware, crawler) - def test_middleware_not_enabled(self): - crawler = get_crawler(Spider) - with pytest.raises(NotConfigured): - self.create_middleware(crawler) - def test_sticky_params(self): - crawler = get_crawler(Spider, {'STICKY_META_KEYS': ['param2']}) - middleware = self.create_middleware(crawler) - request = Request( - self.test_url, - meta={ - 'param': 'Will not be stickied', - 'param2': 'Stickied!' - }) - response = Response(self.test_url, request=request) - result = [ - Request(self.test_url), - MockItem(name='dummy') - ] - results = middleware.process_spider_output(response, result, None) - for result in results: - if isinstance(result, Request): - self.assertEqual(result.meta, {'param2': 'Stickied!'}) +def test_comma_separated_string_setting() -> None: + mw = _make_mw("param1,param2") + assert mw.sticky_meta_keys == ["param1", "param2"] - def test_comma_separated_string_setting(self): - crawler = get_crawler(Spider, {'STICKY_META_KEYS': 'param1,param2'}) - middleware = self.create_middleware(crawler) - assert type(middleware.keys_to_sticky) is list - assert middleware.keys_to_sticky == ['param1', 'param2'] - def test_sticky_param_does_not_override_manually_configured_param(self): - crawler = get_crawler( - Spider, - {'STICKY_META_KEYS': ['param', 'param2']} - ) - middleware = self.create_middleware(crawler) - request = Request( - self.test_url, - meta={ - 'param': 'Stickied!', - 'param2': 'Stickied!' - }) - response = Response(self.test_url, request=request) - result = [ - Request(self.test_url, meta={'param': 'Override stickied'}), - MockItem(name='dummy') - ] - results = middleware.process_spider_output(response, result, None) - for result in results: - if isinstance(result, Request): - self.assertEqual( - result.meta, - { - 'param': 'Override stickied', - 'param2': 'Stickied!' - } - ) +@coroutine_test +async def test_sticky_params() -> None: + mw = _make_mw(["param2"]) + request = Request( + TEST_URL, meta={"param": "Will not be stickied", "param2": "Stickied!"} + ) + response = Response(TEST_URL, request=request) + spider_output = [Request(TEST_URL), {"name": "dummy"}] + for processed in await _run_all_paths(mw, response, spider_output): + assert processed[0].meta == {"param2": "Stickied!"} + assert processed[1] == {"name": "dummy"} + + +@coroutine_test +async def test_sticky_param_does_not_override_manually_configured_param() -> None: + mw = _make_mw(["param", "param2"]) + request = Request(TEST_URL, meta={"param": "Stickied!", "param2": "Stickied!"}) + response = Response(TEST_URL, request=request) + spider_output = [Request(TEST_URL, meta={"param": "Override stickied"})] + for processed in await _run_all_paths(mw, response, spider_output): + assert processed[0].meta == { + "param": "Override stickied", + "param2": "Stickied!", + } + + +@coroutine_test +async def test_start_requests_have_no_response() -> None: + """Start seeds are processed with ``response=None`` and are left untouched.""" + mw = _make_mw(["param"]) + start_request = Request(TEST_URL, meta={"param": "value"}) + processed = await collect_asyncgen( + mw.process_start(as_async_generator([start_request])) + ) + assert processed[0].meta == {"param": "value"} From a2556ce75ca32435307b23f502dd0933848a0e22 Mon Sep 17 00:00:00 2001 From: Adnan Awan Date: Wed, 1 Jul 2026 19:30:48 +0500 Subject: [PATCH 16/17] Document StickyMetaParamsMiddleware in SPIDER_MIDDLEWARES_BASE default Addresses review feedback on PR #7711. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/topics/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7073cbf98..1c21095b1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2033,6 +2033,7 @@ Default: "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, + "scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware": 1000, } A dict containing the spider middlewares enabled by default in Scrapy, and From e80688523274ccedda384018264009f0427b0fd6 Mon Sep 17 00:00:00 2001 From: Adnan Awan Date: Tue, 14 Jul 2026 20:18:18 +0500 Subject: [PATCH 17/17] Address PR review comments on StickyMetaParamsMiddleware Drop the redundant autoclass description and unused module directive in the docs, remove the stale module-level docstring, fix __init__ to properly call super().__init__(crawler) instead of skipping it, simplify the meta copy with setdefault, and use build_from_crawler in the test helper for consistency. --- docs/topics/spider-middleware.rst | 9 +-------- scrapy/spidermiddlewares/stickymeta.py | 15 +++++---------- tests/test_spidermiddleware_stickymeta.py | 2 +- 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index b30ccd335..c7d0c1645 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -455,14 +455,7 @@ StartSpiderMiddleware StickyMetaParamsMiddleware -------------------------- -.. module:: scrapy.spidermiddlewares.stickymeta - :synopsis: Sticky Meta Params Spider Middleware - -.. autoclass:: StickyMetaParamsMiddleware - - Allows copying some request metadata into follow-up requests automatically. - - See the :setting:`STICKY_META_KEYS` setting. +.. autoclass:: scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware UrlLengthMiddleware diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index f18d8e3af..72d7c5752 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -1,9 +1,3 @@ -""" -Sticky Meta Params Spider Middleware - -See documentation in docs/topics/spider-middleware.rst -""" - from __future__ import annotations from typing import TYPE_CHECKING @@ -27,7 +21,8 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware): Keys already present in a follow-up request are not overwritten. """ - def __init__(self, sticky_meta_keys: list[str]): # pylint: disable=super-init-not-called + def __init__(self, crawler: Crawler, sticky_meta_keys: list[str]): + super().__init__(crawler) self.sticky_meta_keys: list[str] = sticky_meta_keys @classmethod @@ -35,7 +30,7 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware): sticky_meta_keys = crawler.settings.getlist("STICKY_META_KEYS") if not sticky_meta_keys: raise NotConfigured - return cls(sticky_meta_keys) + return cls(crawler, sticky_meta_keys) def get_processed_request( self, request: Request, response: Response | None @@ -43,6 +38,6 @@ class StickyMetaParamsMiddleware(BaseSpiderMiddleware): if response is None: return request for key in self.sticky_meta_keys: - if key in response.meta and key not in request.meta: - request.meta[key] = response.meta[key] + if key in response.meta: + request.meta.setdefault(key, response.meta[key]) return request diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index c8dd972a2..d66c1faf8 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -18,7 +18,7 @@ TEST_URL = "http://www.example.com" def _make_mw(sticky_meta_keys: Any) -> StickyMetaParamsMiddleware: crawler = get_crawler(Spider, {"STICKY_META_KEYS": sticky_meta_keys}) - return StickyMetaParamsMiddleware.from_crawler(crawler) + return build_from_crawler(StickyMetaParamsMiddleware, crawler) async def _run_all_paths(