diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a1d15a760..f6ae11117 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1383,6 +1383,16 @@ Default: ``[]`` (empty list) Send Scrapy stats after spiders finish scraping. See :class:`~scrapy.extensions.statsmailer.StatsMailer` for more info. +.. setting:: STICKY_META_KEYS + +STICKY_META_KEYS +---------------- + +Default: ``[]`` (empty list) + +Configure which key/values should be stickied in :attr:`Request.meta`. See +:class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` for more info. + .. setting:: TELNETCONSOLE_ENABLED TELNETCONSOLE_ENABLED diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 16a1de79d..dd313d0c7 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -463,8 +463,7 @@ StickyMetaParamsMiddleware When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` parameters between ``Requests`` and ``Responses``. - The :class:`StickyMetaParamsMiddleware` can be configured through the :attr:`~scrapy.spiders.Spider.sticky_meta_keys` - spider attribute. + The :class:`StickyMetaParamsMiddleware` can be configured through the settings :setting:`STICKY_META_KEYS`. See the difference between :ref:`topics-spiders` with and without the middleware: @@ -505,7 +504,7 @@ StickyMetaParamsMiddleware class StickySpider(Spider): name = 'sticky' start_urls = ['https://www.example.com'] - sticky_meta_keys = ['param'] # Will always forward the meta param "param" + custom_settings = {'STICKY_META_KEYS': ['param']} # Will always forward the meta param "param" def parse(self, response): for param in range(5): diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 181f1608d..d65a43afd 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -196,13 +196,6 @@ scrapy.Spider Called when the spider closes. This method provides a shortcut to signals.connect() for the :signal:`spider_closed` signal. - .. attribute:: sticky_meta_keys - - Keys to be stickied in ``meta`` dictionary. - - For more information about stickied meta parameters see: - :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` - Let's see an example:: import scrapy diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py index b4493f9ba..3ebce594b 100644 --- a/scrapy/spidermiddlewares/stickymeta.py +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -7,7 +7,7 @@ class StickyMetaParamsMiddleware(object): @classmethod def from_crawler(cls, crawler): - keys_to_sticky = getattr(crawler.spider, 'sticky_meta_keys', []) + keys_to_sticky = crawler.settings.get('STICKY_META_KEYS', []) if not keys_to_sticky: raise NotConfigured return cls(keys_to_sticky) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 4c0fef388..9429f6cb2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -20,7 +20,6 @@ class Spider(object_ref): name = None custom_settings = None - sticky_meta_keys = [] def __init__(self, name=None, **kwargs): if name is not None: diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py index bc9074645..c1ab21373 100644 --- a/tests/test_spidermiddleware_stickymeta.py +++ b/tests/test_spidermiddleware_stickymeta.py @@ -22,21 +22,13 @@ class TestStickyMetaParamsMiddleware(TestCase): def create_middleware(self, crawler): return StickyMetaParamsMiddleware.from_crawler(crawler) - def _get_crawler(self, spider): - crawler = get_crawler(Spider) - crawler.spider = spider - return crawler - def test_middleware_not_enabled(self): - spider = Spider('dummy') - crawler = self._get_crawler(spider) + crawler = get_crawler(Spider) with pytest.raises(NotConfigured): self.create_middleware(crawler) def test_sticky_params(self): - spider = Spider('dummy') - spider.sticky_meta_keys = ['param2'] - crawler = self._get_crawler(spider) + crawler = get_crawler(Spider, {'STICKY_META_KEYS': ['param2']}) middleware = self.create_middleware(crawler) request = Request( self.test_url, @@ -49,7 +41,7 @@ class TestStickyMetaParamsMiddleware(TestCase): Request(self.test_url), MockItem(name='dummy') ] - results = middleware.process_spider_output(response, result, spider) + results = middleware.process_spider_output(response, result, None) for result in results: if isinstance(result, Request): self.assertEqual(result.meta, {'param2': 'Stickied!'})