Add StickyMetaParamsMiddleware

Adds a spider middleware that sticky meta parameters, forwarding
desired keys through requests and responses.

docs/topics/spider-middleware.rst

@@ -451,3 +451,86 @@ UrlLengthMiddleware
 
       * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs.
 
+
+StickyMetaParamsMiddleware
+--------------------------
+
+.. module:: scrapy.spidermiddlewares.stickymeta
+   :synopsis: Spider Middleware that forwards meta parameters through requests
+
+.. class:: StickyMetaParamsMiddleware
+
+   When enabled and configured, this middleware forwards the desired :ref:`Request.meta <topics-request-meta>`
+   parameters between ``Requests`` and ``Responses``.
+
+   The :class:`StickyMetaParamsMiddleware` can be configure though the ``sticky_meta_keys``
+   spider attribute.
+
+      * :attr:`~scrapy.spiders.Spider.sticky_meta_keys` - List of keys to be stickied.
+
+   See the difference between :ref:`topics-spiders` with and without the middleware:
+
+   Without the middleware::
+
+    class DontStickySpider(Spider):
+        name = 'dont_sticky'
+        start_urls = ['https://www.example.com']
+
+        def parse(self, response):
+            for param in range(5):
+                yield Request(
+                    'https://www.example.com/next',
+                    meta={'param': param},
+                    callback=self.parse_2
+                )
+
+        def parse_2(self, response):
+            # Get important information from response
+            info = response.xpath('//info/text()').get('info')
+            # We need to get the param from meta and forward it
+            param = response.meta['param']
+            yield Request(
+                'https://www.example.com/next',
+                meta={'info': info, 'param': param},
+                callback=self.parse_3
+            )
+
+        def parse_3(self, response):
+            # Yield item
+            yield {
+                'param': response.meta['param'],
+                'info': response.meta['info']
+            }
+
+   With the middleware::
+
+    class StickySpider(Spider):
+        name = 'sticky'
+        start_urls = ['https://www.example.com']
+        sticky_meta_keys = ['param']  # Will always forward the meta param "param"
+
+        def parse(self, response):
+            for param in range(5):
+                yield Request(
+                    'https://www.example.com/next',
+                    meta={'param': param},
+                    callback=self.parse_2
+                )
+
+        def parse_2(self, response):
+            # Get important information from response
+            info = response.xpath('//info/text()').get('info')
+            # We don't need to get the "param" value from meta and resend it.
+            yield Request(
+                'https://www.example.com/next',
+                meta={'info': info},
+                callback=self.parse_3
+            )
+
+        def parse_3(self, response):
+            # Yield item
+            yield {
+                'param': response.meta['param'],
+                'info': response.meta['info']
+            }
+

docs/topics/spiders.rst

@@ -196,6 +196,13 @@ scrapy.Spider
        Called when the spider closes. This method provides a shortcut to
        signals.connect() for the :signal:`spider_closed` signal.
 
+   .. attribute:: sticky_meta_keys
+
+      Keys to be stickied in ``meta`` dictionary.
+
+      For more information about stickied meta parameters see:
+      :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware`
+
 Let's see an example::
 
     import scrapy

scrapy/spidermiddlewares/stickymeta.py

@@ -0,0 +1,27 @@
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Request
+
+
+class StickyMetaParamsMiddleware(object):
+    """Forward a configurable list of meta keys through subsequent requests"""
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        keys_to_sticky = getattr(crawler.spider, 'sticky_meta_keys', [])
+        if not keys_to_sticky:
+            raise NotConfigured
+        return cls(keys_to_sticky)
+
+    def __init__(self, keys_to_sticky):
+        self.keys_to_sticky = keys_to_sticky
+
+    def process_spider_output(self, response, result, spider):
+        sticky_meta = {k: response.meta[k] for k in self.keys_to_sticky if k in response.meta}
+        for r in result:
+            if not isinstance(r, Request):
+                yield r
+                continue
+            for k, v in sticky_meta.items():
+                if k not in r.meta:
+                    r.meta[k] = v
+            yield r

scrapy/spiders/__init__.py

@@ -21,6 +21,7 @@ class Spider(object_ref):
 
     name = None
     custom_settings = None
+    sticky_meta_keys = []
 
     def __init__(self, name=None, **kwargs):
         if name is not None:

tests/test_spidermiddleware_stickymeta.py

@@ -0,0 +1,54 @@
+import pytest
+
+from unittest import TestCase
+
+from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware
+from scrapy.exceptions import NotConfigured
+from scrapy.http import Response, Request
+from scrapy.item import Item, Field
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+
+
+class MockItem(Item):
+    name = Field()
+
+
+class TestStickyMetaParamsMiddleware(TestCase):
+
+    def setUp(self):
+        self.test_url = 'http://www.example.com'
+
+    def create_middleware(self, crawler):
+        return StickyMetaParamsMiddleware.from_crawler(crawler)
+
+    def _get_crawler(self, spider):
+        crawler = get_crawler(Spider)
+        crawler.spider = spider
+        return crawler
+
+    def test_middleware_not_enabled(self):
+        spider = Spider('dummy')
+        crawler = self._get_crawler(spider)
+        with pytest.raises(NotConfigured):
+            self.create_middleware(crawler)
+
+    def test_sticky_params(self):
+        spider = Spider('dummy')
+        spider.sticky_meta_keys = ['param2']
+        crawler = self._get_crawler(spider)
+        middleware = self.create_middleware(crawler)
+        request = Request(
+            self.test_url,
+            meta={
+                'param': 'Will not be stickied',
+                'param2': 'Stickied!'
+            })
+        response = Response(self.test_url, request=request)
+        result = [
+            Request(self.test_url),
+            MockItem(name='dummy')  # Add a item just to increase the test coverage
+        ]
+        for result in middleware.process_spider_output(response, result, spider):
+            if isinstance(result, Request):
+                self.assertEqual(result.meta, {'param2': 'Stickied!'})