Merge ae6cfec7c6 into c9446931a8

scrapy/spidermiddlewares/stickymeta.py

@@ -0,0 +1,45 @@
+"""
+StickyMeta middleware transfers specified meta keys to further
+requests in spider request chain.
+
+Sticky meta keys can be configured either by:
+* Setting STICKY_META_KEYS eg:
+    STICKY_META_KEYS = ['key1', 'key2']
+* Spider attribute sticky_meta eg:
+    spider.sticky_meta = ['key1','key2']
+* Meta attribute "sticky" eg:
+    Request(meta={'sticky': ['foo'], 'foo': 'bar'})
+resolve priority is meta > spider attribute > setting
+"""
+from scrapy import Request
+
+
+class StickyMeta:
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def __init__(self, settings):
+        self.settings_sticky_meta = settings.getlist('STICKY_META')
+
+    def process_spider_output(self, response, result, spider):
+        # without attached request response will not have meta to pass on
+        if not response.request:
+            for value in result:
+                yield value
+            return
+
+        spider_sticky_meta = getattr(spider, 'sticky_meta', None)
+        for request in result:
+            if not isinstance(request, Request):
+                yield request
+                continue
+            # priority: meta > spider attribute > setting
+            sticky = response.meta.get(
+                'sticky',
+                spider_sticky_meta if spider_sticky_meta is not None else self.settings_sticky_meta
+            )
+            for k, v in response.meta.items():
+                if k in sticky and k not in request.meta:
+                    request.meta[k] = v
+            yield request

tests/test_spidermiddleware_stickymeta.py

@@ -0,0 +1,72 @@
+from unittest import TestCase
+
+from scrapy.spidermiddlewares.stickymeta import StickyMeta
+from scrapy.http import Response, Request
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+
+
+class TestStickyMetaMiddleware(TestCase):
+
+    def setUp(self):
+        crawler = get_crawler(Spider, settings_dict={'STICKY_META': ['settings', 'settings2']})
+        self.spider = crawler._create_spider(name='foobar')
+        self.mw = StickyMeta.from_crawler(crawler)
+
+    def test_process_spider_output(self):
+        # plain, no match
+        resp = Response(
+            'http://scrapytest.org',
+            request=Request(
+                'http://scrapytest.org/',
+                meta={'oh_no': "I'm not sticky! :("}
+            ),
+        )
+        reqs = [Request('http://scrapytest.org')]
+        for result in self.mw.process_spider_output(resp, reqs, self.spider):
+            self.assertEqual(result.meta, {})
+
+        # unattached response
+        wild_resp = Response('http://httpbin.org/headers')
+        reqs = [Request('http://scrapytest.org')]
+        for result in self.mw.process_spider_output(wild_resp, reqs, self.spider):
+            self.assertEqual(result.meta, {})
+
+        # setting STICKY_META
+        reqs = [Request('http://scrapytest.org')]
+        resp = Response(
+            'http://scrapytest.org',
+            request=Request(
+                'http://scrapytest.org/',
+                meta={'settings': "I'm sticky!"}
+            ),
+        )
+
+        out = list(self.mw.process_spider_output(resp, reqs, self.spider))
+        self.assertEqual(out[0].meta, {'settings': "I'm sticky!"})
+
+        # meta["sticky"]
+        reqs = [Request('http://scrapytest.org')]
+        resp = Response(
+            'http://scrapytest.org',
+            request=Request(
+                'http://scrapytest.org/',
+                meta={'meta': "I'm sticky!", 'sticky': ['meta']}
+            ),
+        )
+        out = list(self.mw.process_spider_output(resp, reqs, self.spider))
+        self.assertEqual(out[0].meta, {'meta': "I'm sticky!"})
+
+        # spider.sticky_meta
+        reqs = [Request('http://scrapytest.org')]
+        resp = Response(
+            'http://scrapytest.org',
+            request=Request(
+                'http://scrapytest.org/',
+                meta={'spider': "I'm sticky!"}
+            ),
+        )
+        self.spider.sticky_meta = ['spider']
+        out = list(self.mw.process_spider_output(resp, reqs, self.spider))
+        self.assertEqual(out[0].meta, {'spider': "I'm sticky!"})
+