diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py new file mode 100644 index 000000000..ed68200b0 --- /dev/null +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -0,0 +1,45 @@ +""" +StickyMeta middleware transfers specified meta keys to further +requests in spider request chain. + +Sticky meta keys can be configured either by: +* Setting STICKY_META_KEYS eg: + STICKY_META_KEYS = ['key1', 'key2'] +* Spider attribute sticky_meta eg: + spider.sticky_meta = ['key1','key2'] +* Meta attribute "sticky" eg: + Request(meta={'sticky': ['foo'], 'foo': 'bar'}) +resolve priority is meta > spider attribute > setting +""" +from scrapy import Request + + +class StickyMeta: + @classmethod + def from_crawler(cls, crawler): + return cls(crawler.settings) + + def __init__(self, settings): + self.settings_sticky_meta = settings.getlist('STICKY_META') + + def process_spider_output(self, response, result, spider): + # without attached request response will not have meta to pass on + if not response.request: + for value in result: + yield value + return + + spider_sticky_meta = getattr(spider, 'sticky_meta', None) + for request in result: + if not isinstance(request, Request): + yield request + continue + # priority: meta > spider attribute > setting + sticky = response.meta.get( + 'sticky', + spider_sticky_meta if spider_sticky_meta is not None else self.settings_sticky_meta + ) + for k, v in response.meta.items(): + if k in sticky and k not in request.meta: + request.meta[k] = v + yield request diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py new file mode 100644 index 000000000..c9bb86d76 --- /dev/null +++ b/tests/test_spidermiddleware_stickymeta.py @@ -0,0 +1,72 @@ +from unittest import TestCase + +from scrapy.spidermiddlewares.stickymeta import StickyMeta +from scrapy.http import Response, Request +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +class TestStickyMetaMiddleware(TestCase): + + def setUp(self): + crawler = get_crawler(Spider, settings_dict={'STICKY_META': ['settings', 'settings2']}) + self.spider = crawler._create_spider(name='foobar') + self.mw = StickyMeta.from_crawler(crawler) + + def test_process_spider_output(self): + # plain, no match + resp = Response( + 'http://scrapytest.org', + request=Request( + 'http://scrapytest.org/', + meta={'oh_no': "I'm not sticky! :("} + ), + ) + reqs = [Request('http://scrapytest.org')] + for result in self.mw.process_spider_output(resp, reqs, self.spider): + self.assertEqual(result.meta, {}) + + # unattached response + wild_resp = Response('http://httpbin.org/headers') + reqs = [Request('http://scrapytest.org')] + for result in self.mw.process_spider_output(wild_resp, reqs, self.spider): + self.assertEqual(result.meta, {}) + + # setting STICKY_META + reqs = [Request('http://scrapytest.org')] + resp = Response( + 'http://scrapytest.org', + request=Request( + 'http://scrapytest.org/', + meta={'settings': "I'm sticky!"} + ), + ) + + out = list(self.mw.process_spider_output(resp, reqs, self.spider)) + self.assertEqual(out[0].meta, {'settings': "I'm sticky!"}) + + # meta["sticky"] + reqs = [Request('http://scrapytest.org')] + resp = Response( + 'http://scrapytest.org', + request=Request( + 'http://scrapytest.org/', + meta={'meta': "I'm sticky!", 'sticky': ['meta']} + ), + ) + out = list(self.mw.process_spider_output(resp, reqs, self.spider)) + self.assertEqual(out[0].meta, {'meta': "I'm sticky!"}) + + # spider.sticky_meta + reqs = [Request('http://scrapytest.org')] + resp = Response( + 'http://scrapytest.org', + request=Request( + 'http://scrapytest.org/', + meta={'spider': "I'm sticky!"} + ), + ) + self.spider.sticky_meta = ['spider'] + out = list(self.mw.process_spider_output(resp, reqs, self.spider)) + self.assertEqual(out[0].meta, {'spider': "I'm sticky!"}) +