From 63a7301c2d3d63a91423bc76d5b67eac0595775b Mon Sep 17 00:00:00 2001 From: Luiz Silva Date: Fri, 8 Nov 2019 10:59:04 -0300 Subject: [PATCH] Add StickyMetaParamsMiddleware Adds a spider middleware that sticky meta parameters, forwarding desired keys through requests and responses. --- docs/topics/spider-middleware.rst | 83 +++++++++++++++++++++++ docs/topics/spiders.rst | 7 ++ scrapy/spidermiddlewares/stickymeta.py | 27 ++++++++ scrapy/spiders/__init__.py | 1 + tests/test_spidermiddleware_stickymeta.py | 54 +++++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 scrapy/spidermiddlewares/stickymeta.py create mode 100644 tests/test_spidermiddleware_stickymeta.py diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0e8210130..863d70135 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -451,3 +451,86 @@ UrlLengthMiddleware * :setting:`URLLENGTH_LIMIT` - The maximum URL length to allow for crawled URLs. + +StickyMetaParamsMiddleware +-------------------------- + +.. module:: scrapy.spidermiddlewares.stickymeta + :synopsis: Spider Middleware that forwards meta parameters through requests + +.. class:: StickyMetaParamsMiddleware + + When enabled and configured, this middleware forwards the desired :ref:`Request.meta ` + parameters between ``Requests`` and ``Responses``. + + The :class:`StickyMetaParamsMiddleware` can be configure though the ``sticky_meta_keys`` + spider attribute. + + * :attr:`~scrapy.spiders.Spider.sticky_meta_keys` - List of keys to be stickied. + + See the difference between :ref:`topics-spiders` with and without the middleware: + + Without the middleware:: + + class DontStickySpider(Spider): + name = 'dont_sticky' + start_urls = ['https://www.example.com'] + + def parse(self, response): + for param in range(5): + yield Request( + 'https://www.example.com/next', + meta={'param': param}, + callback=self.parse_2 + ) + + def parse_2(self, response): + # Get important information from response + info = response.xpath('//info/text()').get('info') + # We need to get the param from meta and forward it + param = response.meta['param'] + yield Request( + 'https://www.example.com/next', + meta={'info': info, 'param': param}, + callback=self.parse_3 + ) + + def parse_3(self, response): + # Yield item + yield { + 'param': response.meta['param'], + 'info': response.meta['info'] + } + + With the middleware:: + + class StickySpider(Spider): + name = 'sticky' + start_urls = ['https://www.example.com'] + sticky_meta_keys = ['param'] # Will always forward the meta param "param" + + def parse(self, response): + for param in range(5): + yield Request( + 'https://www.example.com/next', + meta={'param': param}, + callback=self.parse_2 + ) + + def parse_2(self, response): + # Get important information from response + info = response.xpath('//info/text()').get('info') + # We don't need to get the "param" value from meta and resend it. + yield Request( + 'https://www.example.com/next', + meta={'info': info}, + callback=self.parse_3 + ) + + def parse_3(self, response): + # Yield item + yield { + 'param': response.meta['param'], + 'info': response.meta['info'] + } + diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d65a43afd..181f1608d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -196,6 +196,13 @@ scrapy.Spider Called when the spider closes. This method provides a shortcut to signals.connect() for the :signal:`spider_closed` signal. + .. attribute:: sticky_meta_keys + + Keys to be stickied in ``meta`` dictionary. + + For more information about stickied meta parameters see: + :class:`~scrapy.spidermiddlewares.stickymeta.StickyMetaParamsMiddleware` + Let's see an example:: import scrapy diff --git a/scrapy/spidermiddlewares/stickymeta.py b/scrapy/spidermiddlewares/stickymeta.py new file mode 100644 index 000000000..7b5524d46 --- /dev/null +++ b/scrapy/spidermiddlewares/stickymeta.py @@ -0,0 +1,27 @@ +from scrapy.exceptions import NotConfigured +from scrapy.http import Request + + +class StickyMetaParamsMiddleware(object): + """Forward a configurable list of meta keys through subsequent requests""" + + @classmethod + def from_crawler(cls, crawler): + keys_to_sticky = getattr(crawler.spider, 'sticky_meta_keys', []) + if not keys_to_sticky: + raise NotConfigured + return cls(keys_to_sticky) + + def __init__(self, keys_to_sticky): + self.keys_to_sticky = keys_to_sticky + + def process_spider_output(self, response, result, spider): + sticky_meta = {k: response.meta[k] for k in self.keys_to_sticky if k in response.meta} + for r in result: + if not isinstance(r, Request): + yield r + continue + for k, v in sticky_meta.items(): + if k not in r.meta: + r.meta[k] = v + yield r diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 94095bc27..62362dcea 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -21,6 +21,7 @@ class Spider(object_ref): name = None custom_settings = None + sticky_meta_keys = [] def __init__(self, name=None, **kwargs): if name is not None: diff --git a/tests/test_spidermiddleware_stickymeta.py b/tests/test_spidermiddleware_stickymeta.py new file mode 100644 index 000000000..b5baec1dd --- /dev/null +++ b/tests/test_spidermiddleware_stickymeta.py @@ -0,0 +1,54 @@ +import pytest + +from unittest import TestCase + +from scrapy.spidermiddlewares.stickymeta import StickyMetaParamsMiddleware +from scrapy.exceptions import NotConfigured +from scrapy.http import Response, Request +from scrapy.item import Item, Field +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler + + +class MockItem(Item): + name = Field() + + +class TestStickyMetaParamsMiddleware(TestCase): + + def setUp(self): + self.test_url = 'http://www.example.com' + + def create_middleware(self, crawler): + return StickyMetaParamsMiddleware.from_crawler(crawler) + + def _get_crawler(self, spider): + crawler = get_crawler(Spider) + crawler.spider = spider + return crawler + + def test_middleware_not_enabled(self): + spider = Spider('dummy') + crawler = self._get_crawler(spider) + with pytest.raises(NotConfigured): + self.create_middleware(crawler) + + def test_sticky_params(self): + spider = Spider('dummy') + spider.sticky_meta_keys = ['param2'] + crawler = self._get_crawler(spider) + middleware = self.create_middleware(crawler) + request = Request( + self.test_url, + meta={ + 'param': 'Will not be stickied', + 'param2': 'Stickied!' + }) + response = Response(self.test_url, request=request) + result = [ + Request(self.test_url), + MockItem(name='dummy') # Add a item just to increase the test coverage + ] + for result in middleware.process_spider_output(response, result, spider): + if isinstance(result, Request): + self.assertEqual(result.meta, {'param2': 'Stickied!'})