diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 73c34e75f..de8f988c0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -133,6 +133,44 @@ Spider listed here. The subsequent URLs will be generated successively from data contained in the start URLs. + .. attribute:: crawler + + This attribute is set by the :meth:`from_crawler` class method after + initializating the class, and links to the + :class:`~scrapy.crawler.Crawler` object to which this spider instance is + bound. + + Crawlers encapsulate a lot of components in the project for their single + entry access (such as extensions, middlewares, signals managers, etc). + See :ref:`topics-api-crawler` to know more about them. + + .. attribute:: settings + + Configuration on which this spider is been ran. This is a + :class:`~scrapy.settings.Settings` instance, see the + :ref:`topics-settings` topic for a detailed introduction on this subject. + + .. method:: from_crawler(crawler, \*args, \**kwargs) + + This is the class method used by Scrapy to create your spiders. + + You probably won't need to override this directly, since the default + implementation acts as a proxy to the :meth:`__init__` method, calling + it with the given arguments `args` and named arguments `kwargs`. + + Nonetheless, this method sets the :attr:`crawler` and :attr:`settings` + attributes in the new instance, so they can be accessed later inside the + spider's code. + + :param crawler: crawler to which the spider will be bound + :type crawler: :class:`~scrapy.crawler.Crawler` instance + + :param args: arguments passed to the :meth:`__init__` method + :type args: list + + :param kwargs: keyword arguments passed to the :meth:`__init__` method + :type kwargs: dict + .. method:: start_requests() This method must return an iterable with the first Requests to crawl for diff --git a/scrapy/spider.py b/scrapy/spider.py index 8ecfae2a0..89f78d6ba 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -3,11 +3,14 @@ Base class for Scrapy spiders See documentation in docs/topics/spiders.rst """ +import warnings + from scrapy import log from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider from scrapy.utils.deprecate import create_deprecated_class +from scrapy.exceptions import ScrapyDeprecationWarning class Spider(object_ref): @@ -32,18 +35,24 @@ class Spider(object_ref): """ log.msg(message, spider=self, level=level, **kw) + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(*args, **kwargs) + spider._set_crawler(crawler) + return spider + def set_crawler(self, crawler): - assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler - self._crawler = crawler + warnings.warn("set_crawler is deprecated, instantiate and bound the " + "spider to this crawler with from_crawler method " + "instead.", + category=ScrapyDeprecationWarning, stacklevel=2) + assert not hasattr(self, 'crawler'), "Spider already bounded to a " \ + "crawler" + self._set_crawler(crawler) - @property - def crawler(self): - assert hasattr(self, '_crawler'), "Spider not bounded to any crawler" - return self._crawler - - @property - def settings(self): - return self.crawler.settings + def _set_crawler(self, crawler): + self.crawler = crawler + self.settings = crawler.settings def start_requests(self): for url in self.start_urls: diff --git a/tests/test_spider.py b/tests/test_spider.py index 903eff7b1..53daf39fb 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,10 +1,12 @@ import gzip import inspect import warnings -from scrapy.utils.trackref import object_ref from io import BytesIO - from twisted.trial import unittest +try: + from unittest import mock +except ImportError: + import mock from scrapy.spider import Spider, BaseSpider from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse @@ -13,6 +15,8 @@ from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.contrib.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.trackref import object_ref +from scrapy.utils.test import get_crawler class SpiderTest(unittest.TestCase): @@ -46,6 +50,32 @@ class SpiderTest(unittest.TestCase): self.assertRaises(ValueError, self.spider_class) self.assertRaises(ValueError, self.spider_class, somearg='foo') + def test_deprecated_set_crawler_method(self): + spider = self.spider_class('example.com') + crawler = get_crawler() + with warnings.catch_warnings(record=True) as w: + spider.set_crawler(crawler) + self.assertIn("set_crawler", str(w[0].message)) + self.assertTrue(hasattr(spider, 'crawler')) + self.assertIs(spider.crawler, crawler) + self.assertTrue(hasattr(spider, 'settings')) + self.assertIs(spider.settings, crawler.settings) + + def test_from_crawler_crawler_and_settings_population(self): + crawler = get_crawler() + spider = self.spider_class.from_crawler(crawler, 'example.com') + self.assertTrue(hasattr(spider, 'crawler')) + self.assertIs(spider.crawler, crawler) + self.assertTrue(hasattr(spider, 'settings')) + self.assertIs(spider.settings, crawler.settings) + + def test_from_crawler_init_call(self): + with mock.patch.object(self.spider_class, '__init__', + return_value=None) as mock_init: + self.spider_class.from_crawler(get_crawler(), 'example.com', + foo='bar') + mock_init.assert_called_once_with('example.com', foo='bar') + class InitSpiderTest(SpiderTest):