Add from_crawler class method to base Spider

docs/topics/spiders.rst

@@ -133,6 +133,44 @@ Spider
        listed here. The subsequent URLs will be generated successively from data
        contained in the start URLs.
 
+   .. attribute:: crawler
+
+      This attribute is set by the :meth:`from_crawler` class method after
+      initializating the class, and links to the
+      :class:`~scrapy.crawler.Crawler` object to which this spider instance is
+      bound.
+
+      Crawlers encapsulate a lot of components in the project for their single
+      entry access (such as extensions, middlewares, signals managers, etc).
+      See :ref:`topics-api-crawler` to know more about them.
+
+   .. attribute:: settings
+
+      Configuration on which this spider is been ran. This is a
+      :class:`~scrapy.settings.Settings` instance, see the
+      :ref:`topics-settings` topic for a detailed introduction on this subject.
+
+   .. method:: from_crawler(crawler, \*args, \**kwargs)
+
+       This is the class method used by Scrapy to create your spiders.
+
+       You probably won't need to override this directly, since the default
+       implementation acts as a proxy to the :meth:`__init__` method, calling
+       it with the given arguments `args` and named arguments `kwargs`.
+
+       Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
+       attributes in the new instance, so they can be accessed later inside the
+       spider's code.
+
+       :param crawler: crawler to which the spider will be bound
+       :type crawler: :class:`~scrapy.crawler.Crawler` instance
+
+       :param args: arguments passed to the :meth:`__init__` method
+       :type args: list
+
+       :param kwargs: keyword arguments passed to the :meth:`__init__` method
+       :type kwargs: dict
+
    .. method:: start_requests()
 
        This method must return an iterable with the first Requests to crawl for

scrapy/spider.py

@@ -3,11 +3,14 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+import warnings
+
 from scrapy import log
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 from scrapy.utils.deprecate import create_deprecated_class
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 class Spider(object_ref):
@@ -32,18 +35,24 @@ class Spider(object_ref):
         """
         log.msg(message, spider=self, level=level, **kw)
 
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = cls(*args, **kwargs)
+        spider._set_crawler(crawler)
+        return spider
+
     def set_crawler(self, crawler):
-        assert not hasattr(self, '_crawler'), "Spider already bounded to %s" % crawler
-        self._crawler = crawler
+        warnings.warn("set_crawler is deprecated, instantiate and bound the "
+                      "spider to this crawler with from_crawler method "
+                      "instead.",
+                      category=ScrapyDeprecationWarning, stacklevel=2)
+        assert not hasattr(self, 'crawler'), "Spider already bounded to a " \
+                                             "crawler"
+        self._set_crawler(crawler)
 
-    @property
-    def crawler(self):
-        assert hasattr(self, '_crawler'), "Spider not bounded to any crawler"
-        return self._crawler
-
-    @property
-    def settings(self):
-        return self.crawler.settings
+    def _set_crawler(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
 
     def start_requests(self):
         for url in self.start_urls:

tests/test_spider.py

@@ -1,10 +1,12 @@
 import gzip
 import inspect
 import warnings
-from scrapy.utils.trackref import object_ref
 from io import BytesIO
-
 from twisted.trial import unittest
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 
 from scrapy.spider import Spider, BaseSpider
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
@@ -13,6 +15,8 @@ from scrapy.contrib.spiders import CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.contrib.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.trackref import object_ref
+from scrapy.utils.test import get_crawler
 
 
 class SpiderTest(unittest.TestCase):
@@ -46,6 +50,32 @@ class SpiderTest(unittest.TestCase):
         self.assertRaises(ValueError, self.spider_class)
         self.assertRaises(ValueError, self.spider_class, somearg='foo')
 
+    def test_deprecated_set_crawler_method(self):
+        spider = self.spider_class('example.com')
+        crawler = get_crawler()
+        with warnings.catch_warnings(record=True) as w:
+            spider.set_crawler(crawler)
+            self.assertIn("set_crawler", str(w[0].message))
+            self.assertTrue(hasattr(spider, 'crawler'))
+            self.assertIs(spider.crawler, crawler)
+            self.assertTrue(hasattr(spider, 'settings'))
+            self.assertIs(spider.settings, crawler.settings)
+
+    def test_from_crawler_crawler_and_settings_population(self):
+        crawler = get_crawler()
+        spider = self.spider_class.from_crawler(crawler, 'example.com')
+        self.assertTrue(hasattr(spider, 'crawler'))
+        self.assertIs(spider.crawler, crawler)
+        self.assertTrue(hasattr(spider, 'settings'))
+        self.assertIs(spider.settings, crawler.settings)
+
+    def test_from_crawler_init_call(self):
+        with mock.patch.object(self.spider_class, '__init__',
+                               return_value=None) as mock_init:
+            self.spider_class.from_crawler(get_crawler(), 'example.com',
+                                           foo='bar')
+            mock_init.assert_called_once_with('example.com', foo='bar')
+
 
 class InitSpiderTest(SpiderTest):