diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 0329e2a8f..03a0b4124 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -76,12 +76,6 @@ how you :ref:`configure the downloader middlewares For an introduction on extensions and a list of available extensions on Scrapy see :ref:`topics-extensions`. - .. attribute:: spiders - - The spider manager which takes care of loading spiders. - - Most extensions won't need to access this attribute. - .. attribute:: engine The execution engine, which coordinates the core crawling logic diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 352cff6e5..52e57fe83 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ import six import signal +import warnings from twisted.internet import reactor, defer @@ -7,6 +8,7 @@ from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.extension import ExtensionManager from scrapy.signalmanager import SignalManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals @@ -23,14 +25,22 @@ class Crawler(object): self.logformatter = lf_cls.from_crawler(self) self.extensions = ExtensionManager.from_crawler(self) - # Attribute kept for backward compatibility (Use CrawlerRunner.spiders) - spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) - self.spiders = spman_cls.from_settings(self.settings) - self.crawling = False self.spider = None self.engine = None + @property + def spiders(self): + if not hasattr(self, '_spiders'): + warnings.warn("Crawler.spiders is deprecated, use " + "CrawlerRunner.spiders or instantiate " + "scrapy.spidermanager.SpiderManager with your " + "settings.", + category=ScrapyDeprecationWarning, stacklevel=2) + spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) + self._spiders = spman_cls.from_settings(self.settings) + return self._spiders + @defer.inlineCallbacks def crawl(self, *args, **kwargs): assert not self.crawling, "Crawling already taking place" diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index ef88eab7e..f3c667cd0 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -8,6 +8,7 @@ tests/test_contrib_exporter.py tests/test_contrib_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py +tests/test_crawler.py tests/test_djangoitem/__init__.py tests/test_downloader_handlers.py tests/test_downloadermiddleware_ajaxcrawlable.py diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 000000000..55381c030 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,24 @@ +import warnings +import unittest + +from scrapy.crawler import Crawler +from scrapy.settings import Settings +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.misc import load_object + + +class CrawlerTestCase(unittest.TestCase): + + def setUp(self): + self.crawler = Crawler(DefaultSpider, Settings()) + + def test_deprecated_attribute_spiders(self): + with warnings.catch_warnings(record=True) as w: + spiders = self.crawler.spiders + self.assertEqual(len(w), 1) + self.assertIn("Crawler.spiders", str(w[0].message)) + sm_cls = load_object(self.crawler.settings['SPIDER_MANAGER_CLASS']) + self.assertIsInstance(spiders, sm_cls) + + self.crawler.spiders + self.assertEqual(len(w), 1, "Warn deprecated access only once")