Drop support for scrapy.project.crawler (And scrapy.stats consequently)

conftest.py

@@ -4,7 +4,7 @@ from twisted.python import log
 
 from scrapy import optional_features
 
-collect_ignore = ["scrapy/stats.py"]
+collect_ignore = ["scrapy/stats.py", "scrapy/project.py"]
 if 'django' not in optional_features:
     collect_ignore.append("tests/test_djangoitem/models.py")
 

docs/faq.rst

@@ -280,37 +280,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
-
-I'm getting an error: "cannot import name crawler"
---------------------------------------------------
-
-This is caused by Scrapy changes due to the singletons removal. The error is
-most likely raised by a module (extension, middleware, pipeline or spider) in
-your Scrapy project that imports ``crawler`` from ``scrapy.project``. For
-example::
-
-    from scrapy.project import crawler
-
-    class SomeExtension(object):
-        def __init__(self):
-            self.crawler = crawler
-            # ...
-
-This way to access the crawler object is deprecated, the code should be ported
-to use ``from_crawler`` class method, for example::
-
-    class SomeExtension(object):
-
-        @classmethod
-        def from_crawler(cls, crawler):
-            o = cls()
-            o.crawler = crawler
-            return o
-
-Scrapy command line tool has some backwards compatibility in place to support
-the old import mechanism (with a deprecation warning), but this mechanism may
-not work if you use Scrapy differently (for example, as a library).
-
 .. _user agents: http://en.wikipedia.org/wiki/User_agent
 .. _LIFO: http://en.wikipedia.org/wiki/LIFO
 .. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search

docs/topics/shell.rst

@@ -186,7 +186,7 @@ Here's an example of how you would call it from your spider::
             # We want to inspect one specific response.
             if ".org" in response.url:
                 from scrapy.shell import inspect_response
-                inspect_response(response)
+                inspect_response(response, self)
 
             # Rest of parsing code.
 

scrapy/crawler.py

@@ -31,18 +31,6 @@ class Crawler(object):
         self.spider = None
         self.engine = None
 
-    def install(self):
-        # TODO: remove together with scrapy.project.crawler usage
-        import scrapy.project
-        assert not hasattr(scrapy.project, 'crawler'), "crawler already installed"
-        scrapy.project.crawler = self
-
-    def uninstall(self):
-        # TODO: remove together with scrapy.project.crawler usage
-        import scrapy.project
-        assert hasattr(scrapy.project, 'crawler'), "crawler not installed"
-        del scrapy.project.crawler
-
     @defer.inlineCallbacks
     def crawl(self, *args, **kwargs):
         assert not self.crawling, "Crawling already taking place"
@@ -84,9 +72,6 @@ class CrawlerRunner(object):
         crawler = self._create_logged_crawler(spidercls)
         self.crawlers.add(crawler)
 
-        crawler.install()
-        crawler.signals.connect(crawler.uninstall, signals.engine_stopped)
-
         d = crawler.crawl(*args, **kwargs)
         self.crawl_deferreds.add(d)
         return d

scrapy/project.py

@@ -1,13 +1,17 @@
-"""
---------- WARNING: THIS MODULE IS DEPRECATED -----------
 
-This module is deprecated. If you want to get the Scrapy crawler from your
-extension, middleware or pipeline implement the `from_crawler` class method.
+"""
+Obsolete module, kept for giving a meaningful error message when trying to
+import.
+"""
+
+raise ImportError("""scrapy.project usage has become obsolete.
+
+If you want to get the Scrapy crawler from your extension, middleware or
+pipeline implement the `from_crawler` class method (or look up for extending
+components that have already done it, such as spiders).
 
 For example:
 
     @classmethod
     def from_crawler(cls, crawler):
-        return cls(crawler)
-
-"""
+        return cls(crawler)""")

scrapy/shell.py

@@ -123,10 +123,9 @@ class Shell(object):
         return isinstance(value, self.relevant_classes)
 
 
-def inspect_response(response, spider=None):
+def inspect_response(response, spider):
     """Open a shell to inspect the given response"""
-    from scrapy.project import crawler
-    Shell(crawler).start(response=response, spider=spider)
+    Shell(spider.crawler).start(response=response)
 
 
 def _request_deferred(request):

scrapy/spider.py

@@ -93,6 +93,6 @@ class ObsoleteClass(object):
         raise AttributeError(self.message)
 
 spiders = ObsoleteClass("""
-"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute"
+"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings"
 """)
 

scrapy/stats.py

@@ -1,7 +1,8 @@
-from scrapy.project import crawler
-stats = crawler.stats
 
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead",
-    ScrapyDeprecationWarning, stacklevel=2)
+"""
+Obsolete module, kept for giving a meaningful error message when trying to
+import.
+"""
+
+raise ImportError("scrapy.stats usage has become obsolete, use "
+                  "`crawler.stats` attribute instead")

tests/test_engine.py

@@ -93,7 +93,6 @@ class CrawlerRun(object):
                 dispatcher.connect(self.record_signal, signal)
 
         self.crawler = get_crawler(TestSpider)
-        self.crawler.install()
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
@@ -109,7 +108,6 @@ class CrawlerRun(object):
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
                 disconnect_all(signal)
-        self.crawler.uninstall()
         self.deferred.callback(None)
 
     def geturl(self, path):