diff --git a/conftest.py b/conftest.py index 9f9a5bca7..aa27ddd2b 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,7 @@ from twisted.python import log from scrapy import optional_features -collect_ignore = ["scrapy/stats.py"] +collect_ignore = ["scrapy/stats.py", "scrapy/project.py"] if 'django' not in optional_features: collect_ignore.append("tests/test_djangoitem/models.py") diff --git a/docs/faq.rst b/docs/faq.rst index 47bfede71..1d6c56d97 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -280,37 +280,6 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. - -I'm getting an error: "cannot import name crawler" --------------------------------------------------- - -This is caused by Scrapy changes due to the singletons removal. The error is -most likely raised by a module (extension, middleware, pipeline or spider) in -your Scrapy project that imports ``crawler`` from ``scrapy.project``. For -example:: - - from scrapy.project import crawler - - class SomeExtension(object): - def __init__(self): - self.crawler = crawler - # ... - -This way to access the crawler object is deprecated, the code should be ported -to use ``from_crawler`` class method, for example:: - - class SomeExtension(object): - - @classmethod - def from_crawler(cls, crawler): - o = cls() - o.crawler = crawler - return o - -Scrapy command line tool has some backwards compatibility in place to support -the old import mechanism (with a deprecation warning), but this mechanism may -not work if you use Scrapy differently (for example, as a library). - .. _user agents: http://en.wikipedia.org/wiki/User_agent .. _LIFO: http://en.wikipedia.org/wiki/LIFO .. _DFO order: http://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 37268c3ca..5c1cfbd47 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -186,7 +186,7 @@ Here's an example of how you would call it from your spider:: # We want to inspect one specific response. if ".org" in response.url: from scrapy.shell import inspect_response - inspect_response(response) + inspect_response(response, self) # Rest of parsing code. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 597bb2e9d..352cff6e5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,18 +31,6 @@ class Crawler(object): self.spider = None self.engine = None - def install(self): - # TODO: remove together with scrapy.project.crawler usage - import scrapy.project - assert not hasattr(scrapy.project, 'crawler'), "crawler already installed" - scrapy.project.crawler = self - - def uninstall(self): - # TODO: remove together with scrapy.project.crawler usage - import scrapy.project - assert hasattr(scrapy.project, 'crawler'), "crawler not installed" - del scrapy.project.crawler - @defer.inlineCallbacks def crawl(self, *args, **kwargs): assert not self.crawling, "Crawling already taking place" @@ -84,9 +72,6 @@ class CrawlerRunner(object): crawler = self._create_logged_crawler(spidercls) self.crawlers.add(crawler) - crawler.install() - crawler.signals.connect(crawler.uninstall, signals.engine_stopped) - d = crawler.crawl(*args, **kwargs) self.crawl_deferreds.add(d) return d diff --git a/scrapy/project.py b/scrapy/project.py index bbe947761..d8973a6c7 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,13 +1,17 @@ -""" ---------- WARNING: THIS MODULE IS DEPRECATED ----------- -This module is deprecated. If you want to get the Scrapy crawler from your -extension, middleware or pipeline implement the `from_crawler` class method. +""" +Obsolete module, kept for giving a meaningful error message when trying to +import. +""" + +raise ImportError("""scrapy.project usage has become obsolete. + +If you want to get the Scrapy crawler from your extension, middleware or +pipeline implement the `from_crawler` class method (or look up for extending +components that have already done it, such as spiders). For example: @classmethod def from_crawler(cls, crawler): - return cls(crawler) - -""" + return cls(crawler)""") diff --git a/scrapy/shell.py b/scrapy/shell.py index 6c48ef186..8f87fcb41 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -123,10 +123,9 @@ class Shell(object): return isinstance(value, self.relevant_classes) -def inspect_response(response, spider=None): +def inspect_response(response, spider): """Open a shell to inspect the given response""" - from scrapy.project import crawler - Shell(crawler).start(response=response, spider=spider) + Shell(spider.crawler).start(response=response) def _request_deferred(request): diff --git a/scrapy/spider.py b/scrapy/spider.py index df367b700..943925042 100644 --- a/scrapy/spider.py +++ b/scrapy/spider.py @@ -93,6 +93,6 @@ class ObsoleteClass(object): raise AttributeError(self.message) spiders = ObsoleteClass(""" -"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute" +"from scrapy.spider import spiders" no longer works - use "from scrapy.spidermanager import SpiderManager" and instantiate it with your project settings" """) diff --git a/scrapy/stats.py b/scrapy/stats.py index b8128dfc2..710601430 100644 --- a/scrapy/stats.py +++ b/scrapy/stats.py @@ -1,7 +1,8 @@ -from scrapy.project import crawler -stats = crawler.stats -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead", - ScrapyDeprecationWarning, stacklevel=2) +""" +Obsolete module, kept for giving a meaningful error message when trying to +import. +""" + +raise ImportError("scrapy.stats usage has become obsolete, use " + "`crawler.stats` attribute instead") diff --git a/tests/test_engine.py b/tests/test_engine.py index 244d339ef..67fb8ae79 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -93,7 +93,6 @@ class CrawlerRun(object): dispatcher.connect(self.record_signal, signal) self.crawler = get_crawler(TestSpider) - self.crawler.install() self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) @@ -109,7 +108,6 @@ class CrawlerRun(object): for name, signal in vars(signals).items(): if not name.startswith('_'): disconnect_all(signal) - self.crawler.uninstall() self.deferred.callback(None) def geturl(self, path):