diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 28a032bb3..7a7d76edc 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -4,7 +4,7 @@ from scrapy.core.queue import ExecutionQueue from scrapy.project import crawler from scrapy.conf import settings from scrapy.http import Request -from scrapy.spider import spiders +from scrapy.project import crawler from scrapy.utils.url import is_url from collections import defaultdict @@ -39,14 +39,14 @@ class Command(ScrapyCommand): if opts.spider: try: - spider = spiders.create(opts.spider) + spider = crawler.spiders.create(opts.spider) for url in urls: q.append_url(url, spider) except KeyError: log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) else: for name, urls in self._group_urls_by_spider(urls): - spider = spiders.create(name) + spider = crawler.spiders.create(name) for url in urls: q.append_url(url, spider) @@ -56,7 +56,7 @@ class Command(ScrapyCommand): def _group_urls_by_spider(self, urls): spider_urls = defaultdict(list) for url in urls: - spider_names = spiders.find_by_request(Request(url)) + spider_names = crawler.spiders.find_by_request(Request(url)) if not spider_names: log.msg('Could not find spider that handles url: %s' % url, log.ERROR) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 766d7cb5f..a9b7ede4a 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -4,7 +4,8 @@ from scrapy import log from scrapy.command import ScrapyCommand from scrapy.project import crawler from scrapy.http import Request -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import BaseSpider +from scrapy.project import crawler from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -43,7 +44,7 @@ class Command(ScrapyCommand): spider = None if opts.spider: try: - spider = spiders.create(opts.spider) + spider = crawler.spiders.create(opts.spider) except KeyError: log.msg("Could not find spider: %s" % opts.spider, log.ERROR) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 5317ceb87..f451afbc4 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,7 @@ from os import listdir from os.path import join, dirname, abspath, exists, splitext import scrapy -from scrapy.spider import spiders +from scrapy.project import crawler from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.utils.template import render_templatefile, string_camelcase @@ -60,7 +60,7 @@ class Command(ScrapyCommand): name, domain = args[0:2] module = sanitize_module_name(name) try: - spider = spiders.create(name) + spider = crawler.spiders.create(name) except KeyError: pass else: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 228874803..a3fe8a3fa 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,5 +1,5 @@ from scrapy.command import ScrapyCommand -from scrapy.spider import spiders +from scrapy.project import crawler class Command(ScrapyCommand): @@ -10,4 +10,4 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - print "\n".join(spiders.list()) + print "\n".join(crawler.spiders.list()) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index b63129d67..9cce27d53 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -2,7 +2,7 @@ from scrapy.command import ScrapyCommand from scrapy.project import crawler from scrapy.http import Request from scrapy.item import BaseItem -from scrapy.spider import spiders +from scrapy.project import crawler from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output from scrapy.utils.url import is_url @@ -78,12 +78,12 @@ class Command(ScrapyCommand): if opts.spider: try: - spider = spiders.create(opts.spider) + spider = crawler.spiders.create(opts.spider) except KeyError: log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) return else: - spider = spiders.create_for_request(request) + spider = crawler.spiders.create_for_request(request) if spider is None: log.msg('Unable to find spider for URL: %s' % args[0], log.ERROR) return diff --git a/scrapy/contrib/webservice/spiders.py b/scrapy/contrib/webservice/spiders.py index dd64645a1..35997ee6a 100644 --- a/scrapy/contrib/webservice/spiders.py +++ b/scrapy/contrib/webservice/spiders.py @@ -1,10 +1,12 @@ from scrapy.webservice import JsonRpcResource -from scrapy.spider import spiders +from scrapy.project import crawler class SpidersResource(JsonRpcResource): ws_name = 'spiders' - def __init__(self, _spiders=spiders): + def __init__(self, _spiders=None): + if _spiders is None: + _spiders = crawler.spiders JsonRpcResource.__init__(self) self._target = _spiders diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 082cb9483..f7fe32254 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -17,7 +17,6 @@ from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import IgnoreRequest, DontCloseSpider from scrapy.http import Response, Request -from scrapy.spider import spiders from scrapy.utils.misc import load_object from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.defer import mustbe_deferred @@ -278,7 +277,7 @@ class ExecutionEngine(object): dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason)) dfd.addErrback(log.err, "Unhandled error in stats.close_spider()", spider=spider) - dfd.addBoth(lambda _: spiders.close_spider(spider)) + dfd.addBoth(lambda _: self.crawler.spiders.close_spider(spider)) dfd.addErrback(log.err, "Unhandled error in spiders.close_spider()", spider=spider) dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider)) diff --git a/scrapy/core/queue.py b/scrapy/core/queue.py index 354f3a135..428e7a5ab 100644 --- a/scrapy/core/queue.py +++ b/scrapy/core/queue.py @@ -3,14 +3,16 @@ from twisted.internet import defer from scrapy.http import Request from scrapy.utils.misc import arg_to_iter from scrapy import log -from scrapy.spider import spiders class ExecutionQueue(object): polling_delay = 5 - def __init__(self, _spiders=spiders): + def __init__(self, _spiders=None): + if _spiders is None: + from scrapy.project import crawler + _spiders = crawler.spiders self.spider_requests = [] self._spiders = _spiders diff --git a/scrapy/crawler.py b/scrapy/crawler.py index fcf56a79d..a743a86a5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -6,15 +6,15 @@ from scrapy.core.engine import ExecutionEngine from scrapy.core.queue import ExecutionQueue from scrapy.extension import extensions from scrapy import log -from scrapy.spider import spiders from scrapy.utils.ossignal import install_shutdown_handlers, signal_names class Crawler(object): - def __init__(self): + def __init__(self, spiders): self.configured = False self.control_reactor = True + self.spiders = spiders self.engine = ExecutionEngine(self) def configure(self, control_reactor=True, queue=None): @@ -26,8 +26,8 @@ class Crawler(object): log.start() if not extensions.loaded: extensions.load() - if not spiders.loaded: - spiders.load() + if not self.spiders.loaded: + self.spiders.load() log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()), level=log.DEBUG) diff --git a/scrapy/project.py b/scrapy/project.py index cb2ccd26c..9c01e0e3e 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,3 +1,6 @@ +from scrapy.conf import settings +from scrapy.utils.misc import load_object from scrapy.crawler import Crawler -crawler = Crawler() +_spiders = load_object(settings['SPIDER_MANAGER_CLASS'])() +crawler = Crawler(_spiders) diff --git a/scrapy/shell.py b/scrapy/shell.py index 693f0f6e3..f62f19103 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -11,7 +11,7 @@ from twisted.python.failure import Failure from scrapy import log from scrapy.item import BaseItem -from scrapy.spider import BaseSpider, spiders +from scrapy.spider import BaseSpider from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser @@ -58,8 +58,8 @@ class Shell(object): url = any_to_uri(request_or_url) request = Request(url, dont_filter=True) if spider is None: - spider = spiders.create_for_request(request, BaseSpider('default'), \ - log_multiple=True) + spider = self.crawler.spiders.create_for_request(request, \ + BaseSpider('default'), log_multiple=True) self.crawler.engine.open_spider(spider) response = None try: diff --git a/scrapy/spider/models.py b/scrapy/spider.py similarity index 82% rename from scrapy/spider/models.py rename to scrapy/spider.py index 0acbbe364..530dc916a 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider.py @@ -55,3 +55,16 @@ class BaseSpider(object_ref): return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self)) __repr__ = __str__ + + +class ObsoleteClass(object): + def __init__(self, message): + self.message = message + + def __getattr__(self, name): + raise AttributeError(self.message) + +spiders = ObsoleteClass(""" +"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute" +""") + diff --git a/scrapy/spider/__init__.py b/scrapy/spider/__init__.py deleted file mode 100644 index 875bec845..000000000 --- a/scrapy/spider/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from scrapy.spider.models import BaseSpider -from scrapy.utils.misc import load_object -from scrapy.conf import settings - -spiders = load_object(settings['SPIDER_MANAGER_CLASS'])() diff --git a/scrapy/telnet.py b/scrapy/telnet.py index 4a57888ab..c5dbced59 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -13,7 +13,6 @@ from twisted.internet import reactor, protocol from scrapy.extension import extensions from scrapy.exceptions import NotConfigured from scrapy.project import crawler -from scrapy.spider import spiders from scrapy.stats import stats from scrapy.utils.signal import send_catch_log from scrapy.utils.trackref import print_live_refs @@ -52,7 +51,7 @@ class TelnetConsole(protocol.ServerFactory): 'manager': crawler, 'extensions': extensions, 'stats': stats, - 'spiders': spiders, + 'spiders': crawler.spiders, 'settings': settings, 'est': print_engine_status, 'p': pprint.pprint,