Moved scrapy.spider.spiders singleton to a "spiders" attribute of the scrapy.project.crawler singleton. Refs #189

Warning: this is a backwards incompatible change.

--HG--
rename : scrapy/spider/models.py => scrapy/spider.py

scrapy/commands/crawl.py

@@ -4,7 +4,7 @@ from scrapy.core.queue import ExecutionQueue
 from scrapy.project import crawler
 from scrapy.conf import settings
 from scrapy.http import Request
-from scrapy.spider import spiders
+from scrapy.project import crawler
 from scrapy.utils.url import is_url
 
 from collections import defaultdict
@@ -39,14 +39,14 @@ class Command(ScrapyCommand):
 
         if opts.spider:
             try:
-                spider = spiders.create(opts.spider)
+                spider = crawler.spiders.create(opts.spider)
                 for url in urls:
                     q.append_url(url, spider)
             except KeyError:
                 log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
         else:
             for name, urls in self._group_urls_by_spider(urls):
-                spider = spiders.create(name)
+                spider = crawler.spiders.create(name)
                 for url in urls:
                     q.append_url(url, spider)
 
@@ -56,7 +56,7 @@ class Command(ScrapyCommand):
     def _group_urls_by_spider(self, urls):
         spider_urls = defaultdict(list)
         for url in urls:
-            spider_names = spiders.find_by_request(Request(url))
+            spider_names = crawler.spiders.find_by_request(Request(url))
             if not spider_names:
                 log.msg('Could not find spider that handles url: %s' % url,
                         log.ERROR)

scrapy/commands/fetch.py

@@ -4,7 +4,8 @@ from scrapy import log
 from scrapy.command import ScrapyCommand
 from scrapy.project import crawler
 from scrapy.http import Request
-from scrapy.spider import BaseSpider, spiders
+from scrapy.spider import BaseSpider
+from scrapy.project import crawler
 from scrapy.utils.url import is_url
 
 class Command(ScrapyCommand):
@@ -43,7 +44,7 @@ class Command(ScrapyCommand):
         spider = None
         if opts.spider:
             try:
-                spider = spiders.create(opts.spider)
+                spider = crawler.spiders.create(opts.spider)
             except KeyError:
                 log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
 

scrapy/commands/genspider.py

@@ -4,7 +4,7 @@ from os import listdir
 from os.path import join, dirname, abspath, exists, splitext
 
 import scrapy
-from scrapy.spider import spiders
+from scrapy.project import crawler
 from scrapy.command import ScrapyCommand
 from scrapy.conf import settings
 from scrapy.utils.template import render_templatefile, string_camelcase
@@ -60,7 +60,7 @@ class Command(ScrapyCommand):
         name, domain = args[0:2]
         module = sanitize_module_name(name)
         try:
-            spider = spiders.create(name)
+            spider = crawler.spiders.create(name)
         except KeyError:
             pass
         else:

scrapy/commands/list.py

@@ -1,5 +1,5 @@
 from scrapy.command import ScrapyCommand
-from scrapy.spider import spiders
+from scrapy.project import crawler
 
 class Command(ScrapyCommand):
 
@@ -10,4 +10,4 @@ class Command(ScrapyCommand):
         return "List available spiders"
 
     def run(self, args, opts):
-        print "\n".join(spiders.list())
+        print "\n".join(crawler.spiders.list())

scrapy/commands/parse.py

@@ -2,7 +2,7 @@ from scrapy.command import ScrapyCommand
 from scrapy.project import crawler
 from scrapy.http import Request
 from scrapy.item import BaseItem
-from scrapy.spider import spiders
+from scrapy.project import crawler
 from scrapy.utils import display
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.url import is_url
@@ -78,12 +78,12 @@ class Command(ScrapyCommand):
 
         if opts.spider:
             try:
-                spider = spiders.create(opts.spider)
+                spider = crawler.spiders.create(opts.spider)
             except KeyError:
                 log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
                 return
         else:
-            spider = spiders.create_for_request(request)
+            spider = crawler.spiders.create_for_request(request)
             if spider is None:
                 log.msg('Unable to find spider for URL: %s' % args[0], log.ERROR)
                 return

scrapy/contrib/webservice/spiders.py

@@ -1,10 +1,12 @@
 from scrapy.webservice import JsonRpcResource
-from scrapy.spider import spiders
+from scrapy.project import crawler
 
 class SpidersResource(JsonRpcResource):
 
     ws_name = 'spiders'
 
-    def __init__(self, _spiders=spiders):
+    def __init__(self, _spiders=None):
+        if _spiders is None:
+            _spiders = crawler.spiders
         JsonRpcResource.__init__(self)
         self._target = _spiders

scrapy/core/engine.py

@@ -17,7 +17,6 @@ from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import IgnoreRequest, DontCloseSpider
 from scrapy.http import Response, Request
-from scrapy.spider import spiders
 from scrapy.utils.misc import load_object
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
 from scrapy.utils.defer import mustbe_deferred
@@ -278,7 +277,7 @@ class ExecutionEngine(object):
         dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
         dfd.addErrback(log.err, "Unhandled error in stats.close_spider()",
             spider=spider)
-        dfd.addBoth(lambda _: spiders.close_spider(spider))
+        dfd.addBoth(lambda _: self.crawler.spiders.close_spider(spider))
         dfd.addErrback(log.err, "Unhandled error in spiders.close_spider()",
             spider=spider)
         dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))

scrapy/core/queue.py

@@ -3,14 +3,16 @@ from twisted.internet import defer
 from scrapy.http import Request
 from scrapy.utils.misc import arg_to_iter
 from scrapy import log
-from scrapy.spider import spiders
 
 
 class ExecutionQueue(object):
 
     polling_delay = 5
 
-    def __init__(self, _spiders=spiders):
+    def __init__(self, _spiders=None):
+        if _spiders is None:
+            from scrapy.project import crawler
+            _spiders = crawler.spiders
         self.spider_requests = []
         self._spiders = _spiders
 

scrapy/crawler.py

@@ -6,15 +6,15 @@ from scrapy.core.engine import ExecutionEngine
 from scrapy.core.queue import ExecutionQueue
 from scrapy.extension import extensions
 from scrapy import log
-from scrapy.spider import spiders
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 
 
 class Crawler(object):
 
-    def __init__(self):
+    def __init__(self, spiders):
         self.configured = False
         self.control_reactor = True
+        self.spiders = spiders
         self.engine = ExecutionEngine(self)
 
     def configure(self, control_reactor=True, queue=None):
@@ -26,8 +26,8 @@ class Crawler(object):
             log.start()
         if not extensions.loaded:
             extensions.load()
-        if not spiders.loaded:
-            spiders.load()
+        if not self.spiders.loaded:
+            self.spiders.load()
         log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()),
             level=log.DEBUG)
 

scrapy/project.py

@@ -1,3 +1,6 @@
+from scrapy.conf import settings
+from scrapy.utils.misc import load_object
 from scrapy.crawler import Crawler
 
-crawler = Crawler()
+_spiders = load_object(settings['SPIDER_MANAGER_CLASS'])()
+crawler = Crawler(_spiders)

scrapy/shell.py

@@ -11,7 +11,7 @@ from twisted.python.failure import Failure
 
 from scrapy import log
 from scrapy.item import BaseItem
-from scrapy.spider import BaseSpider, spiders
+from scrapy.spider import BaseSpider
 from scrapy.selector import XPathSelector, XmlXPathSelector, HtmlXPathSelector
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
@@ -58,8 +58,8 @@ class Shell(object):
             url = any_to_uri(request_or_url)
             request = Request(url, dont_filter=True)
         if spider is None:
-            spider = spiders.create_for_request(request, BaseSpider('default'), \
-                log_multiple=True)
+            spider = self.crawler.spiders.create_for_request(request, \
+                BaseSpider('default'), log_multiple=True)
         self.crawler.engine.open_spider(spider)
         response = None
         try:

scrapy/spider.py

@@ -55,3 +55,16 @@ class BaseSpider(object_ref):
         return "<%s %r at 0x%0x>" % (type(self).__name__, self.name, id(self))
 
     __repr__ = __str__
+
+
+class ObsoleteClass(object):
+    def __init__(self, message):
+        self.message = message
+
+    def __getattr__(self, name):
+        raise AttributeError(self.message)
+
+spiders = ObsoleteClass("""
+"from scrapy.spider import spiders" no longer works - use "from scrapy.project import crawler" and then access crawler.spiders attribute"
+""")
+

scrapy/spider/__init__.py

@@ -1,5 +0,0 @@
-from scrapy.spider.models import BaseSpider
-from scrapy.utils.misc import load_object
-from scrapy.conf import settings
-
-spiders = load_object(settings['SPIDER_MANAGER_CLASS'])()

scrapy/telnet.py

@@ -13,7 +13,6 @@ from twisted.internet import reactor, protocol
 from scrapy.extension import extensions
 from scrapy.exceptions import NotConfigured
 from scrapy.project import crawler
-from scrapy.spider import spiders
 from scrapy.stats import stats
 from scrapy.utils.signal import send_catch_log
 from scrapy.utils.trackref import print_live_refs
@@ -52,7 +51,7 @@ class TelnetConsole(protocol.ServerFactory):
             'manager': crawler,
             'extensions': extensions,
             'stats': stats,
-            'spiders': spiders,
+            'spiders': crawler.spiders,
             'settings': settings,
             'est': print_engine_status,
             'p': pprint.pprint,