Moved scrapymanager singleton to scrapy.project module. Refs #189

Detail of changes:

* Moved scrapy.core.manager.ExecutionManager class to scrapy.crawler.Crawler
* Added scrapy.project.crawler singleton to reference a singleton instance of
  Crawler class (previously known as scrapymanager)
* Left an alias scrapy.core.manager.scrapymanager to scrapy.project.crawler for
  backwards compatibility (to be removed in Scrapy 0.11)

docs/topics/telnetconsole.rst

@@ -45,7 +45,7 @@ convenience:
 +================+===================================================================+
 | ``engine``     | the Scrapy engine object (``scrapy.core.engine.scrapyengine``)    |
 +----------------+-------------------------------------------------------------------+
-| ``manager``    | the Scrapy manager object (``scrapy.core.manager.scrapymanager``) |
+| ``manager``    | the Project Crawler object (``scrapy.project.crawler``)           |
 +----------------+-------------------------------------------------------------------+
 | ``extensions`` | the extensions object (``scrapy.extension.extensions``)           |
 +----------------+-------------------------------------------------------------------+

scrapy/cmdline.py

@@ -113,8 +113,8 @@ def execute(argv=None):
 
     settings.defaults.update(cmd.default_settings)
     del args[0]  # remove command name from args
-    from scrapy.core.manager import scrapymanager
-    scrapymanager.configure(control_reactor=True)
+    from scrapy.project import crawler
+    crawler.configure(control_reactor=True)
     ret = _run_command(cmd, args, opts)
     if ret is False:
         parser.print_help()

scrapy/commands/crawl.py

@@ -1,7 +1,7 @@
 from scrapy import log
 from scrapy.command import ScrapyCommand
 from scrapy.core.queue import ExecutionQueue
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.conf import settings
 from scrapy.http import Request
 from scrapy.spider import spiders
@@ -50,8 +50,8 @@ class Command(ScrapyCommand):
                 for url in urls:
                     q.append_url(url, spider)
 
-        scrapymanager.queue = q
-        scrapymanager.start()
+        crawler.queue = q
+        crawler.start()
 
     def _group_urls_by_spider(self, urls):
         spider_urls = defaultdict(list)

scrapy/commands/fetch.py

@@ -2,7 +2,7 @@ import pprint
 
 from scrapy import log
 from scrapy.command import ScrapyCommand
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.http import Request
 from scrapy.spider import BaseSpider, spiders
 from scrapy.utils.url import is_url
@@ -47,8 +47,8 @@ class Command(ScrapyCommand):
             except KeyError:
                 log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
 
-        scrapymanager.configure()
-        scrapymanager.queue.append_request(request, spider, \
+        crawler.configure()
+        crawler.queue.append_request(request, spider, \
             default_spider=BaseSpider('default'))
-        scrapymanager.start()
+        crawler.start()
 

scrapy/commands/parse.py

@@ -1,5 +1,5 @@
 from scrapy.command import ScrapyCommand
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.http import Request
 from scrapy.item import BaseItem
 from scrapy.spider import spiders
@@ -88,9 +88,9 @@ class Command(ScrapyCommand):
                 log.msg('Unable to find spider for URL: %s' % args[0], log.ERROR)
                 return
 
-        scrapymanager.configure()
-        scrapymanager.queue.append_request(request, spider)
-        scrapymanager.start()
+        crawler.configure()
+        crawler.queue.append_request(request, spider)
+        crawler.start()
 
         if not responses:
             log.msg('No response returned', log.ERROR, spider=spider)

scrapy/commands/runserver.py

@@ -1,5 +1,5 @@
 from scrapy.command import ScrapyCommand
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.utils.misc import load_object
 from scrapy.conf import settings
 
@@ -12,5 +12,5 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         queue_class = load_object(settings['SERVICE_QUEUE'])
-        scrapymanager.queue = queue_class()
-        scrapymanager.start()
+        crawler.queue = queue_class()
+        crawler.start()

scrapy/commands/runspider.py

@@ -4,7 +4,7 @@ import os
 from scrapy import log
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.command import ScrapyCommand
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 
 def _import_file(filepath):
     abspath = os.path.abspath(filepath)
@@ -52,5 +52,5 @@ class Command(ScrapyCommand):
             return
         spider = spclasses.pop()()
         # schedule spider and start engine
-        scrapymanager.queue.append_spider(spider)
-        scrapymanager.start()
+        crawler.queue.append_spider(spider)
+        crawler.start()

scrapy/commands/shell.py

@@ -4,7 +4,7 @@ Scrapy Shell
 See documentation in docs/topics/shell.rst
 """
 
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.core.queue import KeepAliveExecutionQueue
 from scrapy.command import ScrapyCommand
 from scrapy.shell import Shell
@@ -30,7 +30,7 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         url = args[0] if args else None
-        shell = Shell(scrapymanager, update_vars=self.update_vars, inthread=True)
-        shell.start(url=url).addBoth(lambda _: scrapymanager.stop())
-        scrapymanager.queue = KeepAliveExecutionQueue()
-        scrapymanager.start()
+        shell = Shell(crawler, update_vars=self.update_vars, inthread=True)
+        shell.start(url=url).addBoth(lambda _: crawler.stop())
+        crawler.queue = KeepAliveExecutionQueue()
+        crawler.start()

scrapy/contrib/closespider.py

@@ -10,7 +10,7 @@ from twisted.internet import reactor
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import signals
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.conf import settings
 
 class CloseSpider(object):
@@ -30,13 +30,13 @@ class CloseSpider(object):
 
     def spider_opened(self, spider):
         self.tasks[spider] = reactor.callLater(self.timeout, \
-            scrapymanager.engine.close_spider, spider=spider, \
+            crawler.engine.close_spider, spider=spider, \
             reason='closespider_timeout')
         
     def item_passed(self, item, spider):
         self.counts[spider] += 1
         if self.counts[spider] == self.itempassed:
-            scrapymanager.engine.close_spider(spider, 'closespider_itempassed')
+            crawler.engine.close_spider(spider, 'closespider_itempassed')
 
     def spider_closed(self, spider):
         self.counts.pop(spider, None)

scrapy/contrib/downloadermiddleware/robotstxt.py

@@ -9,7 +9,7 @@ import robotparser
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import signals
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
@@ -42,7 +42,7 @@ class RobotsTxtMiddleware(object):
             self._parsers[netloc] = None
             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
             robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
-            dfd = scrapymanager.engine.download(robotsreq, spider)
+            dfd = crawler.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
             self._spider_netlocs[spider].add(netloc)
         return self._parsers[netloc]

scrapy/contrib/memusage.py

@@ -13,7 +13,7 @@ from twisted.internet import task
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy import signals
 from scrapy import log
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.conf import settings
@@ -74,7 +74,7 @@ class MemoryUsage(object):
                         (settings['BOT_NAME'], mem, socket.gethostname())
                 self._send_report(self.notify_mails, subj)
                 stats.set_value('memusage/limit_notified', 1)
-            scrapymanager.stop()
+            crawler.stop()
 
     def _check_warning(self):
         if self.warned: # warn only once

scrapy/contrib/pipeline/media.py

@@ -2,7 +2,7 @@ from twisted.internet.defer import Deferred, DeferredList
 
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy import log
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
@@ -87,7 +87,7 @@ class MediaPipeline(object):
     def download(self, request, info):
         """Defines how to download the media request"""
         request.priority = self.DOWNLOAD_PRIORITY
-        return scrapymanager.engine.download(request, info.spider)
+        return crawler.engine.download(request, info.spider)
 
     def media_to_download(self, request, info):
         """Check request before starting download"""

scrapy/contrib/spiderclosedelay.py

@@ -9,7 +9,7 @@ from scrapy.xlib.pydispatch import dispatcher
 from collections import defaultdict
 
 from scrapy import signals
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.exceptions import NotConfigured, DontCloseSpider
 from scrapy.conf import settings
 
@@ -26,7 +26,7 @@ class SpiderCloseDelay(object):
 
     def spider_idle(self, spider):
         try:
-            lastseen = scrapymanager.engine.downloader.sites[spider].lastseen
+            lastseen = crawler.engine.downloader.sites[spider].lastseen
         except KeyError:
             lastseen = None
         if not lastseen:

scrapy/contrib/spidermiddleware/requestlimit.py

@@ -7,7 +7,7 @@ from itertools import imap
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import signals
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.conf import settings
 from scrapy.http import Request
@@ -61,5 +61,5 @@ class RequestLimitMiddleware(object):
             return request_or_other
 
     def _pending_count(self, spider):
-        pending = scrapymanager.engine.scheduler.pending_requests.get(spider, [])
+        pending = crawler.engine.scheduler.pending_requests.get(spider, [])
         return len(pending)

scrapy/contrib/webservice/enginestatus.py

@@ -1,12 +1,12 @@
 from scrapy.webservice import JsonResource
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.utils.engine import get_engine_status
 
 class EngineStatusResource(JsonResource):
 
     ws_name = 'enginestatus'
 
-    def __init__(self, spider_name=None, _manager=scrapymanager):
+    def __init__(self, spider_name=None, _manager=crawler):
         JsonResource.__init__(self)
         self._spider_name = spider_name
         self.isLeaf = spider_name is not None

scrapy/contrib/webservice/manager.py

@@ -1,10 +1,10 @@
 from scrapy.webservice import JsonRpcResource
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 
 class ManagerResource(JsonRpcResource):
 
     ws_name = 'manager'
 
-    def __init__(self, _manager=scrapymanager):
+    def __init__(self, _manager=crawler):
         JsonRpcResource.__init__(self)
         self._target = _manager

scrapy/core/engine.py

@@ -24,7 +24,8 @@ from scrapy.utils.defer import mustbe_deferred
 
 class ExecutionEngine(object):
 
-    def __init__(self):
+    def __init__(self, crawler):
+        self.crawler = crawler
         self.configured = False
         self.closing = {} # dict (spider -> reason) of spiders being closed
         self.closing_dfds = {} # dict (spider -> deferred) of spiders being closed

scrapy/core/manager.py

@@ -1,98 +1,6 @@
-import signal
+import warnings
+warnings.warn("scrapy.core.manager.scrapymanager is deprecated and will be removed in Scrapy 0.11, use scrapy.project.crawler instead", \
+    DeprecationWarning, stacklevel=2)
 
-from twisted.internet import reactor, defer
-
-from scrapy.core.engine import ExecutionEngine
-from scrapy.core.queue import ExecutionQueue
-from scrapy.extension import extensions
-from scrapy import log
-from scrapy.spider import spiders
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-
-
-class ExecutionManager(object):
-
-    def __init__(self):
-        self.configured = False
-        self.control_reactor = True
-        self.engine = ExecutionEngine()
-
-    def configure(self, control_reactor=True, queue=None):
-        self.control_reactor = control_reactor
-        if control_reactor:
-            install_shutdown_handlers(self._signal_shutdown)
-
-        if not log.started:
-            log.start()
-        if not extensions.loaded:
-            extensions.load()
-        if not spiders.loaded:
-            spiders.load()
-        log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()),
-            level=log.DEBUG)
-
-        self.queue = queue or ExecutionQueue()
-        self.engine.configure(self._spider_closed)
-        self.configured = True
-
-    @defer.inlineCallbacks
-    def _start_next_spider(self):
-        spider, requests = yield defer.maybeDeferred(self.queue.get_next)
-        if spider:
-            self._start_spider(spider, requests)
-        if self.engine.has_capacity() and not self._nextcall.active():
-            self._nextcall = reactor.callLater(self.queue.polling_delay, \
-                self._spider_closed)
-
-    @defer.inlineCallbacks
-    def _start_spider(self, spider, requests):
-        """Don't call this method. Use self.queue to start new spiders"""
-        yield defer.maybeDeferred(self.engine.open_spider, spider)
-        for request in requests:
-            self.engine.crawl(request, spider)
-
-    @defer.inlineCallbacks
-    def _spider_closed(self, spider=None):
-        if not self.engine.open_spiders:
-            is_finished = yield defer.maybeDeferred(self.queue.is_finished)
-            if is_finished:
-                self.stop()
-                return
-        if self.engine.has_capacity():
-            self._start_next_spider()
-
-    @defer.inlineCallbacks
-    def start(self):
-        yield defer.maybeDeferred(self.engine.start)
-        self._nextcall = reactor.callLater(0, self._start_next_spider)
-        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
-        if self.control_reactor:
-            reactor.run(installSignalHandlers=False)
-
-    @defer.inlineCallbacks
-    def stop(self):
-        if self._nextcall.active():
-            self._nextcall.cancel()
-        if self.engine.running:
-            yield defer.maybeDeferred(self.engine.stop)
-        try:
-            reactor.stop()
-        except RuntimeError: # raised if already stopped or in shutdown stage
-            pass
-
-    def _signal_shutdown(self, signum, _):
-        signame = signal_names[signum]
-        log.msg("Received %s, shutting down gracefully. Send again to force " \
-            "unclean shutdown" % signame, level=log.INFO)
-        reactor.callFromThread(self.stop)
-        install_shutdown_handlers(self._signal_kill)
-
-    def _signal_kill(self, signum, _):
-        signame = signal_names[signum]
-        log.msg('Received %s twice, forcing unclean shutdown' % signame, \
-            level=log.INFO)
-        log.log_level = log.SILENT # disable logging of confusing tracebacks
-        reactor.callFromThread(self.engine.kill)
-        install_shutdown_handlers(signal.SIG_IGN)
-
-scrapymanager = ExecutionManager()
+from scrapy.project import crawler
+scrapymanager = crawler

scrapy/crawler.py

@@ -0,0 +1,96 @@
+import signal
+
+from twisted.internet import reactor, defer
+
+from scrapy.core.engine import ExecutionEngine
+from scrapy.core.queue import ExecutionQueue
+from scrapy.extension import extensions
+from scrapy import log
+from scrapy.spider import spiders
+from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+
+
+class Crawler(object):
+
+    def __init__(self):
+        self.configured = False
+        self.control_reactor = True
+        self.engine = ExecutionEngine(self)
+
+    def configure(self, control_reactor=True, queue=None):
+        self.control_reactor = control_reactor
+        if control_reactor:
+            install_shutdown_handlers(self._signal_shutdown)
+
+        if not log.started:
+            log.start()
+        if not extensions.loaded:
+            extensions.load()
+        if not spiders.loaded:
+            spiders.load()
+        log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()),
+            level=log.DEBUG)
+
+        self.queue = queue or ExecutionQueue()
+        self.engine.configure(self._spider_closed)
+        self.configured = True
+
+    @defer.inlineCallbacks
+    def _start_next_spider(self):
+        spider, requests = yield defer.maybeDeferred(self.queue.get_next)
+        if spider:
+            self._start_spider(spider, requests)
+        if self.engine.has_capacity() and not self._nextcall.active():
+            self._nextcall = reactor.callLater(self.queue.polling_delay, \
+                self._spider_closed)
+
+    @defer.inlineCallbacks
+    def _start_spider(self, spider, requests):
+        """Don't call this method. Use self.queue to start new spiders"""
+        yield defer.maybeDeferred(self.engine.open_spider, spider)
+        for request in requests:
+            self.engine.crawl(request, spider)
+
+    @defer.inlineCallbacks
+    def _spider_closed(self, spider=None):
+        if not self.engine.open_spiders:
+            is_finished = yield defer.maybeDeferred(self.queue.is_finished)
+            if is_finished:
+                self.stop()
+                return
+        if self.engine.has_capacity():
+            self._start_next_spider()
+
+    @defer.inlineCallbacks
+    def start(self):
+        yield defer.maybeDeferred(self.engine.start)
+        self._nextcall = reactor.callLater(0, self._start_next_spider)
+        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
+        if self.control_reactor:
+            reactor.run(installSignalHandlers=False)
+
+    @defer.inlineCallbacks
+    def stop(self):
+        if self._nextcall.active():
+            self._nextcall.cancel()
+        if self.engine.running:
+            yield defer.maybeDeferred(self.engine.stop)
+        try:
+            reactor.stop()
+        except RuntimeError: # raised if already stopped or in shutdown stage
+            pass
+
+    def _signal_shutdown(self, signum, _):
+        signame = signal_names[signum]
+        log.msg("Received %s, shutting down gracefully. Send again to force " \
+            "unclean shutdown" % signame, level=log.INFO)
+        reactor.callFromThread(self.stop)
+        install_shutdown_handlers(self._signal_kill)
+
+    def _signal_kill(self, signum, _):
+        signame = signal_names[signum]
+        log.msg('Received %s twice, forcing unclean shutdown' % signame, \
+            level=log.INFO)
+        log.log_level = log.SILENT # disable logging of confusing tracebacks
+        reactor.callFromThread(self.engine.kill)
+        install_shutdown_handlers(signal.SIG_IGN)

scrapy/project.py

@@ -0,0 +1,3 @@
+from scrapy.crawler import Crawler
+
+crawler = Crawler()

scrapy/shell.py

@@ -107,5 +107,5 @@ class Shell(object):
 
 def inspect_response(response, spider=None):
     """Open a shell to inspect the given response"""
-    from scrapy.core.manager import scrapymanager
-    Shell(scrapymanager).start(response=response, spider=spider)
+    from scrapy.project import crawler
+    Shell(crawler).start(response=response, spider=spider)

scrapy/telnet.py

@@ -12,7 +12,7 @@ from twisted.internet import reactor, protocol
 
 from scrapy.extension import extensions
 from scrapy.exceptions import NotConfigured
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.spider import spiders
 from scrapy.stats import stats
 from scrapy.utils.signal import send_catch_log
@@ -48,8 +48,8 @@ class TelnetConsole(protocol.ServerFactory):
     def _get_telnet_vars(self):
         # Note: if you add entries here also update topics/telnetconsole.rst
         telnet_vars = {
-            'engine': scrapymanager.engine,
-            'manager': scrapymanager,
+            'engine': crawler.engine,
+            'manager': crawler,
             'extensions': extensions,
             'stats': stats,
             'spiders': spiders,

scrapy/tests/test_engine.py

@@ -8,7 +8,7 @@ from twisted.internet import reactor
 from twisted.web import server, resource, static, util
 
 from scrapy import signals
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy.tests import tests_datadir
 from scrapy.spider import BaseSpider
@@ -96,9 +96,9 @@ class CrawlingSession(object):
             dispatcher.connect(self.request_received, signals.request_received)
             dispatcher.connect(self.response_downloaded, signals.response_downloaded)
 
-            scrapymanager.configure()
-            scrapymanager.queue.append_spider(self.spider)
-            scrapymanager.start()
+            crawler.configure()
+            crawler.queue.append_spider(self.spider)
+            crawler.start()
             self.port.stopListening()
             self.wasrun = True
             # FIXME: extremly ugly hack to avoid propagating errors to other

scrapy/utils/engine.py

@@ -1,13 +1,11 @@
 """Some debugging functions for working with the Scrapy engine"""
 
-from time import time
-
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 
 def get_engine_status(engine=None):
     """Return a report of the current engine status"""
     if engine is None:
-        engine = scrapymanager.engine
+        engine = crawler.engine
 
     global_tests = [
         "time()-engine.start_time",

scrapy/utils/serialize.py

@@ -4,7 +4,7 @@ import decimal
 
 from twisted.internet import defer
 
-from scrapy.core.manager import scrapymanager
+from scrapy.project import crawler
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
 from scrapy.utils.py26 import json
@@ -21,7 +21,7 @@ class SpiderReferencer(object):
     spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$')
 
     def __init__(self, manager=None):
-        self.manager = manager or scrapymanager
+        self.manager = manager or crawler
 
     def get_reference_from_spider(self, spider):
         return 'spider:%x:%s' % (id(spider), spider.name)