From dff763c6830a09181a4c3859a20c849ec9213700 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 26 May 2010 10:29:32 -0300
Subject: [PATCH] Removed Scrapy engine singleton from
 scrapy.core.engine.scrapyengine. Now engine can only be accesed through
 Scrapy Manager 'engine' attribute - ie. scrapy.core.manager.engine.

---
 scrapy/contrib/closespider.py                    | 9 +++++----
 scrapy/contrib/downloadermiddleware/robotstxt.py | 4 ++--
 scrapy/contrib/itemsampler.py                    | 4 ++--
 scrapy/contrib/memusage.py                       | 1 -
 scrapy/contrib/pipeline/media.py                 | 4 ++--
 scrapy/contrib/spiderclosedelay.py               | 4 ++--
 scrapy/contrib/spidermiddleware/requestlimit.py  | 4 ++--
 scrapy/contrib/webconsole/livestats.py           | 6 +++---
 scrapy/contrib/webconsole/scheduler.py           | 4 ++--
 scrapy/contrib/webconsole/spiderctl.py           | 7 +++----
 scrapy/core/engine.py                            | 2 --
 scrapy/core/manager.py                           | 4 ++--
 scrapy/management/telnet.py                      | 3 +--
 scrapy/management/web.py                         | 4 ++--
 scrapy/shell.py                                  | 5 ++---
 scrapy/utils/engine.py                           | 4 ++--
 16 files changed, 32 insertions(+), 37 deletions(-)

diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py
index db2c0b3cd..66c89349c 100644
--- a/scrapy/contrib/closespider.py
+++ b/scrapy/contrib/closespider.py
@@ -10,7 +10,7 @@ from twisted.internet import reactor
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.conf import settings
 
 class CloseSpider(object):
@@ -29,13 +29,14 @@ class CloseSpider(object):
         dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
 
     def spider_opened(self, spider):
-        self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \
-            spider=spider, reason='closespider_timeout')
+        self.tasks[spider] = reactor.callLater(self.timeout, \
+            scrapymanager.engine.close_spider, spider=spider, \
+            reason='closespider_timeout')
         
     def item_passed(self, item, spider):
         self.counts[spider] += 1
         if self.counts[spider] == self.itempassed:
-            scrapyengine.close_spider(spider, 'closespider_itempassed')
+            scrapymanager.engine.close_spider(spider, 'closespider_itempassed')
 
     def spider_closed(self, spider):
         self.counts.pop(spider, None)
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
index 3784ff9d2..1f877708c 100644
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ b/scrapy/contrib/downloadermiddleware/robotstxt.py
@@ -9,7 +9,7 @@ import robotparser
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.core.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
@@ -42,7 +42,7 @@ class RobotsTxtMiddleware(object):
             self._parsers[netloc] = None
             robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
             robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
-            dfd = scrapyengine.download(robotsreq, spider)
+            dfd = scrapymanager.engine.download(robotsreq, spider)
             dfd.addCallback(self._parse_robots)
             self._spider_netlocs[spider].add(netloc)
         return self._parsers[netloc]
diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py
index 9f8666467..9098f09e1 100644
--- a/scrapy/contrib/itemsampler.py
+++ b/scrapy/contrib/itemsampler.py
@@ -27,7 +27,7 @@ import cPickle as pickle
 
 from scrapy.xlib.pydispatch import dispatcher
 
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.core.exceptions import NotConfigured
 from scrapy.core import signals
 from scrapy.utils.response import response_httprepr
@@ -60,7 +60,7 @@ class ItemSamplerPipeline(object):
             stats.set_value("items_sampled", sampled, spider=spider)
             log.msg("Sampled %s" % item, spider=spider, level=log.INFO)
             if close_spider and sampled == items_per_spider:
-                scrapyengine.close_spider(spider)
+                scrapymanager.engine.close_spider(spider)
         return item
 
     def engine_stopped(self):
diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
index cb53e44e5..0274753f2 100644
--- a/scrapy/contrib/memusage.py
+++ b/scrapy/contrib/memusage.py
@@ -13,7 +13,6 @@ from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core import signals
 from scrapy import log
 from scrapy.core.manager import scrapymanager
-from scrapy.core.engine import scrapyengine
 from scrapy.core.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.conf import settings
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
index 96c804be3..10debf84d 100644
--- a/scrapy/contrib/pipeline/media.py
+++ b/scrapy/contrib/pipeline/media.py
@@ -4,7 +4,7 @@ from twisted.internet.defer import Deferred, DeferredList
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy import log
 from scrapy.core import signals
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 
@@ -103,7 +103,7 @@ class MediaPipeline(object):
 
         """
         request.priority = self.DOWNLOAD_PRIORITY
-        return scrapyengine.download(request, info.spider)
+        return scrapymanager.engine.download(request, info.spider)
 
     def media_to_download(self, request, info):
         """ Ongoing request hook pre-cache
diff --git a/scrapy/contrib/spiderclosedelay.py b/scrapy/contrib/spiderclosedelay.py
index 3ecee8dfb..f8e92cfee 100644
--- a/scrapy/contrib/spiderclosedelay.py
+++ b/scrapy/contrib/spiderclosedelay.py
@@ -9,7 +9,7 @@ from scrapy.xlib.pydispatch import dispatcher
 from collections import defaultdict
 
 from scrapy.core import signals
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.core.exceptions import NotConfigured, DontCloseSpider
 from scrapy.conf import settings
 
@@ -26,7 +26,7 @@ class SpiderCloseDelay(object):
 
     def spider_idle(self, spider):
         try:
-            lastseen = scrapyengine.downloader.sites[spider].lastseen
+            lastseen = scrapymanager.engine.downloader.sites[spider].lastseen
         except KeyError:
             lastseen = None
         if not lastseen:
diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py
index 3d8516801..04138588d 100644
--- a/scrapy/contrib/spidermiddleware/requestlimit.py
+++ b/scrapy/contrib/spidermiddleware/requestlimit.py
@@ -7,7 +7,7 @@ from itertools import imap
 from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.core.exceptions import NotConfigured
 from scrapy.conf import settings
 from scrapy.http import Request
@@ -61,5 +61,5 @@ class RequestLimitMiddleware(object):
             return request_or_other
 
     def _pending_count(self, spider):
-        pending = scrapyengine.scheduler.pending_requests.get(spider, [])
+        pending = scrapymanager.engine.scheduler.pending_requests.get(spider, [])
         return len(pending)
diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py
index 2e61e8bc3..c5badee0e 100644
--- a/scrapy/contrib/webconsole/livestats.py
+++ b/scrapy/contrib/webconsole/livestats.py
@@ -4,7 +4,7 @@ Live statistics extension
 from datetime import datetime
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core import signals
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.management.web import banner, webconsole_discover_module
 
 class SpiderStats(object):
@@ -44,8 +44,8 @@ class LiveStats(object):
             self.domains[spider].crawled += 1
             
     def webconsole_render(self, wc_request):
-        sch = scrapyengine.scheduler
-        dwl = scrapyengine.downloader
+        sch = scrapymanager.engine.scheduler
+        dwl = scrapymanager.engine.downloader
 
         totdomains = totscraped = totcrawled = totscheduled = totactive = totdqueued = tottransf = 0
         s = banner(self)
diff --git a/scrapy/contrib/webconsole/scheduler.py b/scrapy/contrib/webconsole/scheduler.py
index d94db13ac..756bc3b6d 100644
--- a/scrapy/contrib/webconsole/scheduler.py
+++ b/scrapy/contrib/webconsole/scheduler.py
@@ -5,7 +5,7 @@ See documentation in docs/topics/extensions.rst
 """
 
 from scrapy.xlib.pydispatch import dispatcher
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.management.web import banner, webconsole_discover_module
 
 class SchedulerQueue(object):
@@ -21,7 +21,7 @@ class SchedulerQueue(object):
     def webconsole_render(self, wc_request):
         s = banner(self)
         s += "<ul>\n"
-        for domain, request_queue in scrapyengine.scheduler.pending_requests.iteritems():
+        for domain, request_queue in scrapymanager.engine.scheduler.pending_requests.iteritems():
             s += "<li>\n"
             s += "%s (<b>%s</b> requests)\n" % (domain, len(request_queue))
             s += "<ul>\n"
diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py
index a17c6a3d9..0c3f24176 100644
--- a/scrapy/contrib/webconsole/spiderctl.py
+++ b/scrapy/contrib/webconsole/spiderctl.py
@@ -6,7 +6,6 @@ from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy.core import signals
 from scrapy.core.manager import scrapymanager
-from scrapy.core.engine import scrapyengine
 from scrapy.spider import spiders
 from scrapy.management.web import banner, webconsole_discover_module
 from scrapy.conf import settings
@@ -117,7 +116,7 @@ class Spiderctl(object):
             stopped_spiders = []
             for name in args["stop_running_spiders"]:
                 if name in self.running:
-                    scrapyengine.close_spider(self.running[name])
+                    scrapymanager.engine.close_spider(self.running[name])
                     stopped_spiders.append(name)
             s += "Stopped spiders: <ul><li>%s</li></ul>" % "</li><li>".join(stopped_spiders)
             s += "</p>"
@@ -132,14 +131,14 @@ class Spiderctl(object):
                 s += "</p>"
         if "add_pending_spiders" in args:
             for name in args["add_pending_spiders"]:
-                if name not in scrapyengine.scheduler.pending_requests:
+                if name not in scrapymanager.engine.scheduler.pending_requests:
                     scrapymanager.queue.append_spider_name(name)
             s += "<p>"
             s += "Scheduled spiders: <ul><li>%s</li></ul>" % "</li><li>".join(args["add_pending_spiders"])
             s += "</p>"
         if "rerun_finished_spiders" in args:
             for name in args["rerun_finished_spiders"]:
-                if name not in scrapyengine.scheduler.pending_requests:
+                if name not in scrapymanager.engine.scheduler.pending_requests:
                     scrapymanager.queue.append_spider_name(name)
                 self.finished.remove(name)
             s += "<p>"
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 8bc90ccbe..11ce4b35a 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -286,5 +286,3 @@ class ExecutionEngine(object):
 
     def _finish_stopping_engine(self):
         send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
-
-scrapyengine = ExecutionEngine()
diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py
index d297ed0af..7bac4a3e8 100644
--- a/scrapy/core/manager.py
+++ b/scrapy/core/manager.py
@@ -2,7 +2,7 @@ import signal
 
 from twisted.internet import reactor, defer
 
-from scrapy.core.engine import scrapyengine
+from scrapy.core.engine import ExecutionEngine
 from scrapy.core.queue import ExecutionQueue
 from scrapy.extension import extensions
 from scrapy import log
@@ -15,7 +15,7 @@ class ExecutionManager(object):
     def __init__(self):
         self.configured = False
         self.control_reactor = True
-        self.engine = scrapyengine
+        self.engine = ExecutionEngine()
 
     def configure(self, control_reactor=True, queue=None):
         self.control_reactor = control_reactor
diff --git a/scrapy/management/telnet.py b/scrapy/management/telnet.py
index 9d0a234ea..ba1315bb8 100644
--- a/scrapy/management/telnet.py
+++ b/scrapy/management/telnet.py
@@ -13,7 +13,6 @@ from twisted.internet import reactor, protocol
 from scrapy.extension import extensions
 from scrapy.core.exceptions import NotConfigured
 from scrapy.core.manager import scrapymanager
-from scrapy.core.engine import scrapyengine
 from scrapy.spider import spiders
 from scrapy.stats import stats
 from scrapy.utils.trackref import print_live_refs
@@ -28,7 +27,7 @@ except ImportError:
 
 # if you add entries here also update topics/telnetconsole.rst
 telnet_namespace = {
-    'engine': scrapyengine,
+    'engine': scrapymanager.engine,
     'manager': scrapymanager,
     'extensions': extensions,
     'stats': stats,
diff --git a/scrapy/management/web.py b/scrapy/management/web.py
index e2036bf49..498613a5c 100644
--- a/scrapy/management/web.py
+++ b/scrapy/management/web.py
@@ -13,7 +13,7 @@ from twisted.web import server, resource
 
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core.exceptions import NotConfigured
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 from scrapy.conf import settings
 
 # web management signals
@@ -38,7 +38,7 @@ def banner(module=None):
     s += "<head><title>Scrapy</title></head>\n"
     s += "<body>\n"
     s += "<h1><a href='/'>Scrapy web console</a></h1>\n"
-    uptime = time() - scrapyengine.start_time
+    uptime = time() - scrapymanager.engine.start_time
     s += "<p>Bot: <b>%s</b> | Host: <b>%s</b> | Uptime: <b>%ds</b></p>\n" % \
         (settings['BOT_NAME'], socket.gethostname(), uptime)
     if module:
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 97af70869..4d75e8072 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -17,7 +17,6 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
 from scrapy.conf import settings
 from scrapy.core.manager import scrapymanager
-from scrapy.core.engine import scrapyengine
 from scrapy.core.queue import KeepAliveExecutionQueue
 from scrapy.http import Request, TextResponse
 
@@ -59,8 +58,8 @@ class Shell(object):
             log_multiple=True)
 
         print "Fetching %s..." % request
-        scrapyengine.open_spider(spider)
-        response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \
+        scrapymanager.engine.open_spider(spider)
+        response = threads.blockingCallFromThread(reactor, scrapymanager.engine.schedule, \
             request, spider)
         if response:
             self.populate_vars(url, response, request, spider)
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index e573730ab..6f37055a5 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -2,12 +2,12 @@
 
 from time import time
 
-from scrapy.core.engine import scrapyengine
+from scrapy.core.manager import scrapymanager
 
 def get_engine_status(engine=None):
     """Return a report of the current engine status"""
     if engine is None:
-        engine = scrapyengine
+        engine = scrapymanager.engine
 
     global_tests = [
         "time()-engine.start_time",