diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index ad2fa75b2..8b9efa5a7 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -37,17 +37,32 @@ Available aliases in the telnet console The telnet console is like a regular Python shell running inside the Scrapy process, so you can do anything from it including imports, etc. -However, the telnet console comes with some default aliases defined for +However, the telnet console comes with some default shortcuts defined for convenience: - * ``engine``: the Scrapy engine object (``scrapy.core.engine.scrapyengine``) - * ``manager``: the Scrapy manager object (``scrapy.core.manager.scrapymanager``) - * ``extensions``: the extensions object (``scrapy.extension.extensions``) - * ``stats``: the Scrapy stats object (``scrapy.stats.stats``) - * ``settings``: the Scrapy settings object (``scrapy.conf.settings``) - * ``p``: the pprint function (``pprint.pprint``) - * ``prefs``: for memory debugging (see :ref:`topics-leaks`) - * ``hpy``: for memory debugging (see :ref:`topics-leaks`) ++----------------+-------------------------------------------------------------------+ +| Shortcut | Description | ++================+===================================================================+ +| ``engine`` | the Scrapy engine object (``scrapy.core.engine.scrapyengine``) | ++----------------+-------------------------------------------------------------------+ +| ``manager`` | the Scrapy manager object (``scrapy.core.manager.scrapymanager``) | ++----------------+-------------------------------------------------------------------+ +| ``extensions`` | the extensions object (``scrapy.extension.extensions``) | ++----------------+-------------------------------------------------------------------+ +| ``stats`` | the Scrapy stats object (``scrapy.stats.stats``) | ++----------------+-------------------------------------------------------------------+ +| ``settings`` | the Scrapy settings object (``scrapy.conf.settings``) | ++----------------+-------------------------------------------------------------------+ +| ``est`` | print a report of the current engine status | ++----------------+-------------------------------------------------------------------+ +| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) | ++----------------+-------------------------------------------------------------------+ +| ``p`` | a shortcut to the `pprint.pprint`_ function | ++----------------+-------------------------------------------------------------------+ +| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) | ++----------------+-------------------------------------------------------------------+ + +.. _pprint.pprint: http://docs.python.org/library/pprint.html#pprint.pprint Some example of using the telnet console ======================================== @@ -61,7 +76,7 @@ You can use the ``st()`` method of the Scrapy engine to quickly show its state using the telnet console:: telnet localhost 6023 - >>> engine.st() + >>> est() Execution engine status datetime.now()-self.start_time : 0:00:09.051588 diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 196e4d2dd..cb53e44e5 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -19,6 +19,7 @@ from scrapy.mail import MailSender from scrapy.conf import settings from scrapy.stats import stats from scrapy.utils.memory import get_vmvalue_from_procfs +from scrapy.utils.engine import get_engine_status class MemoryUsage(object): @@ -97,6 +98,6 @@ class MemoryUsage(object): s += "ENGINE STATUS ------------------------------------------------------- \r\n" s += "\r\n" - s += scrapyengine.getstatus() + s += get_engine_status() s += "\r\n" self.mail.send(rcpts, subject, s) diff --git a/scrapy/contrib/webconsole/enginestatus.py b/scrapy/contrib/webconsole/enginestatus.py index f34d3a375..623f777ef 100644 --- a/scrapy/contrib/webconsole/enginestatus.py +++ b/scrapy/contrib/webconsole/enginestatus.py @@ -2,7 +2,7 @@ Scheduler information module for Scrapy webconsole """ from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.engine import scrapyengine +from scrapy.utils.engine import get_engine_status from scrapy.management.web import banner class EngineStatus(object): @@ -16,7 +16,7 @@ class EngineStatus(object): def webconsole_render(self, wc_request): s = banner(self) s += "
\n"
-        s += scrapyengine.getstatus()
+        s += get_engine_status()
         s += "
\n" s += "\n" s += "\n" diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 70b33dc40..ba85f3283 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -286,56 +286,4 @@ class ExecutionEngine(object): log.msg("Domain closed (%s)" % reason, domain=domain) self._mainloop() - def getstatus(self): - """ - Return a report of the current engine status - """ - s = "Execution engine status\n\n" - - global_tests = [ - "datetime.utcnow()-self.start_time", - "self.is_idle()", - "self.scheduler.is_idle()", - "len(self.scheduler.pending_requests)", - "self.downloader.is_idle()", - "len(self.downloader.sites)", - "self.downloader.has_capacity()", - "self.scraper.is_idle()", - "len(self.scraper.sites)", - ] - domain_tests = [ - "self.domain_is_idle(domain)", - "self.closing.get(domain)", - "self.scheduler.domain_has_pending_requests(domain)", - "len(self.scheduler.pending_requests[domain])", - "len(self.downloader.sites[domain].queue)", - "len(self.downloader.sites[domain].active)", - "len(self.downloader.sites[domain].transferring)", - "self.downloader.sites[domain].closing", - "self.downloader.sites[domain].lastseen", - "len(self.scraper.sites[domain].queue)", - "len(self.scraper.sites[domain].active)", - "self.scraper.sites[domain].active_size", - "self.scraper.sites[domain].itemproc_size", - "self.scraper.sites[domain].needs_backout()", - ] - - for test in global_tests: - try: - s += "%-47s : %s\n" % (test, eval(test)) - except Exception, e: - s += "%-47s : %s (exception)\n" % (test, type(e).__name__) - s += "\n" - for domain in self.downloader.sites: - s += "%s\n" % domain - for test in domain_tests: - try: - s += " %-50s : %s\n" % (test, eval(test)) - except Exception, e: - s += " %-50s : %s (exception)\n" % (test, type(e).__name__) - return s - - def st(self): # shortcut for printing engine status (useful in telnet console) - print self.getstatus() - scrapyengine = ExecutionEngine() diff --git a/scrapy/management/telnet.py b/scrapy/management/telnet.py index 6283a50d7..816592f6c 100644 --- a/scrapy/management/telnet.py +++ b/scrapy/management/telnet.py @@ -16,6 +16,7 @@ from scrapy.core.engine import scrapyengine from scrapy.spider import spiders from scrapy.stats import stats from scrapy.utils.trackref import print_live_refs +from scrapy.utils.engine import print_engine_status from scrapy.conf import settings try: @@ -32,6 +33,7 @@ telnet_namespace = { 'stats': stats, 'spiders': spiders, 'settings': settings, + 'est': print_engine_status, 'p': pprint.pprint, 'prefs': print_live_refs, 'hpy': hpy, @@ -39,8 +41,7 @@ telnet_namespace = { def makeProtocol(): return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, - insults.ServerProtocol, - manhole.Manhole, telnet_namespace) + insults.ServerProtocol, manhole.Manhole, telnet_namespace) class TelnetConsole(protocol.ServerFactory): diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py new file mode 100644 index 000000000..697cf7f86 --- /dev/null +++ b/scrapy/utils/engine.py @@ -0,0 +1,57 @@ +"""Some debugging functions for working with the Scrapy engine""" + +from scrapy.core.engine import scrapyengine + +def get_engine_status(engine=None): + """Return a report of the current engine status""" + if engine is None: + engine = scrapyengine + + global_tests = [ + "datetime.utcnow()-engine.start_time", + "engine.is_idle()", + "engine.scheduler.is_idle()", + "len(engine.scheduler.pending_requests)", + "engine.downloader.is_idle()", + "len(engine.downloader.sites)", + "engine.downloader.has_capacity()", + "engine.scraper.is_idle()", + "len(engine.scraper.sites)", + ] + domain_tests = [ + "engine.domain_is_idle(domain)", + "engine.closing.get(domain)", + "engine.scheduler.domain_has_pending_requests(domain)", + "len(engine.scheduler.pending_requests[domain])", + "len(engine.downloader.sites[domain].queue)", + "len(engine.downloader.sites[domain].active)", + "len(engine.downloader.sites[domain].transferring)", + "engine.downloader.sites[domain].closing", + "engine.downloader.sites[domain].lastseen", + "len(engine.scraper.sites[domain].queue)", + "len(engine.scraper.sites[domain].active)", + "engine.scraper.sites[domain].active_size", + "engine.scraper.sites[domain].itemproc_size", + "engine.scraper.sites[domain].needs_backout()", + ] + + s = "Execution engine status\n\n" + + for test in global_tests: + try: + s += "%-47s : %s\n" % (test, eval(test)) + except Exception, e: + s += "%-47s : %s (exception)\n" % (test, type(e).__name__) + s += "\n" + for domain in engine.downloader.sites: + s += "%s\n" % domain + for test in domain_tests: + try: + s += " %-50s : %s\n" % (test, eval(test)) + except Exception, e: + s += " %-50s : %s (exception)\n" % (test, type(e).__name__) + return s + +def print_engine_status(engine=None): + print get_engine_status(engine) +