RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

moved engine.getstatus() method to scrapy.utils.engine function, to leave reporting logic out of engine code. added est() shortcut to telnet console

This commit is contained in:
Pablo Hoffman 2009-08-31 12:44:32 -03:00
parent dd80f6acdf
commit 8fab524978
6 changed files with 89 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
docs/topics/telnetconsole.rst
View File

@ -37,17 +37,32 @@ Available aliases in the telnet console
The telnet console is like a regular Python shell running inside the Scrapy
process, so you can do anything from it including imports, etc.
However, the telnet console comes with some default aliases defined for
However, the telnet console comes with some default shortcuts defined for
convenience:
* ``engine``: the Scrapy engine object (``scrapy.core.engine.scrapyengine``)
* ``manager``: the Scrapy manager object (``scrapy.core.manager.scrapymanager``)
* ``extensions``: the extensions object (``scrapy.extension.extensions``)
* ``stats``: the Scrapy stats object (``scrapy.stats.stats``)
* ``settings``: the Scrapy settings object (``scrapy.conf.settings``)
* ``p``: the pprint function (``pprint.pprint``)
* ``prefs``: for memory debugging (see :ref:`topics-leaks`)
* ``hpy``: for memory debugging (see :ref:`topics-leaks`)
+----------------+-------------------------------------------------------------------+
| Shortcut | Description |
+================+===================================================================+
| ``engine`` | the Scrapy engine object (``scrapy.core.engine.scrapyengine``) |
+----------------+-------------------------------------------------------------------+
| ``manager`` | the Scrapy manager object (``scrapy.core.manager.scrapymanager``) |
+----------------+-------------------------------------------------------------------+
| ``extensions`` | the extensions object (``scrapy.extension.extensions``) |
+----------------+-------------------------------------------------------------------+
| ``stats`` | the Scrapy stats object (``scrapy.stats.stats``) |
+----------------+-------------------------------------------------------------------+
| ``settings`` | the Scrapy settings object (``scrapy.conf.settings``) |
+----------------+-------------------------------------------------------------------+
| ``est`` | print a report of the current engine status |
+----------------+-------------------------------------------------------------------+
| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
| ``p`` | a shortcut to the `pprint.pprint`_ function |
+----------------+-------------------------------------------------------------------+
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
.. _pprint.pprint: http://docs.python.org/library/pprint.html#pprint.pprint
Some example of using the telnet console
========================================
@ -61,7 +76,7 @@ You can use the ``st()`` method of the Scrapy engine to quickly show its state
using the telnet console::
telnet localhost 6023
>>> engine.st()
>>> est()
Execution engine status
datetime.now()-self.start_time : 0:00:09.051588

3
scrapy/contrib/memusage.py
View File

@ -19,6 +19,7 @@ from scrapy.mail import MailSender
from scrapy.conf import settings
from scrapy.stats import stats
from scrapy.utils.memory import get_vmvalue_from_procfs
from scrapy.utils.engine import get_engine_status
class MemoryUsage(object):
@ -97,6 +98,6 @@ class MemoryUsage(object):
s += "ENGINE STATUS ------------------------------------------------------- \r\n"
s += "\r\n"
s += scrapyengine.getstatus()
s += get_engine_status()
s += "\r\n"
self.mail.send(rcpts, subject, s)

4
scrapy/contrib/webconsole/enginestatus.py
View File

@ -2,7 +2,7 @@
Scheduler information module for Scrapy webconsole
"""
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core.engine import scrapyengine
from scrapy.utils.engine import get_engine_status
from scrapy.management.web import banner
class EngineStatus(object):
@ -16,7 +16,7 @@ class EngineStatus(object):
def webconsole_render(self, wc_request):
s = banner(self)
s += "<pre><code>\n"
s += scrapyengine.getstatus()
s += get_engine_status()
s += "</pre></code>\n"
s += "</body>\n"
s += "</html>\n"

52
scrapy/core/engine.py
View File

@ -286,56 +286,4 @@ class ExecutionEngine(object):
log.msg("Domain closed (%s)" % reason, domain=domain)
self._mainloop()
def getstatus(self):
"""
Return a report of the current engine status
"""
s = "Execution engine status\n\n"
global_tests = [
"datetime.utcnow()-self.start_time",
"self.is_idle()",
"self.scheduler.is_idle()",
"len(self.scheduler.pending_requests)",
"self.downloader.is_idle()",
"len(self.downloader.sites)",
"self.downloader.has_capacity()",
"self.scraper.is_idle()",
"len(self.scraper.sites)",
]
domain_tests = [
"self.domain_is_idle(domain)",
"self.closing.get(domain)",
"self.scheduler.domain_has_pending_requests(domain)",
"len(self.scheduler.pending_requests[domain])",
"len(self.downloader.sites[domain].queue)",
"len(self.downloader.sites[domain].active)",
"len(self.downloader.sites[domain].transferring)",
"self.downloader.sites[domain].closing",
"self.downloader.sites[domain].lastseen",
"len(self.scraper.sites[domain].queue)",
"len(self.scraper.sites[domain].active)",
"self.scraper.sites[domain].active_size",
"self.scraper.sites[domain].itemproc_size",
"self.scraper.sites[domain].needs_backout()",
]
for test in global_tests:
try:
s += "%-47s : %s\n" % (test, eval(test))
except Exception, e:
s += "%-47s : %s (exception)\n" % (test, type(e).__name__)
s += "\n"
for domain in self.downloader.sites:
s += "%s\n" % domain
for test in domain_tests:
try:
s += " %-50s : %s\n" % (test, eval(test))
except Exception, e:
s += " %-50s : %s (exception)\n" % (test, type(e).__name__)
return s
def st(self): # shortcut for printing engine status (useful in telnet console)
print self.getstatus()
scrapyengine = ExecutionEngine()

5
scrapy/management/telnet.py
View File

@ -16,6 +16,7 @@ from scrapy.core.engine import scrapyengine
from scrapy.spider import spiders
from scrapy.stats import stats
from scrapy.utils.trackref import print_live_refs
from scrapy.utils.engine import print_engine_status
from scrapy.conf import settings
try:
@ -32,6 +33,7 @@ telnet_namespace = {
'stats': stats,
'spiders': spiders,
'settings': settings,
'est': print_engine_status,
'p': pprint.pprint,
'prefs': print_live_refs,
'hpy': hpy,
@ -39,8 +41,7 @@ telnet_namespace = {
def makeProtocol():
return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
insults.ServerProtocol,
manhole.Manhole, telnet_namespace)
insults.ServerProtocol, manhole.Manhole, telnet_namespace)
class TelnetConsole(protocol.ServerFactory):

57
scrapy/utils/engine.py Normal file
View File

@ -0,0 +1,57 @@
"""Some debugging functions for working with the Scrapy engine"""
from scrapy.core.engine import scrapyengine
def get_engine_status(engine=None):
"""Return a report of the current engine status"""
if engine is None:
engine = scrapyengine
global_tests = [
"datetime.utcnow()-engine.start_time",
"engine.is_idle()",
"engine.scheduler.is_idle()",
"len(engine.scheduler.pending_requests)",
"engine.downloader.is_idle()",
"len(engine.downloader.sites)",
"engine.downloader.has_capacity()",
"engine.scraper.is_idle()",
"len(engine.scraper.sites)",
]
domain_tests = [
"engine.domain_is_idle(domain)",
"engine.closing.get(domain)",
"engine.scheduler.domain_has_pending_requests(domain)",
"len(engine.scheduler.pending_requests[domain])",
"len(engine.downloader.sites[domain].queue)",
"len(engine.downloader.sites[domain].active)",
"len(engine.downloader.sites[domain].transferring)",
"engine.downloader.sites[domain].closing",
"engine.downloader.sites[domain].lastseen",
"len(engine.scraper.sites[domain].queue)",
"len(engine.scraper.sites[domain].active)",
"engine.scraper.sites[domain].active_size",
"engine.scraper.sites[domain].itemproc_size",
"engine.scraper.sites[domain].needs_backout()",
]
s = "Execution engine status\n\n"
for test in global_tests:
try:
s += "%-47s : %s\n" % (test, eval(test))
except Exception, e:
s += "%-47s : %s (exception)\n" % (test, type(e).__name__)
s += "\n"
for domain in engine.downloader.sites:
s += "%s\n" % domain
for test in domain_tests:
try:
s += " %-50s : %s\n" % (test, eval(test))
except Exception, e:
s += " %-50s : %s (exception)\n" % (test, type(e).__name__)
return s
def print_engine_status(engine=None):
print get_engine_status(engine)