diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index fb5220e9d..598826b4c 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -36,7 +36,7 @@ by a string: the full Python path to the extension's class name. For example:: EXTENSIONS = { 'scrapy.extensions.corestats.CoreStats': 500, - 'scrapy.telnet.TelnetConsole': 500, + 'scrapy.extensions.telnet.TelnetConsole': 500, } @@ -181,10 +181,10 @@ enabled (see :ref:`topics-stats`). Telnet console extension ~~~~~~~~~~~~~~~~~~~~~~~~ -.. module:: scrapy.telnet +.. module:: scrapy.extensions.telnet :synopsis: Telnet console -.. class:: scrapy.telnet.TelnetConsole +.. class:: scrapy.extensions.telnet.TelnetConsole Provides a telnet console for getting into a Python interpreter inside the currently running Scrapy process, which can be very useful for debugging. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 484065406..9f36ec34a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -565,7 +565,7 @@ Default:: { 'scrapy.extensions.corestats.CoreStats': 0, - 'scrapy.telnet.TelnetConsole': 0, + 'scrapy.extensions.telnet.TelnetConsole': 0, 'scrapy.extensions.memusage.MemoryUsage': 0, 'scrapy.extensions.memdebug.MemoryDebugger': 0, 'scrapy.extensions.closespider.CloseSpider': 0, diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index b1ff88771..ce79c9f35 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -4,7 +4,7 @@ Telnet Console ============== -.. module:: scrapy.telnet +.. module:: scrapy.extensions.telnet :synopsis: The Telnet Console Scrapy comes with a built-in telnet console for inspecting and controlling a diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py new file mode 100644 index 000000000..d9add1d97 --- /dev/null +++ b/scrapy/extensions/telnet.py @@ -0,0 +1,88 @@ +""" +Scrapy Telnet Console extension + +See documentation in docs/topics/telnetconsole.rst +""" + +import pprint +import logging + +from twisted.internet import protocol +try: + from twisted.conch import manhole, telnet + from twisted.conch.insults import insults + TWISTED_CONCH_AVAILABLE = True +except ImportError: + TWISTED_CONCH_AVAILABLE = False + +from scrapy.exceptions import NotConfigured +from scrapy import signals +from scrapy.utils.trackref import print_live_refs +from scrapy.utils.engine import print_engine_status +from scrapy.utils.reactor import listen_tcp + +try: + import guppy + hpy = guppy.hpy() +except ImportError: + hpy = None + +logger = logging.getLogger(__name__) + +# signal to update telnet variables +# args: telnet_vars +update_telnet_vars = object() + + +class TelnetConsole(protocol.ServerFactory): + + def __init__(self, crawler): + if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): + raise NotConfigured + if not TWISTED_CONCH_AVAILABLE: + raise NotConfigured + self.crawler = crawler + self.noisy = False + self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] + self.host = crawler.settings['TELNETCONSOLE_HOST'] + self.crawler.signals.connect(self.start_listening, signals.engine_started) + self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def start_listening(self): + self.port = listen_tcp(self.portrange, self.host, self) + h = self.port.getHost() + logger.debug("Telnet console listening on %(host)s:%(port)d", + {'host': h.host, 'port': h.port}, + extra={'crawler': self.crawler}) + + def stop_listening(self): + self.port.stopListening() + + def protocol(self): + telnet_vars = self._get_telnet_vars() + return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, + insults.ServerProtocol, manhole.Manhole, telnet_vars) + + def _get_telnet_vars(self): + # Note: if you add entries here also update topics/telnetconsole.rst + telnet_vars = { + 'engine': self.crawler.engine, + 'spider': self.crawler.engine.spider, + 'slot': self.crawler.engine.slot, + 'crawler': self.crawler, + 'extensions': self.crawler.extensions, + 'stats': self.crawler.stats, + 'settings': self.crawler.settings, + 'est': lambda: print_engine_status(self.crawler.engine), + 'p': pprint.pprint, + 'prefs': print_live_refs, + 'hpy': hpy, + 'help': "This is Scrapy telnet console. For more info see: " \ + "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", + } + self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) + return telnet_vars diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5d48ac1a7..8435b0354 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -120,7 +120,7 @@ EXTENSIONS = {} EXTENSIONS_BASE = { 'scrapy.extensions.corestats.CoreStats': 0, - 'scrapy.telnet.TelnetConsole': 0, + 'scrapy.extensions.telnet.TelnetConsole': 0, 'scrapy.extensions.memusage.MemoryUsage': 0, 'scrapy.extensions.memdebug.MemoryDebugger': 0, 'scrapy.extensions.closespider.CloseSpider': 0, diff --git a/scrapy/telnet.py b/scrapy/telnet.py index d9add1d97..70915f486 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -1,88 +1,7 @@ -""" -Scrapy Telnet Console extension +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.telnet` is deprecated, " + "use `scrapy.extensions.telnet` instead", + ScrapyDeprecationWarning, stacklevel=2) -See documentation in docs/topics/telnetconsole.rst -""" - -import pprint -import logging - -from twisted.internet import protocol -try: - from twisted.conch import manhole, telnet - from twisted.conch.insults import insults - TWISTED_CONCH_AVAILABLE = True -except ImportError: - TWISTED_CONCH_AVAILABLE = False - -from scrapy.exceptions import NotConfigured -from scrapy import signals -from scrapy.utils.trackref import print_live_refs -from scrapy.utils.engine import print_engine_status -from scrapy.utils.reactor import listen_tcp - -try: - import guppy - hpy = guppy.hpy() -except ImportError: - hpy = None - -logger = logging.getLogger(__name__) - -# signal to update telnet variables -# args: telnet_vars -update_telnet_vars = object() - - -class TelnetConsole(protocol.ServerFactory): - - def __init__(self, crawler): - if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): - raise NotConfigured - if not TWISTED_CONCH_AVAILABLE: - raise NotConfigured - self.crawler = crawler - self.noisy = False - self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] - self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.crawler.signals.connect(self.start_listening, signals.engine_started) - self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) - - @classmethod - def from_crawler(cls, crawler): - return cls(crawler) - - def start_listening(self): - self.port = listen_tcp(self.portrange, self.host, self) - h = self.port.getHost() - logger.debug("Telnet console listening on %(host)s:%(port)d", - {'host': h.host, 'port': h.port}, - extra={'crawler': self.crawler}) - - def stop_listening(self): - self.port.stopListening() - - def protocol(self): - telnet_vars = self._get_telnet_vars() - return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, - insults.ServerProtocol, manhole.Manhole, telnet_vars) - - def _get_telnet_vars(self): - # Note: if you add entries here also update topics/telnetconsole.rst - telnet_vars = { - 'engine': self.crawler.engine, - 'spider': self.crawler.engine.spider, - 'slot': self.crawler.engine.slot, - 'crawler': self.crawler, - 'extensions': self.crawler.extensions, - 'stats': self.crawler.stats, - 'settings': self.crawler.settings, - 'est': lambda: print_engine_status(self.crawler.engine), - 'p': pprint.pprint, - 'prefs': print_live_refs, - 'hpy': hpy, - 'help': "This is Scrapy telnet console. For more info see: " \ - "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", - } - self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) - return telnet_vars +from scrapy.extensions.telnet import * diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 2e660d7cf..822812c9a 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -56,7 +56,7 @@ NEWSPIDER_MODULE = '$project_name.spiders' # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { -# 'scrapy.telnet.TelnetConsole': None, +# 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 37e94fae9..0fc33e0c4 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -138,6 +138,7 @@ DEPRECATION_RULES = [ ('scrapy.command.', 'scrapy.commands.'), ('scrapy.dupefilter.', 'scrapy.dupefilters.'), ('scrapy.linkextractor.', 'scrapy.linkextractors.'), + ('scrapy.telnet.', 'scrapy.extensions.telnet.'), ('scrapy.spider.', 'scrapy.spiders.'), ('scrapy.squeue.', 'scrapy.squeues.'), ('scrapy.statscol.', 'scrapy.statscollectors.'),