From e65f7e0c91ccb16525bb318cb50339979387fdbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Sep 2018 10:49:46 -0300 Subject: [PATCH 1/9] Working POC for authenticating telnet console --- scrapy/extensions/telnet.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3024ddfaa..5e9fce7c5 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -22,6 +22,7 @@ from scrapy import signals from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp +from scrapy.utils.decorators import defers try: import guppy @@ -49,6 +50,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] + self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy') + self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy') self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -67,9 +70,25 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self): - telnet_vars = self._get_telnet_vars() - return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, - insults.ServerProtocol, manhole.Manhole, telnet_vars) + class Portal: + """An implementation of IPortal""" + @defers + def login(self_, credentials, mind, *interfaces): + if not (credentials.username == self.username + and credentials.checkPassword(self.password)): + raise ValueError("Invalid credentials") + + protocol = telnet.TelnetBootstrapProtocol( + insults.ServerProtocol, + manhole.Manhole, + self._get_telnet_vars() + ) + return (interfaces[0], protocol, lambda: None) + + return telnet.TelnetTransport( + telnet.AuthenticatingTelnetProtocol, + Portal() + ) def _get_telnet_vars(self): # Note: if you add entries here also update topics/telnetconsole.rst From eb64214c8a0053627e625debdb55373c8e17ef1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Sep 2018 10:54:40 -0300 Subject: [PATCH 2/9] Move telnetconsole settings defaults to scrapy defaults --- scrapy/extensions/telnet.py | 4 ++-- scrapy/settings/default_settings.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 5e9fce7c5..93342f225 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -50,8 +50,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy') - self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy') + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..2b7bc173c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { From 37cfb49805c86168af7a831fc33ec4aeb83e53da Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 24 Sep 2018 16:42:49 -0300 Subject: [PATCH 3/9] Randomly generate telnet credentials by default --- scrapy/extensions/telnet.py | 25 +++++++++--- scrapy/settings/default_settings.py | 2 - tests/test_extension_telnet.py | 59 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/test_extension_telnet.py diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 93342f225..3d0afeffb 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -7,6 +7,8 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging import traceback +import binascii +import os from twisted.internet import protocol try: @@ -50,8 +52,21 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] - self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] + + username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) + if username: + self.username = username.encode('utf8') + else: + self.username = binascii.hexlify(os.urandom(8)) + + password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) + if password: + self.password = password.encode('utf8') + else: + self.password = binascii.hexlify(os.urandom(8)) + + logger.info('Telnet Username: %s' % self.username) + logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -74,8 +89,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username - and credentials.checkPassword(self.password)): + if not (credentials.username == self.username and + credentials.checkPassword(self.password)): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( @@ -104,7 +119,7 @@ class TelnetConsole(protocol.ServerFactory): 'p': pprint.pprint, 'prefs': print_live_refs, 'hpy': hpy, - 'help': "This is Scrapy telnet console. For more info see: " \ + 'help': "This is Scrapy telnet console. For more info see: " "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b7bc173c..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,8 +277,6 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' -TELNETCONSOLE_USERNAME = 'scrapy' -TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py new file mode 100644 index 000000000..ffea1c463 --- /dev/null +++ b/tests/test_extension_telnet.py @@ -0,0 +1,59 @@ +try: + import unittest.mock as mock +except ImportError: + import mock + +from twisted.trial import unittest +from twisted.conch.telnet import ITelnetProtocol +from twisted.cred import credentials +from twisted.internet import defer + +from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.utils.test import get_crawler + + +class TelnetExtensionTest(unittest.TestCase): + def _get_console_and_portal(self, settings=None): + crawler = get_crawler(settings_dict=settings) + console = TelnetConsole(crawler) + username = console.username + password = console.password + + def _get_telnet_vars(): + # This function has some side effects we don't need for this test + return {} + console._get_telnet_vars = _get_telnet_vars + + console.start_listening() + protocol = console.protocol() + portal = protocol.protocolArgs[0] + + return console, portal + + @defer.inlineCallbacks + def test_bad_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(b'username', b'password') + d = portal.login(creds, None, ITelnetProtocol) + yield self.assertFailure(d, ValueError) + console.stop_listening() + + @defer.inlineCallbacks + def test_good_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(console.username, console.password) + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() + + @defer.inlineCallbacks + def test_custom_credentials(self): + settings = { + 'TELNETCONSOLE_USERNAME': 'user', + 'TELNETCONSOLE_PASSWORD': 'pass', + } + console, portal = self._get_console_and_portal(settings=settings) + creds = credentials.UsernamePassword(b'user', b'pass') + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() From e57a629efc0846ed396247baf22d7846689b82e4 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 11:54:57 -0300 Subject: [PATCH 4/9] Generate only password, encode username/password only on login --- scrapy/extensions/telnet.py | 22 ++++++++-------------- scrapy/settings/default_settings.py | 2 ++ tests/test_extension_telnet.py | 5 ++++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3d0afeffb..6df435cef 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -53,20 +53,14 @@ class TelnetConsole(protocol.ServerFactory): self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) - if username: - self.username = username.encode('utf8') - else: - self.username = binascii.hexlify(os.urandom(8)) + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] - password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) - if password: - self.password = password.encode('utf8') - else: - self.password = binascii.hexlify(os.urandom(8)) + if not self.password: + self.password = binascii.hexlify(os.urandom(8)).decode('utf8') + logger.info('Telnet Username: %s', self.username) + logger.info('Telnet Password: %s', self.password) - logger.info('Telnet Username: %s' % self.username) - logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -89,8 +83,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username and - credentials.checkPassword(self.password)): + if not (credentials.username == self.username.encode('utf8') and + credentials.checkPassword(self.password.encode('utf8'))): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..3734a0a58 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = None SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index ffea1c463..487c7c29f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -41,7 +41,10 @@ class TelnetExtensionTest(unittest.TestCase): @defer.inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() - creds = credentials.UsernamePassword(console.username, console.password) + creds = credentials.UsernamePassword( + console.username.encode('utf8'), + console.password.encode('utf8') + ) d = portal.login(creds, None, ITelnetProtocol) yield d console.stop_listening() From 5f9931d2ada7a2a05df77b1c061eeb482fcda347 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:07:04 -0300 Subject: [PATCH 5/9] do not log username --- scrapy/extensions/telnet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 6df435cef..a3d55f3c6 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -58,7 +58,6 @@ class TelnetConsole(protocol.ServerFactory): if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode('utf8') - logger.info('Telnet Username: %s', self.username) logger.info('Telnet Password: %s', self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) From 441e1e750fe7ad970adafc4c1f42834f7db86d1d Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:28:34 -0300 Subject: [PATCH 6/9] Style changes --- scrapy/extensions/telnet.py | 3 +-- tests/test_extension_telnet.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index a3d55f3c6..dcf73eb88 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -52,7 +52,6 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] @@ -113,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 487c7c29f..4f389e5cb 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -12,17 +12,15 @@ from scrapy.extensions.telnet import TelnetConsole, logger from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) username = console.username password = console.password - def _get_telnet_vars(): - # This function has some side effects we don't need for this test - return {} - console._get_telnet_vars = _get_telnet_vars + # This function has some side effects we don't need for this test + console._get_telnet_vars = lambda: {} console.start_listening() protocol = console.protocol() From 92b7955d75eba3ddad1e4815cb80cf60c7a9a7a9 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Tue, 16 Oct 2018 14:50:00 -0300 Subject: [PATCH 7/9] Add Telnet console authentication docs --- docs/topics/telnetconsole.rst | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index ce79c9f35..49c372598 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -26,8 +26,21 @@ The telnet console listens in the TCP port defined in the the console you need to type:: telnet localhost 6023 + Trying localhost... + Connected to localhost. + Escape character is '^]'. + Username: + Password: >>> - + +By default Username is ``scrapy`` and Password is autogenerated. The +autogenerated Password can be seen on scrapy logs like the example bellow:: + + 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 + +Default Username and Password can be overriden by the settings +:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD` + You need the telnet program which comes installed by default in Windows, and most Linux distros. @@ -160,3 +173,24 @@ Default: ``'127.0.0.1'`` The interface the telnet console should listen on + +.. setting:: TELNETCONSOLE_USERNAME + +TELNETCONSOLE_USERNAME +------------------ + +Default: ``'scrapy'`` + +The username used for the telnet console + + +.. setting:: TELNETCONSOLE_PASSWORD + +TELNETCONSOLE_PASSWORD +------------------ + +Default: ``None`` + +The password used for the telnet console, default behaviour is to have it +autogenerated + From 44f8e28b3c8608f65dbc7836b36bc231e38393b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 16 Oct 2018 19:53:20 -0300 Subject: [PATCH 8/9] Fix headings' underlines --- docs/topics/telnetconsole.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 49c372598..4db9cafb2 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -177,7 +177,7 @@ The interface the telnet console should listen on .. setting:: TELNETCONSOLE_USERNAME TELNETCONSOLE_USERNAME ------------------- +---------------------- Default: ``'scrapy'`` @@ -187,7 +187,7 @@ The username used for the telnet console .. setting:: TELNETCONSOLE_PASSWORD TELNETCONSOLE_PASSWORD ------------------- +---------------------- Default: ``None`` From 7c26701012c8e41a3e2c2644e05ce852d7472bc3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 01:33:58 +0500 Subject: [PATCH 9/9] DOC warn about telnet console being insecure --- docs/topics/telnetconsole.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 4db9cafb2..bf2ffa443 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -16,6 +16,17 @@ The telnet console is a :ref:`built-in Scrapy extension disable it if you want. For more information about the extension itself see :ref:`topics-extensions-ref-telnetconsole`. +.. warning:: + It is not secure to use telnet console via public networks, as telnet + doesn't provide any transport-layer security. Having username/password + authentication doesn't change that. + + Intended usage is connecting to a running Scrapy spider locally + (spider process and telnet client are on the same machine) + or over a secure connection (VPN, SSH tunnel). + Please avoid using telnet console over insecure connections, + or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option. + .. highlight:: none How to access the telnet console @@ -39,7 +50,12 @@ autogenerated Password can be seen on scrapy logs like the example bellow:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 Default Username and Password can be overriden by the settings -:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD` +:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. + +.. warning:: + Username and password provide only a limited protection, as telnet + is not using secure transport - by default traffic is not encrypted + even if username and password are set. You need the telnet program which comes installed by default in Windows, and most Linux distros.