From 37cfb49805c86168af7a831fc33ec4aeb83e53da Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 24 Sep 2018 16:42:49 -0300 Subject: [PATCH 1/4] Randomly generate telnet credentials by default --- scrapy/extensions/telnet.py | 25 +++++++++--- scrapy/settings/default_settings.py | 2 - tests/test_extension_telnet.py | 59 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/test_extension_telnet.py diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 93342f225..3d0afeffb 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -7,6 +7,8 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging import traceback +import binascii +import os from twisted.internet import protocol try: @@ -50,8 +52,21 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] - self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] + + username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) + if username: + self.username = username.encode('utf8') + else: + self.username = binascii.hexlify(os.urandom(8)) + + password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) + if password: + self.password = password.encode('utf8') + else: + self.password = binascii.hexlify(os.urandom(8)) + + logger.info('Telnet Username: %s' % self.username) + logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -74,8 +89,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username - and credentials.checkPassword(self.password)): + if not (credentials.username == self.username and + credentials.checkPassword(self.password)): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( @@ -104,7 +119,7 @@ class TelnetConsole(protocol.ServerFactory): 'p': pprint.pprint, 'prefs': print_live_refs, 'hpy': hpy, - 'help': "This is Scrapy telnet console. For more info see: " \ + 'help': "This is Scrapy telnet console. For more info see: " "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b7bc173c..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,8 +277,6 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' -TELNETCONSOLE_USERNAME = 'scrapy' -TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py new file mode 100644 index 000000000..ffea1c463 --- /dev/null +++ b/tests/test_extension_telnet.py @@ -0,0 +1,59 @@ +try: + import unittest.mock as mock +except ImportError: + import mock + +from twisted.trial import unittest +from twisted.conch.telnet import ITelnetProtocol +from twisted.cred import credentials +from twisted.internet import defer + +from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.utils.test import get_crawler + + +class TelnetExtensionTest(unittest.TestCase): + def _get_console_and_portal(self, settings=None): + crawler = get_crawler(settings_dict=settings) + console = TelnetConsole(crawler) + username = console.username + password = console.password + + def _get_telnet_vars(): + # This function has some side effects we don't need for this test + return {} + console._get_telnet_vars = _get_telnet_vars + + console.start_listening() + protocol = console.protocol() + portal = protocol.protocolArgs[0] + + return console, portal + + @defer.inlineCallbacks + def test_bad_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(b'username', b'password') + d = portal.login(creds, None, ITelnetProtocol) + yield self.assertFailure(d, ValueError) + console.stop_listening() + + @defer.inlineCallbacks + def test_good_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(console.username, console.password) + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() + + @defer.inlineCallbacks + def test_custom_credentials(self): + settings = { + 'TELNETCONSOLE_USERNAME': 'user', + 'TELNETCONSOLE_PASSWORD': 'pass', + } + console, portal = self._get_console_and_portal(settings=settings) + creds = credentials.UsernamePassword(b'user', b'pass') + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() From e57a629efc0846ed396247baf22d7846689b82e4 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 11:54:57 -0300 Subject: [PATCH 2/4] Generate only password, encode username/password only on login --- scrapy/extensions/telnet.py | 22 ++++++++-------------- scrapy/settings/default_settings.py | 2 ++ tests/test_extension_telnet.py | 5 ++++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3d0afeffb..6df435cef 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -53,20 +53,14 @@ class TelnetConsole(protocol.ServerFactory): self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) - if username: - self.username = username.encode('utf8') - else: - self.username = binascii.hexlify(os.urandom(8)) + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] - password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) - if password: - self.password = password.encode('utf8') - else: - self.password = binascii.hexlify(os.urandom(8)) + if not self.password: + self.password = binascii.hexlify(os.urandom(8)).decode('utf8') + logger.info('Telnet Username: %s', self.username) + logger.info('Telnet Password: %s', self.password) - logger.info('Telnet Username: %s' % self.username) - logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -89,8 +83,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username and - credentials.checkPassword(self.password)): + if not (credentials.username == self.username.encode('utf8') and + credentials.checkPassword(self.password.encode('utf8'))): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..3734a0a58 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = None SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index ffea1c463..487c7c29f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -41,7 +41,10 @@ class TelnetExtensionTest(unittest.TestCase): @defer.inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() - creds = credentials.UsernamePassword(console.username, console.password) + creds = credentials.UsernamePassword( + console.username.encode('utf8'), + console.password.encode('utf8') + ) d = portal.login(creds, None, ITelnetProtocol) yield d console.stop_listening() From 5f9931d2ada7a2a05df77b1c061eeb482fcda347 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:07:04 -0300 Subject: [PATCH 3/4] do not log username --- scrapy/extensions/telnet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 6df435cef..a3d55f3c6 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -58,7 +58,6 @@ class TelnetConsole(protocol.ServerFactory): if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode('utf8') - logger.info('Telnet Username: %s', self.username) logger.info('Telnet Password: %s', self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) From 441e1e750fe7ad970adafc4c1f42834f7db86d1d Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:28:34 -0300 Subject: [PATCH 4/4] Style changes --- scrapy/extensions/telnet.py | 3 +-- tests/test_extension_telnet.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index a3d55f3c6..dcf73eb88 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -52,7 +52,6 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] @@ -113,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 487c7c29f..4f389e5cb 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -12,17 +12,15 @@ from scrapy.extensions.telnet import TelnetConsole, logger from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) username = console.username password = console.password - def _get_telnet_vars(): - # This function has some side effects we don't need for this test - return {} - console._get_telnet_vars = _get_telnet_vars + # This function has some side effects we don't need for this test + console._get_telnet_vars = lambda: {} console.start_listening() protocol = console.protocol()