From 5bac43676425d25169830d4410db60a98a11911f Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Fri, 17 Aug 2018 15:07:37 -0300 Subject: [PATCH 01/34] Make lazy loading Download Handlers optional --- scrapy/core/downloader/handlers/__init__.py | 14 ++++++++++-- scrapy/core/downloader/handlers/s3.py | 1 + tests/test_downloader_handlers.py | 25 ++++++++++++++++----- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index bc5cd742e..ebe6f5b78 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -24,6 +24,13 @@ class DownloadHandlers(object): crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath + for scheme in self._schemes: + path = self._schemes[scheme] + dhcls = load_object(path) + lazy = getattr(dhcls, 'lazy', False) + if lazy: + continue + self._load_handler(scheme, dhcls) crawler.signals.connect(self._close, signals.engine_stopped) @@ -40,8 +47,12 @@ class DownloadHandlers(object): return None path = self._schemes[scheme] + dhcls = load_object(path) + self._load_handler(scheme, dhcls) + return self._handlers[scheme] + + def _load_handler(self, scheme, dhcls): try: - dhcls = load_object(path) dh = dhcls(self._crawler.settings) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) @@ -54,7 +65,6 @@ class DownloadHandlers(object): return None else: self._handlers[scheme] = dh - return self._handlers[scheme] def download_request(self, request, spider): scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d8bbdd326..e723e616d 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -31,6 +31,7 @@ def _get_boto_connection(): class S3DownloadHandler(object): + lazy = True def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HTTPDownloadHandler, **kw): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2f8973054..116942ebe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -41,12 +41,20 @@ from scrapy.exceptions import NotConfigured from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider + class DummyDH(object): def __init__(self, crawler): pass +class DummyLazyDH(object): + lazy = True + + def __init__(self, crawler): + pass + + class OffDH(object): def __init__(self, crawler): @@ -60,8 +68,6 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) self.assertIn('scheme', dh._handlers) self.assertNotIn('scheme', dh._notconfigured) @@ -70,8 +76,6 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers - dh._get_handler(scheme) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) @@ -80,11 +84,22 @@ class LoadTestCase(unittest.TestCase): crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) dh = DownloadHandlers(crawler) self.assertNotIn('scheme', dh._schemes) - for scheme in handlers: # force load handlers + for scheme in handlers: # force load handlers dh._get_handler(scheme) self.assertNotIn('scheme', dh._handlers) self.assertIn('scheme', dh._notconfigured) + def test_lazy_handlers(self): + handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'} + crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers}) + dh = DownloadHandlers(crawler) + self.assertIn('scheme', dh._schemes) + self.assertNotIn('scheme', dh._handlers) + for scheme in handlers: # force load lazy handler + dh._get_handler(scheme) + self.assertIn('scheme', dh._handlers) + self.assertNotIn('scheme', dh._notconfigured) + class FileTestCase(unittest.TestCase): From 167211ffb0e7fa7756483aa9f20e1ab5589c7c4a Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 20 Aug 2018 15:54:04 -0300 Subject: [PATCH 02/34] Default is lazy, load_object exception handling, code improvements --- scrapy/core/downloader/handlers/__init__.py | 22 +++++++++------------ scrapy/core/downloader/handlers/datauri.py | 2 ++ scrapy/core/downloader/handlers/file.py | 2 ++ scrapy/core/downloader/handlers/ftp.py | 3 +++ scrapy/core/downloader/handlers/http10.py | 1 + scrapy/core/downloader/handlers/http11.py | 1 + scrapy/core/downloader/handlers/s3.py | 1 - tests/test_downloader_handlers.py | 4 +++- 8 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ebe6f5b78..0b55d32fa 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -24,13 +24,7 @@ class DownloadHandlers(object): crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) for scheme, clspath in six.iteritems(handlers): self._schemes[scheme] = clspath - for scheme in self._schemes: - path = self._schemes[scheme] - dhcls = load_object(path) - lazy = getattr(dhcls, 'lazy', False) - if lazy: - continue - self._load_handler(scheme, dhcls) + self._load_handler(scheme, skip_lazy=True) crawler.signals.connect(self._close, signals.engine_stopped) @@ -46,13 +40,14 @@ class DownloadHandlers(object): self._notconfigured[scheme] = 'no handler available for that scheme' return None - path = self._schemes[scheme] - dhcls = load_object(path) - self._load_handler(scheme, dhcls) - return self._handlers[scheme] + return self._load_handler(scheme) - def _load_handler(self, scheme, dhcls): + def _load_handler(self, scheme, skip_lazy=False): + path = self._schemes[scheme] try: + dhcls = load_object(path) + if skip_lazy and getattr(dhcls, 'lazy', True): + return None dh = dhcls(self._crawler.settings) except NotConfigured as ex: self._notconfigured[scheme] = str(ex) @@ -60,11 +55,12 @@ class DownloadHandlers(object): except Exception as ex: logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"', {"clspath": path, "scheme": scheme}, - exc_info=True, extra={'crawler': self._crawler}) + exc_info=True, extra={'crawler': self._crawler}) self._notconfigured[scheme] = str(ex) return None else: self._handlers[scheme] = dh + return dh def download_request(self, request, spider): scheme = urlparse_cached(request).scheme diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index d102f2b73..ad25beb3b 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -6,6 +6,8 @@ from scrapy.utils.decorators import defers class DataURIDownloadHandler(object): + lazy = False + def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 9346ce08d..23f25d28d 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -2,7 +2,9 @@ from w3lib.url import file_uri_to_path from scrapy.responsetypes import responsetypes from scrapy.utils.decorators import defers + class FileDownloadHandler(object): + lazy = False def __init__(self, settings): pass diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 933bc7e8d..c342d4ab1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -60,7 +60,10 @@ class ReceivedDataProtocol(Protocol): self.body.close() if self.filename else self.body.seek(0) _CODE_RE = re.compile("\d+") + + class FTPDownloadHandler(object): + lazy = False CODE_MAPPING = { "550": 404, diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 0322bbe49..d875fb1e4 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -6,6 +6,7 @@ from scrapy.utils.python import to_unicode class HTTP10DownloadHandler(object): + lazy = False def __init__(self, settings): self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 038db7b47..0673188a1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -33,6 +33,7 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler(object): + lazy = False def __init__(self, settings): self._pool = HTTPConnectionPool(reactor, persistent=True) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index e723e616d..d8bbdd326 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -31,7 +31,6 @@ def _get_boto_connection(): class S3DownloadHandler(object): - lazy = True def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HTTPDownloadHandler, **kw): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 116942ebe..0d0829793 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -43,19 +43,21 @@ from tests.spiders import SingleRequestSpider class DummyDH(object): + lazy = False def __init__(self, crawler): pass class DummyLazyDH(object): - lazy = True + # Default is lazy for backwards compatibility def __init__(self, crawler): pass class OffDH(object): + lazy = False def __init__(self, crawler): raise NotConfigured From e65f7e0c91ccb16525bb318cb50339979387fdbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Sep 2018 10:49:46 -0300 Subject: [PATCH 03/34] Working POC for authenticating telnet console --- scrapy/extensions/telnet.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3024ddfaa..5e9fce7c5 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -22,6 +22,7 @@ from scrapy import signals from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp +from scrapy.utils.decorators import defers try: import guppy @@ -49,6 +50,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] + self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy') + self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy') self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -67,9 +70,25 @@ class TelnetConsole(protocol.ServerFactory): self.port.stopListening() def protocol(self): - telnet_vars = self._get_telnet_vars() - return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, - insults.ServerProtocol, manhole.Manhole, telnet_vars) + class Portal: + """An implementation of IPortal""" + @defers + def login(self_, credentials, mind, *interfaces): + if not (credentials.username == self.username + and credentials.checkPassword(self.password)): + raise ValueError("Invalid credentials") + + protocol = telnet.TelnetBootstrapProtocol( + insults.ServerProtocol, + manhole.Manhole, + self._get_telnet_vars() + ) + return (interfaces[0], protocol, lambda: None) + + return telnet.TelnetTransport( + telnet.AuthenticatingTelnetProtocol, + Portal() + ) def _get_telnet_vars(self): # Note: if you add entries here also update topics/telnetconsole.rst From eb64214c8a0053627e625debdb55373c8e17ef1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Sep 2018 10:54:40 -0300 Subject: [PATCH 04/34] Move telnetconsole settings defaults to scrapy defaults --- scrapy/extensions/telnet.py | 4 ++-- scrapy/settings/default_settings.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 5e9fce7c5..93342f225 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -50,8 +50,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings.get('TELNETCONSOLE_USERNAME', 'scrapy') - self.password = crawler.settings.get('TELNETCONSOLE_PASSWORD', 'scrapy') + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..2b7bc173c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { From 37cfb49805c86168af7a831fc33ec4aeb83e53da Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Mon, 24 Sep 2018 16:42:49 -0300 Subject: [PATCH 05/34] Randomly generate telnet credentials by default --- scrapy/extensions/telnet.py | 25 +++++++++--- scrapy/settings/default_settings.py | 2 - tests/test_extension_telnet.py | 59 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/test_extension_telnet.py diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 93342f225..3d0afeffb 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -7,6 +7,8 @@ See documentation in docs/topics/telnetconsole.rst import pprint import logging import traceback +import binascii +import os from twisted.internet import protocol try: @@ -50,8 +52,21 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] - self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] + + username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) + if username: + self.username = username.encode('utf8') + else: + self.username = binascii.hexlify(os.urandom(8)) + + password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) + if password: + self.password = password.encode('utf8') + else: + self.password = binascii.hexlify(os.urandom(8)) + + logger.info('Telnet Username: %s' % self.username) + logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -74,8 +89,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username - and credentials.checkPassword(self.password)): + if not (credentials.username == self.username and + credentials.checkPassword(self.password)): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( @@ -104,7 +119,7 @@ class TelnetConsole(protocol.ServerFactory): 'p': pprint.pprint, 'prefs': print_live_refs, 'hpy': hpy, - 'help': "This is Scrapy telnet console. For more info see: " \ + 'help': "This is Scrapy telnet console. For more info see: " "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2b7bc173c..ca004aedd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,8 +277,6 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' -TELNETCONSOLE_USERNAME = 'scrapy' -TELNETCONSOLE_PASSWORD = 'scrapy' SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py new file mode 100644 index 000000000..ffea1c463 --- /dev/null +++ b/tests/test_extension_telnet.py @@ -0,0 +1,59 @@ +try: + import unittest.mock as mock +except ImportError: + import mock + +from twisted.trial import unittest +from twisted.conch.telnet import ITelnetProtocol +from twisted.cred import credentials +from twisted.internet import defer + +from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.utils.test import get_crawler + + +class TelnetExtensionTest(unittest.TestCase): + def _get_console_and_portal(self, settings=None): + crawler = get_crawler(settings_dict=settings) + console = TelnetConsole(crawler) + username = console.username + password = console.password + + def _get_telnet_vars(): + # This function has some side effects we don't need for this test + return {} + console._get_telnet_vars = _get_telnet_vars + + console.start_listening() + protocol = console.protocol() + portal = protocol.protocolArgs[0] + + return console, portal + + @defer.inlineCallbacks + def test_bad_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(b'username', b'password') + d = portal.login(creds, None, ITelnetProtocol) + yield self.assertFailure(d, ValueError) + console.stop_listening() + + @defer.inlineCallbacks + def test_good_credentials(self): + console, portal = self._get_console_and_portal() + creds = credentials.UsernamePassword(console.username, console.password) + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() + + @defer.inlineCallbacks + def test_custom_credentials(self): + settings = { + 'TELNETCONSOLE_USERNAME': 'user', + 'TELNETCONSOLE_PASSWORD': 'pass', + } + console, portal = self._get_console_and_portal(settings=settings) + creds = credentials.UsernamePassword(b'user', b'pass') + d = portal.login(creds, None, ITelnetProtocol) + yield d + console.stop_listening() From e57a629efc0846ed396247baf22d7846689b82e4 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 11:54:57 -0300 Subject: [PATCH 06/34] Generate only password, encode username/password only on login --- scrapy/extensions/telnet.py | 22 ++++++++-------------- scrapy/settings/default_settings.py | 2 ++ tests/test_extension_telnet.py | 5 ++++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 3d0afeffb..6df435cef 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -53,20 +53,14 @@ class TelnetConsole(protocol.ServerFactory): self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - username = crawler.settings.get('TELNETCONSOLE_USERNAME', None) - if username: - self.username = username.encode('utf8') - else: - self.username = binascii.hexlify(os.urandom(8)) + self.username = crawler.settings['TELNETCONSOLE_USERNAME'] + self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] - password = crawler.settings.get('TELNETCONSOLE_PASSWORD', None) - if password: - self.password = password.encode('utf8') - else: - self.password = binascii.hexlify(os.urandom(8)) + if not self.password: + self.password = binascii.hexlify(os.urandom(8)).decode('utf8') + logger.info('Telnet Username: %s', self.username) + logger.info('Telnet Password: %s', self.password) - logger.info('Telnet Username: %s' % self.username) - logger.info('Telnet Password: %s' % self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @@ -89,8 +83,8 @@ class TelnetConsole(protocol.ServerFactory): """An implementation of IPortal""" @defers def login(self_, credentials, mind, *interfaces): - if not (credentials.username == self.username and - credentials.checkPassword(self.password)): + if not (credentials.username == self.username.encode('utf8') and + credentials.checkPassword(self.password.encode('utf8'))): raise ValueError("Invalid credentials") protocol = telnet.TelnetBootstrapProtocol( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..3734a0a58 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -277,6 +277,8 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] TELNETCONSOLE_HOST = '127.0.0.1' +TELNETCONSOLE_USERNAME = 'scrapy' +TELNETCONSOLE_PASSWORD = None SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index ffea1c463..487c7c29f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -41,7 +41,10 @@ class TelnetExtensionTest(unittest.TestCase): @defer.inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() - creds = credentials.UsernamePassword(console.username, console.password) + creds = credentials.UsernamePassword( + console.username.encode('utf8'), + console.password.encode('utf8') + ) d = portal.login(creds, None, ITelnetProtocol) yield d console.stop_listening() From 5f9931d2ada7a2a05df77b1c061eeb482fcda347 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:07:04 -0300 Subject: [PATCH 07/34] do not log username --- scrapy/extensions/telnet.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 6df435cef..a3d55f3c6 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -58,7 +58,6 @@ class TelnetConsole(protocol.ServerFactory): if not self.password: self.password = binascii.hexlify(os.urandom(8)).decode('utf8') - logger.info('Telnet Username: %s', self.username) logger.info('Telnet Password: %s', self.password) self.crawler.signals.connect(self.start_listening, signals.engine_started) From 441e1e750fe7ad970adafc4c1f42834f7db86d1d Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Wed, 26 Sep 2018 13:28:34 -0300 Subject: [PATCH 08/34] Style changes --- scrapy/extensions/telnet.py | 3 +-- tests/test_extension_telnet.py | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index a3d55f3c6..dcf73eb88 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -52,7 +52,6 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')] self.host = crawler.settings['TELNETCONSOLE_HOST'] - self.username = crawler.settings['TELNETCONSOLE_USERNAME'] self.password = crawler.settings['TELNETCONSOLE_PASSWORD'] @@ -113,7 +112,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " - "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 487c7c29f..4f389e5cb 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -12,17 +12,15 @@ from scrapy.extensions.telnet import TelnetConsole, logger from scrapy.utils.test import get_crawler -class TelnetExtensionTest(unittest.TestCase): +class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) username = console.username password = console.password - def _get_telnet_vars(): - # This function has some side effects we don't need for this test - return {} - console._get_telnet_vars = _get_telnet_vars + # This function has some side effects we don't need for this test + console._get_telnet_vars = lambda: {} console.start_listening() protocol = console.protocol() From 92b7955d75eba3ddad1e4815cb80cf60c7a9a7a9 Mon Sep 17 00:00:00 2001 From: Henrique Coura Date: Tue, 16 Oct 2018 14:50:00 -0300 Subject: [PATCH 09/34] Add Telnet console authentication docs --- docs/topics/telnetconsole.rst | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index ce79c9f35..49c372598 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -26,8 +26,21 @@ The telnet console listens in the TCP port defined in the the console you need to type:: telnet localhost 6023 + Trying localhost... + Connected to localhost. + Escape character is '^]'. + Username: + Password: >>> - + +By default Username is ``scrapy`` and Password is autogenerated. The +autogenerated Password can be seen on scrapy logs like the example bellow:: + + 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 + +Default Username and Password can be overriden by the settings +:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD` + You need the telnet program which comes installed by default in Windows, and most Linux distros. @@ -160,3 +173,24 @@ Default: ``'127.0.0.1'`` The interface the telnet console should listen on + +.. setting:: TELNETCONSOLE_USERNAME + +TELNETCONSOLE_USERNAME +------------------ + +Default: ``'scrapy'`` + +The username used for the telnet console + + +.. setting:: TELNETCONSOLE_PASSWORD + +TELNETCONSOLE_PASSWORD +------------------ + +Default: ``None`` + +The password used for the telnet console, default behaviour is to have it +autogenerated + From 44f8e28b3c8608f65dbc7836b36bc231e38393b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 16 Oct 2018 19:53:20 -0300 Subject: [PATCH 10/34] Fix headings' underlines --- docs/topics/telnetconsole.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 49c372598..4db9cafb2 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -177,7 +177,7 @@ The interface the telnet console should listen on .. setting:: TELNETCONSOLE_USERNAME TELNETCONSOLE_USERNAME ------------------- +---------------------- Default: ``'scrapy'`` @@ -187,7 +187,7 @@ The username used for the telnet console .. setting:: TELNETCONSOLE_PASSWORD TELNETCONSOLE_PASSWORD ------------------- +---------------------- Default: ``None`` From f97e3e90f25c5077b47d9ec11a4cf84ea777227e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 29 Oct 2018 12:40:20 -0300 Subject: [PATCH 11/34] Use collections.deque instead of list to store methods --- scrapy/core/downloader/middleware.py | 4 ++-- scrapy/core/spidermw.py | 6 +++--- scrapy/middleware.py | 6 +++--- tests/test_middleware.py | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index c3b23e284..f5e2fca63 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_request'): self.methods['process_request'].append(mw.process_request) if hasattr(mw, 'process_response'): - self.methods['process_response'].insert(0, mw.process_response) + self.methods['process_response'].appendleft(mw.process_response) if hasattr(mw, 'process_exception'): - self.methods['process_exception'].insert(0, mw.process_exception) + self.methods['process_exception'].appendleft(mw.process_exception) def download(self, download_func, request, spider): @defer.inlineCallbacks diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a206e4b0c..16b8435ab 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_spider_output'): - self.methods['process_spider_output'].insert(0, mw.process_spider_output) + self.methods['process_spider_output'].appendleft(mw.process_spider_output) if hasattr(mw, 'process_spider_exception'): - self.methods['process_spider_exception'].insert(0, mw.process_spider_exception) + self.methods['process_spider_exception'].appendleft(mw.process_spider_exception) if hasattr(mw, 'process_start_requests'): - self.methods['process_start_requests'].insert(0, mw.process_start_requests) + self.methods['process_start_requests'].appendleft(mw.process_start_requests) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f2240984c..1cfd8a782 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,4 +1,4 @@ -from collections import defaultdict +from collections import defaultdict, deque import logging import pprint @@ -16,7 +16,7 @@ class MiddlewareManager(object): def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(list) + self.methods = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -56,7 +56,7 @@ class MiddlewareManager(object): if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): - self.methods['close_spider'].insert(0, mw.close_spider) + self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname, obj, *args): return process_parallel(self.methods[methodname], obj, *args) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6d885330..aea0be825 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase): def test_init(self): m1, m2, m3 = M1(), M2(), M3() mwman = TestMiddlewareManager(m1, m2, m3) - self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider]) - self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider]) - self.assertEqual(mwman.methods['process'], [m1.process, m3.process]) + self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider]) + self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider]) + self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process]) def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) From 491929c212999aa816e561aeed19a902664d01e5 Mon Sep 17 00:00:00 2001 From: Todd Date: Fri, 16 Nov 2018 13:38:19 -0500 Subject: [PATCH 12/34] Include additional files in sdists In particular this includes files needed for running the tests, as well as the changelog. --- MANIFEST.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 94de4f3bf..ae7db51fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,13 +3,24 @@ include AUTHORS include INSTALL include LICENSE include MANIFEST.in +include NEWS + include scrapy/VERSION include scrapy/mime.types + +include codecov.yml +include conftest.py +include pytest.ini +include requirements-*.txt +include tox.ini + recursive-include scrapy/templates * recursive-include scrapy license.txt recursive-include docs * prune docs/build + recursive-include extras * recursive-include bin * recursive-include tests * + global-exclude __pycache__ *.py[cod] From 127bf499f1d6b4a924d87e39ff89b528586c78c7 Mon Sep 17 00:00:00 2001 From: Frederik Elwert Date: Fri, 16 Nov 2018 22:15:03 +0100 Subject: [PATCH 13/34] Add documentation to `scrapy shell` command. The special syntax required for local files (`./file.html`) is not documented as part of the `scrapy shell --help` output. This patch adds that. --- scrapy/commands/shell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 40a58d94a..e05084272 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -28,7 +28,8 @@ class Command(ScrapyCommand): return "Interactive scraping console" def long_desc(self): - return "Interactive console for scraping the given url" + return ("Interactive console for scraping the given url or file. " + "Use ./file.html syntax or full path for local file.") def add_options(self, parser): ScrapyCommand.add_options(self, parser) From 274b65dff4dc8b8300d872171679f173fbe0a746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 16:36:05 +0100 Subject: [PATCH 14/34] Add a troubleshooting section to the installation instructions Its initial content covers the workaround for #2473. --- docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4a9aa3cfb..daec7fcb7 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. -For more detailed and platform specifics instructions, read on. +For more detailed and platform specifics instructions, as well as +troubleshooting information, read on. Things that are good to know @@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency. To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +.. _intro-install-troubleshooting: + +Troubleshooting +=============== + +AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' +---------------------------------------------------------------- + +After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an +exception with the following traceback:: + + […] + File "[…]/site-packages/twisted/protocols/tls.py", line 63, in + from twisted.internet._sslverify import _setAcceptableProtocols + File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in + TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1, + AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' + +The reason you get this exception is that your system or virtual environment +has a version of pyOpenSSL that your version of Twisted does not support. + +To install a version of pyOpenSSL that your version of Twisted supports, +reinstall Twisted with the :code:`tls` extra option:: + + pip install twisted[tls] + +For details, see `Issue #2473 `_. + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ From 62f3349c1aee54599ab7ee8755d2b31090639105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 17:14:10 +0100 Subject: [PATCH 15/34] Document the SCRAPY_PROJECT environment variable Fixes #1109 --- docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ef9c45196..97f8311de 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`) -* ``SCRAPY_PROJECT`` +* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`) * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) .. _topics-project-structure: @@ -71,6 +71,33 @@ the project settings. Here is an example:: [settings] default = myproject.settings +.. _topics-project-envvar: + +Sharing the root directory between projects +=========================================== + +A project root directory, the one that contains the ``scrapy.cfg``, may be +shared by multiple Scrapy projects, each with its own settings module. + +In that case, you must define one or more aliases for those settings modules +under ``[settings]`` in your ``scrapy.cfg`` file:: + + [settings] + default = myproject1.settings + project1 = myproject1.settings + project2 = myproject2.settings + +By default, the ``scrapy`` command-line tool will use the ``default`` settings. +Use the ``SCRAPY_PROJECT`` environment variable to specify a different project +for ``scrapy`` to use:: + + $ scrapy settings --get BOT_NAME + Project 1 Bot + $ export SCRAPY_PROJECT=project2 + $ scrapy settings --get BOT_NAME + Project 2 Bot + + Using the ``scrapy`` tool ========================= From d7c8eee2fc918d07feb708c41089da21b2b9aea5 Mon Sep 17 00:00:00 2001 From: fpghost Date: Tue, 4 Dec 2018 10:57:51 +0100 Subject: [PATCH 16/34] the strip() isnt needed --- scrapy/downloadermiddlewares/httpproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 1dd47359f..2c35d1b90 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -30,7 +30,7 @@ class HttpProxyMiddleware(object): user_pass = to_bytes( '%s:%s' % (unquote(username), unquote(password)), encoding=self.auth_encoding) - return base64.b64encode(user_pass).strip() + return base64.b64encode(user_pass) def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) From 4d48759978ac2405bc2cb30f84af948693e4cad3 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 10 Dec 2018 14:44:15 +0800 Subject: [PATCH 17/34] remove "sudo: false" now that travis no longer supports it https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4218d13bf..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: python -sudo: false branches: only: - master From cd9d8e28cdf49ff63e1b3f9126e6651fcd77e0fa Mon Sep 17 00:00:00 2001 From: hsiao yi Date: Tue, 11 Dec 2018 19:21:07 +0800 Subject: [PATCH 18/34] unify the quote style --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 9d7c94d39..8b2fef065 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -26,7 +26,7 @@ http://quotes.toscrape.com, following the pagination:: class QuotesSpider(scrapy.Spider): - name = "quotes" + name = 'quotes' start_urls = [ 'http://quotes.toscrape.com/tag/humor/', ] From f6dfc5f3dd56b7c823e3f53f7f9f63515ca7c3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:23:23 -0300 Subject: [PATCH 19/34] Fix boto import error under Jessie testing environment --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 4218d13bf..252c783d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,11 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [ "$TOXENV" = "jessie" ]; then + # Not used directly but allows boto GCE plugins to load. + # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + pip install google-compute-engine + fi - pip install -U tox twine wheel codecov script: tox From 8ed6beb7f9199e8924cd03bd34a46194c3d82e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:39:29 -0300 Subject: [PATCH 20/34] Needs to be installed within tox env --- .travis.yml | 5 ----- tox.ini | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 252c783d7..4218d13bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,11 +43,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [ "$TOXENV" = "jessie" ]; then - # Not used directly but allows boto GCE plugins to load. - # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 - pip install google-compute-engine - fi - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e5543fe2a..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -51,6 +51,9 @@ deps = cssselect==0.9.1 zope.interface==4.1.1 -rtests/requirements-py2.txt +# Not used directly but allows boto GCE plugins to load. +# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + google-compute-engine==2.8.12 [testenv:trunk] basepython = python2.7 From 7c26701012c8e41a3e2c2644e05ce852d7472bc3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 01:33:58 +0500 Subject: [PATCH 21/34] DOC warn about telnet console being insecure --- docs/topics/telnetconsole.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 4db9cafb2..bf2ffa443 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -16,6 +16,17 @@ The telnet console is a :ref:`built-in Scrapy extension disable it if you want. For more information about the extension itself see :ref:`topics-extensions-ref-telnetconsole`. +.. warning:: + It is not secure to use telnet console via public networks, as telnet + doesn't provide any transport-layer security. Having username/password + authentication doesn't change that. + + Intended usage is connecting to a running Scrapy spider locally + (spider process and telnet client are on the same machine) + or over a secure connection (VPN, SSH tunnel). + Please avoid using telnet console over insecure connections, + or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option. + .. highlight:: none How to access the telnet console @@ -39,7 +50,12 @@ autogenerated Password can be seen on scrapy logs like the example bellow:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 Default Username and Password can be overriden by the settings -:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD` +:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. + +.. warning:: + Username and password provide only a limited protection, as telnet + is not using secure transport - by default traffic is not encrypted + even if username and password are set. You need the telnet program which comes installed by default in Windows, and most Linux distros. From cdd04dfb1d9a2e6fd8c188dccae26bbdd3454ebd Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 13:13:49 +0500 Subject: [PATCH 22/34] declare Python 3.7 support in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8c47f67ce..bd666e93c 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 71e47629b1cb65a61d8e4809177817c1a833f73c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 26 Dec 2018 16:35:05 +0500 Subject: [PATCH 23/34] DOC fix docs for AWS_... settings. A follow-up to GH-2609. --- docs/topics/settings.rst | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 47b6cf13d..0ac26a9bd 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -178,35 +178,48 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. -.. setting:: BOT_NAME +.. setting:: AWS_ENDPOINT_URL AWS_ENDPOINT_URL ---------------- Default: ``None`` -Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality. +Endpoint URL used for S3-like storage, for example Minio or s3.scality. +Only supported with ``botocore`` library. -.. setting:: AWS_ENDPOINT_URL +.. setting:: AWS_USE_SSL AWS_USE_SSL ----------- Default: ``None`` -Use this option if you want to disable SSL connection for communication with S3 or S3-like storage. -By default SSL will be used. +Use this option if you want to disable SSL connection for communication with +S3 or S3-like storage. By default SSL will be used. +Only supported with ``botocore`` library. -.. setting:: AWS_USE_SSL +.. setting:: AWS_VERIFY AWS_VERIFY ---------- Default: ``None`` -Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur. +Verify SSL connection between Scrapy and S3 or S3-like storage. By default +SSL verification will occur. Only supported with ``botocore`` library. -.. setting:: AWS_VERIFY +.. setting:: AWS_REGION_NAME + +AWS_REGION_NAME +--------------- + +Default: ``None`` + +The name of the region associated with the AWS client. +Only supported with ``botocore`` library. + +.. setting:: BOT_NAME BOT_NAME -------- From a5e1b7bb4724bafa26b476a87a9f12b4d6479661 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 29 Nov 2018 18:19:14 -0300 Subject: [PATCH 24/34] add sitemap_filter attribute to SitemapSpider class it makes it possible to filter sitemap urls by any available attribute for example, you can filter urls with lastmod greater than a given datetime it can be helpful when the url loc itself does not aggregate that information --- docs/topics/spiders.rst | 26 ++++++++++++++++++++++++++ scrapy/spiders/sitemap.py | 10 ++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a08dc30f2..b0b9e0483 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,6 +680,32 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. + .. attribute:: sitemap_filter + + Specifies a function to filter sitemap entries and their attributes. + + For example:: + + + http://example.com/ + 2005-01-01 + + + We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + + def sitemap_filter(urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield url + + This would retrieve only ``urls`` modified on 2005 and the following + years. + + If you omit this attribute, all urls found in sitemaps will be + processed, observing other attributes and their settings. + SitemapSpider examples ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 0ee8ba5e7..907aba243 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,6 +17,7 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False + sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -43,12 +44,17 @@ class SitemapSpider(Spider): return s = Sitemap(body) + if callable(self.sitemap_filter): + it = self.sitemap_filter(s) + else: + it = s + if s.type == 'sitemapindex': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s, self.sitemap_alternate_links): + for loc in iterloc(it, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) From 672385a371453c84faa2f31425e3701b25260629 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 29 Nov 2018 18:33:20 -0300 Subject: [PATCH 25/34] using a method definition instead of a None attribute --- docs/topics/spiders.rst | 4 ++-- scrapy/spiders/sitemap.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0b9e0483..127c8d03e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. attribute:: sitemap_filter + .. method:: sitemap_filter(urls) Specifies a function to filter sitemap entries and their attributes. @@ -703,7 +703,7 @@ SitemapSpider This would retrieve only ``urls`` modified on 2005 and the following years. - If you omit this attribute, all urls found in sitemaps will be + If you omit this method, all urls found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 907aba243..c86e986db 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -17,7 +17,6 @@ class SitemapSpider(Spider): sitemap_rules = [('', 'parse')] sitemap_follow = [''] sitemap_alternate_links = False - sitemap_filter = None def __init__(self, *a, **kw): super(SitemapSpider, self).__init__(*a, **kw) @@ -32,6 +31,14 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) + def sitemap_filter(self, urls): + """This method can be used to filter sitemap entries by their + attributes, for example, you can filter locs with lastmod greater + than a given date (see docs). + """ + for url in urls: + yield url + def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): for url in sitemap_urls_from_robots(response.text, base_url=response.url): @@ -44,10 +51,7 @@ class SitemapSpider(Spider): return s = Sitemap(body) - if callable(self.sitemap_filter): - it = self.sitemap_filter(s) - else: - it = s + it = self.sitemap_filter(s) if s.type == 'sitemapindex': for loc in iterloc(it, self.sitemap_alternate_links): From d7d5917ff12ecb8db7cd04592f7cc18b0ab1a996 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 30 Nov 2018 11:20:12 -0300 Subject: [PATCH 26/34] add tests for the sitemap_filter method in the SitemapSpider class --- tests/test_spider.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index f26da2334..871852ab2 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml 'http://www.example.com/schweiz-deutsch/', 'http://www.example.com/italiano/']) + def test_sitemap_filter(self): + sitemap = b""" + + + http://www.example.com/english/ + 2010-01-01 + + + http://www.example.com/portuguese/ + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, urls): + from datetime import datetime + for url in urls: + date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + if date_time.year > 2008: + yield url + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/portuguese/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + class DeprecationTest(unittest.TestCase): From 657f0663b3cb97ca1c1a498c066de444bd30fa82 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 20 Dec 2018 13:35:52 -0300 Subject: [PATCH 27/34] rename param from urls to entries --- docs/topics/spiders.rst | 16 ++++++++-------- scrapy/spiders/sitemap.py | 6 +++--- tests/test_spider.py | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 127c8d03e..918f1cc36 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -680,7 +680,7 @@ SitemapSpider Default is ``sitemap_alternate_links`` disabled. - .. method:: sitemap_filter(urls) + .. method:: sitemap_filter(entries) Specifies a function to filter sitemap entries and their attributes. @@ -691,19 +691,19 @@ SitemapSpider 2005-01-01 - We can define a ``sitemap_filter`` function to filter ``urls`` by date:: + We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - def sitemap_filter(urls): + def sitemap_filter(entries): from datetime import datetime - for url in urls: - date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year >= 2005: - yield url + yield entry - This would retrieve only ``urls`` modified on 2005 and the following + This would retrieve only ``entries`` modified on 2005 and the following years. - If you omit this method, all urls found in sitemaps will be + If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c86e986db..534c45c70 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -31,13 +31,13 @@ class SitemapSpider(Spider): for url in self.sitemap_urls: yield Request(url, self._parse_sitemap) - def sitemap_filter(self, urls): + def sitemap_filter(self, entries): """This method can be used to filter sitemap entries by their attributes, for example, you can filter locs with lastmod greater than a given date (see docs). """ - for url in urls: - yield url + for entry in entries: + yield entry def _parse_sitemap(self, response): if response.url.endswith('/robots.txt'): diff --git a/tests/test_spider.py b/tests/test_spider.py index 871852ab2..d5d10c9ea 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml """ class FilteredSitemapSpider(self.spider_class): - def sitemap_filter(self, urls): + def sitemap_filter(self, entries): from datetime import datetime - for url in urls: - date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d') + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year > 2008: - yield url + yield entry r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) spider = self.spider_class("example.com") From 5e7ecf9dc1954060fd0445dce5fb54e020dd3e59 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 21 Dec 2018 17:31:52 -0300 Subject: [PATCH 28/34] add tests for sitemapindex --- tests/test_spider.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index d5d10c9ea..8b56cfec1 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,37 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemapindex_filter(self): + sitemap = b""" + + + http://www.example.com/sitemap1.xml + 2004-01-01T20:00:00+00:00 + + + http://www.example.com/sitemap2.xml + 2005-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d') + if date_time.year > 2004: + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap1.xml', + 'http://www.example.com/sitemap2.xml']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/sitemap2.xml']) + class DeprecationTest(unittest.TestCase): From 10f46bca54b2879da02641159e53453fe0cc97dc Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 11:20:18 -0300 Subject: [PATCH 29/34] documenting sitemap entries as suggested by @kmike --- docs/topics/spiders.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 918f1cc36..9d4ed6ca6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -703,6 +703,16 @@ SitemapSpider This would retrieve only ``entries`` modified on 2005 and the following years. + Entries are dict objects extracted from the sitemap document. + Usually, the key is the tag name and the value is the text inside it. + + It's important to notice that: + + - as the loc attribute is required, entries without this tag are discarded + - alternate links are stored in a list with the key ``alternate`` + (see ``sitemap_alternate_links``) + - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar`` + If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. From fe283bcd058734f88977a2033dfa36664e7ee619 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 12:32:22 -0300 Subject: [PATCH 30/34] add test case for sitemap filter with alternate links --- tests/test_spider.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index 8b56cfec1..fefdaa403 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -407,6 +407,41 @@ Sitemap: /sitemap-relative-url.xml self.assertEqual([req.url for req in spider._parse_sitemap(r)], ['http://www.example.com/english/']) + def test_sitemap_filter_with_alternate_links(self): + sitemap = b""" + + + http://www.example.com/english/article_1/ + 2010-01-01 + + + + http://www.example.com/english/article_2/ + 2015-01-01 + + """ + + class FilteredSitemapSpider(self.spider_class): + def sitemap_filter(self, entries): + for entry in entries: + alternate_links = entry.get('alternate', tuple()) + for link in alternate_links: + if '/deutsch/' in link: + entry['loc'] = link + yield entry + + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/article_1/', + 'http://www.example.com/english/article_2/']) + + spider = FilteredSitemapSpider("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/deutsch/article_1/']) + def test_sitemapindex_filter(self): sitemap = b""" From e1597f7c420ead9a563677aab61f18f9b89640a9 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Wed, 26 Dec 2018 15:05:21 -0300 Subject: [PATCH 31/34] improve readability --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 9d4ed6ca6..c47a2fca0 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -711,7 +711,7 @@ SitemapSpider - as the loc attribute is required, entries without this tag are discarded - alternate links are stored in a list with the key ``alternate`` (see ``sitemap_alternate_links``) - - namespaces are removed, so lxml tags named as ``{foo}bar`` become only ``bar`` + - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname`` If you omit this method, all entries found in sitemaps will be processed, observing other attributes and their settings. From b68308779a6d2ce7deda3675d0bcdf671a4fb935 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 17:37:59 -0300 Subject: [PATCH 32/34] improving docs --- docs/topics/spiders.rst | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c47a2fca0..4f7135309 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -682,7 +682,8 @@ SitemapSpider .. method:: sitemap_filter(entries) - Specifies a function to filter sitemap entries and their attributes. + This is a filter funtion that could be overridden to select sitemap entries + based on their attributes. For example:: @@ -693,12 +694,17 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - def sitemap_filter(entries): - from datetime import datetime - for entry in entries: - date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') - if date_time.year >= 2005: - yield entry + class FilteredSitemapSpider(scrapy.SitemapSpider): + name = 'filtered_sitemap_spider' + allowed_domains = ['example.com'] + sitemap_urls = ['http://example.com/sitemap.xml'] + + def sitemap_filter(self, entries): + from datetime import datetime + for entry in entries: + date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') + if date_time.year >= 2005: + yield entry This would retrieve only ``entries`` modified on 2005 and the following years. From bfbcf52e9df77af7a7c9a8a7a711e06612be4763 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 18:12:31 -0300 Subject: [PATCH 33/34] fix SitemapSpider import --- docs/topics/spiders.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4f7135309..39410d66e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -694,7 +694,9 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - class FilteredSitemapSpider(scrapy.SitemapSpider): + from scrapy.spiders.sitemap import SitemapSpider + + class FilteredSitemapSpider(SitemapSpider): name = 'filtered_sitemap_spider' allowed_domains = ['example.com'] sitemap_urls = ['http://example.com/sitemap.xml'] From 5a824c906c501a204624ea7b4fb99904807c8b81 Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Thu, 27 Dec 2018 18:34:41 -0300 Subject: [PATCH 34/34] using shorter import version and moving datetime import to the beginning of the code snippet --- docs/topics/spiders.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 39410d66e..742a88659 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -694,7 +694,8 @@ SitemapSpider We can define a ``sitemap_filter`` function to filter ``entries`` by date:: - from scrapy.spiders.sitemap import SitemapSpider + from datetime import datetime + from scrapy.spiders import SitemapSpider class FilteredSitemapSpider(SitemapSpider): name = 'filtered_sitemap_spider' @@ -702,7 +703,6 @@ SitemapSpider sitemap_urls = ['http://example.com/sitemap.xml'] def sitemap_filter(self, entries): - from datetime import datetime for entry in entries: date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d') if date_time.year >= 2005: