From 37efdde3e3dd4efecf7dccd361fceab6c851d0ff Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 6 Jul 2016 14:18:59 +0200 Subject: [PATCH 1/3] Catch and ignore TLS verification exception for IP-address hosts Fixes GH-2092 --- scrapy/core/downloader/tls.py | 5 +++++ tests/test_downloader_handlers.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 5fa8e2723..9bf6a575b 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -48,6 +48,11 @@ try: 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) + except ValueError as e: + logger.warning( + 'SSL/TLS verification failed for hostname "{}"; {}'.format( + self._hostnameASCII, e)) + except ImportError: # ImportError should not matter for older Twisted versions # as the above is not used in the fallback ScrapyClientContextFactory diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45a806f2e..fb73c43ee 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -335,6 +335,14 @@ class Https11WrongHostnameTestCase(Http11TestCase): certfile = 'keys/example-com.cert.pem' +class Https11InvalidDNSId(Https11TestCase): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super(Https11InvalidDNSId, self).setUp() + self.host = '127.0.0.1' + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" if twisted_version < (11, 1, 0): From 859bcf48206369d4ff40f85bd55bbfdcadf65e25 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jul 2016 17:53:19 +0200 Subject: [PATCH 2/3] Rephrase warning --- scrapy/core/downloader/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 9bf6a575b..5d2d68d82 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -50,7 +50,7 @@ try: except ValueError as e: logger.warning( - 'SSL/TLS verification failed for hostname "{}"; {}'.format( + 'Ignoring remote certificate verification failure for hostname "{}"; {}'.format( self._hostnameASCII, e)) except ImportError: From 005cf949b8618259427c811da35e89f29c73feed Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jul 2016 11:01:30 +0200 Subject: [PATCH 3/3] Change wording of warning + docstring for ScrapyClientTLSOptions --- scrapy/core/downloader/tls.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 5d2d68d82..00c94ee2e 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -34,9 +34,15 @@ try: VerificationError) class ScrapyClientTLSOptions(ClientTLSOptions): - # same as Twisted's ClientTLSOptions, - # except that VerificationError is caught - # and doesn't close the connection + """ + SSL Client connection creator ignoring certificate verification errors + (for genuinely invalid certificates or bugs in verification code). + + Same as Twisted's private _sslverify.ClientTLSOptions, + except that VerificationError and ValueError exceptions are caught, + so that the connection is not closed, only logging warnings. + """ + def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL_CB_HANDSHAKE_START: _maybeSetHostNameIndication(connection, self._hostnameBytes) @@ -50,8 +56,9 @@ try: except ValueError as e: logger.warning( - 'Ignoring remote certificate verification failure for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + 'Ignoring error while verifying certificate ' + 'from host "{}" (exception: {})'.format( + self._hostnameASCII, repr(e))) except ImportError: # ImportError should not matter for older Twisted versions