From 9d10ce56a0c1488c7b16f22c9ac6736dcee037fa Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 2 Feb 2025 00:36:05 +0500 Subject: [PATCH] Don't run the full test suite for special HTTP cases. --- tests/test_downloader_handlers.py | 91 +++++++++++++------------ tests/test_downloader_handlers_http2.py | 73 ++++++++------------ 2 files changed, 73 insertions(+), 91 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 81a0a62ee..b3ca16c84 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -609,49 +609,12 @@ class Https11TestCase(Http11TestCase): yield download_handler.close() -class Https11WrongHostnameTestCase(Http11TestCase): - scheme = "https" - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" - - -class Https11InvalidDNSId(Https11TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" - - -class Https11InvalidDNSPattern(Https11TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() - - -class Https11CustomCiphers(unittest.TestCase): - scheme = "https" +class SimpleHttpsTest(unittest.TestCase): + """Base class for special cases tested with just one simple request""" keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" + cipher_string: str | None = None @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: @@ -667,14 +630,16 @@ class Https11CustomCiphers(unittest.TestCase): 0, self.site, ssl_context_factory( - self.keyfile, self.certfile, cipher_string="CAMELLIA256-SHA" + self.keyfile, self.certfile, cipher_string=self.cipher_string ), interface=self.host, ) self.portno = self.port.getHost().port - crawler = get_crawler( - settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"} - ) + if self.cipher_string is not None: + settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} + else: + settings_dict = None + crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) self.download_request = self.download_handler.download_request @@ -686,7 +651,7 @@ class Https11CustomCiphers(unittest.TestCase): shutil.rmtree(self.tmpname) def getURL(self, path): - return f"{self.scheme}://{self.host}:{self.portno}/{path}" + return f"https://{self.host}:{self.portno}/{path}" def test_download(self): request = Request(self.getURL("file")) @@ -696,6 +661,42 @@ class Https11CustomCiphers(unittest.TestCase): return d +class Https11WrongHostnameTestCase(SimpleHttpsTest): + # above tests use a server certificate for "localhost", + # client connection to "localhost" too. + # here we test that even if the server certificate is for another domain, + # "www.example.com" in this case, + # the tests still pass + keyfile = "keys/example-com.key.pem" + certfile = "keys/example-com.cert.pem" + + +class Https11InvalidDNSId(SimpleHttpsTest): + """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" + + def setUp(self): + super().setUp() + self.host = "127.0.0.1" + + +class Https11InvalidDNSPattern(SimpleHttpsTest): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = "keys/localhost.ip.key" + certfile = "keys/localhost.ip.crt" + + def setUp(self): + try: + from service_identity.exceptions import CertificateError # noqa: F401 + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") + super().setUp() + + +class Https11CustomCiphers(SimpleHttpsTest): + cipher_string = "CAMELLIA256-SHA" + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" diff --git a/tests/test_downloader_handlers_http2.py b/tests/test_downloader_handlers_http2.py index d699263ac..5cca1bcc8 100644 --- a/tests/test_downloader_handlers_http2.py +++ b/tests/test_downloader_handlers_http2.py @@ -19,11 +19,20 @@ from tests.test_downloader_handlers import ( Http11MockServerTestCase, Http11ProxyTestCase, Https11CustomCiphers, + Https11InvalidDNSId, + Https11InvalidDNSPattern, Https11TestCase, + Https11WrongHostnameTestCase, UriResource, ) +def _get_dh() -> type[DownloadHandlerProtocol]: + from scrapy.core.downloader.handlers.http2 import H2DownloadHandler + + return H2DownloadHandler + + @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2TestCase(Https11TestCase): scheme = "https" @@ -31,9 +40,7 @@ class Https2TestCase(Https11TestCase): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - return H2DownloadHandler + return _get_dh() def test_protocol(self): request = Request(self.getURL("host"), method="GET") @@ -151,56 +158,32 @@ class Https2TestCase(Https11TestCase): return d -class Https2WrongHostnameTestCase(Https2TestCase): - tls_log_message = ( - 'SSL connection certificate: issuer "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", ' - 'subject "/C=XW/ST=XW/L=The ' - 'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"' - ) - - # above tests use a server certificate for "localhost", - # client connection to "localhost" too. - # here we test that even if the server certificate is for another domain, - # "www.example.com" in this case, - # the tests still pass - keyfile = "keys/example-com.key.pem" - certfile = "keys/example-com.cert.pem" +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2WrongHostnameTestCase(Https11WrongHostnameTestCase): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Https2InvalidDNSId(Https2TestCase): - """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" - - def setUp(self): - super().setUp() - self.host = "127.0.0.1" +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2InvalidDNSId(Https11InvalidDNSId): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() -class Https2InvalidDNSPattern(Https2TestCase): - """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" - - keyfile = "keys/localhost.ip.key" - certfile = "keys/localhost.ip.crt" - - def setUp(self): - try: - from service_identity.exceptions import CertificateError # noqa: F401 - except ImportError: - raise unittest.SkipTest("cryptography lib is too old") - self.tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' - 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' - ) - super().setUp() +@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +class Https2InvalidDNSPattern(Https11InvalidDNSPattern): + @property + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + return _get_dh() @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") class Https2CustomCiphers(Https11CustomCiphers): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - return H2DownloadHandler + return _get_dh() @skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") @@ -228,9 +211,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - from scrapy.core.downloader.handlers.http2 import H2DownloadHandler - - return H2DownloadHandler + return _get_dh() def setUp(self): site = server.Site(UriResource(), timeout=None)