RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Don't run the full test suite for special HTTP cases.

This commit is contained in:
Andrey Rakhmatullin 2025-02-02 00:36:05 +05:00
parent fcc4d30df0
commit 9d10ce56a0
2 changed files with 73 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
tests/test_downloader_handlers.py
View File

@ -609,49 +609,12 @@ class Https11TestCase(Http11TestCase):
yield download_handler.close()
class Https11WrongHostnameTestCase(Http11TestCase):
scheme = "https"
# above tests use a server certificate for "localhost",
# client connection to "localhost" too.
# here we test that even if the server certificate is for another domain,
# "www.example.com" in this case,
# the tests still pass
keyfile = "keys/example-com.key.pem"
certfile = "keys/example-com.cert.pem"
class Https11InvalidDNSId(Https11TestCase):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
def setUp(self):
super().setUp()
self.host = "127.0.0.1"
class Https11InvalidDNSPattern(Https11TestCase):
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
keyfile = "keys/localhost.ip.key"
certfile = "keys/localhost.ip.crt"
def setUp(self):
try:
from service_identity.exceptions import CertificateError # noqa: F401
except ImportError:
raise unittest.SkipTest("cryptography lib is too old")
self.tls_log_message = (
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
)
super().setUp()
class Https11CustomCiphers(unittest.TestCase):
scheme = "https"
class SimpleHttpsTest(unittest.TestCase):
"""Base class for special cases tested with just one simple request"""
keyfile = "keys/localhost.key"
certfile = "keys/localhost.crt"
cipher_string: str | None = None
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
@ -667,14 +630,16 @@ class Https11CustomCiphers(unittest.TestCase):
0,
self.site,
ssl_context_factory(
self.keyfile, self.certfile, cipher_string="CAMELLIA256-SHA"
self.keyfile, self.certfile, cipher_string=self.cipher_string
),
interface=self.host,
)
self.portno = self.port.getHost().port
crawler = get_crawler(
settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": "CAMELLIA256-SHA"}
)
if self.cipher_string is not None:
settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string}
else:
settings_dict = None
crawler = get_crawler(settings_dict=settings_dict)
self.download_handler = build_from_crawler(self.download_handler_cls, crawler)
self.download_request = self.download_handler.download_request
@ -686,7 +651,7 @@ class Https11CustomCiphers(unittest.TestCase):
shutil.rmtree(self.tmpname)
def getURL(self, path):
return f"{self.scheme}://{self.host}:{self.portno}/{path}"
return f"https://{self.host}:{self.portno}/{path}"
def test_download(self):
request = Request(self.getURL("file"))
@ -696,6 +661,42 @@ class Https11CustomCiphers(unittest.TestCase):
return d
class Https11WrongHostnameTestCase(SimpleHttpsTest):
# above tests use a server certificate for "localhost",
# client connection to "localhost" too.
# here we test that even if the server certificate is for another domain,
# "www.example.com" in this case,
# the tests still pass
keyfile = "keys/example-com.key.pem"
certfile = "keys/example-com.cert.pem"
class Https11InvalidDNSId(SimpleHttpsTest):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
def setUp(self):
super().setUp()
self.host = "127.0.0.1"
class Https11InvalidDNSPattern(SimpleHttpsTest):
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
keyfile = "keys/localhost.ip.key"
certfile = "keys/localhost.ip.crt"
def setUp(self):
try:
from service_identity.exceptions import CertificateError # noqa: F401
except ImportError:
raise unittest.SkipTest("cryptography lib is too old")
super().setUp()
class Https11CustomCiphers(SimpleHttpsTest):
cipher_string = "CAMELLIA256-SHA"
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""

73
tests/test_downloader_handlers_http2.py
View File

@ -19,11 +19,20 @@ from tests.test_downloader_handlers import (
Http11MockServerTestCase,
Http11ProxyTestCase,
Https11CustomCiphers,
Https11InvalidDNSId,
Https11InvalidDNSPattern,
Https11TestCase,
Https11WrongHostnameTestCase,
UriResource,
)
def _get_dh() -> type[DownloadHandlerProtocol]:
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
return H2DownloadHandler
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2TestCase(Https11TestCase):
scheme = "https"
@ -31,9 +40,7 @@ class Https2TestCase(Https11TestCase):
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
return H2DownloadHandler
return _get_dh()
def test_protocol(self):
request = Request(self.getURL("host"), method="GET")
@ -151,56 +158,32 @@ class Https2TestCase(Https11TestCase):
return d
class Https2WrongHostnameTestCase(Https2TestCase):
tls_log_message = (
'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", '
'subject "/C=XW/ST=XW/L=The '
'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"'
)
# above tests use a server certificate for "localhost",
# client connection to "localhost" too.
# here we test that even if the server certificate is for another domain,
# "www.example.com" in this case,
# the tests still pass
keyfile = "keys/example-com.key.pem"
certfile = "keys/example-com.cert.pem"
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2WrongHostnameTestCase(Https11WrongHostnameTestCase):
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return _get_dh()
class Https2InvalidDNSId(Https2TestCase):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
def setUp(self):
super().setUp()
self.host = "127.0.0.1"
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2InvalidDNSId(Https11InvalidDNSId):
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return _get_dh()
class Https2InvalidDNSPattern(Https2TestCase):
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
keyfile = "keys/localhost.ip.key"
certfile = "keys/localhost.ip.crt"
def setUp(self):
try:
from service_identity.exceptions import CertificateError # noqa: F401
except ImportError:
raise unittest.SkipTest("cryptography lib is too old")
self.tls_log_message = (
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
)
super().setUp()
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2InvalidDNSPattern(Https11InvalidDNSPattern):
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
return _get_dh()
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
class Https2CustomCiphers(Https11CustomCiphers):
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
return H2DownloadHandler
return _get_dh()
@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled")
@ -228,9 +211,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
return H2DownloadHandler
return _get_dh()
def setUp(self):
site = server.Site(UriResource(), timeout=None)