mirror of https://github.com/scrapy/scrapy.git
test: H2DownloadHandler
Following tests are skipped as Content-Length header not matching the data received is considered as a ProtocolError - test_download_broken_content_cause_data_loss - test_download_broken_chunked_content_cause_data_loss - test_download_broken_content_allow_data_loss - test_download_broken_chunked_content_allow_data_loss - test_download_broken_content_allow_data_loss_via_setting - test_download_broken_chunked_content_allow_data_loss_via_setting BREAKING CHANGES The following tests currently fail - test_content_length_zero_bodyless_post_request_headers - test_host_header_seted_in_request_headers - test_download_with_maxsize_very_large_file
This commit is contained in:
Aditya
parent
92bec38591
commit
e8342996f6
|
|
@ -23,8 +23,8 @@ from scrapy.core.downloader.handlers.file import FileDownloadHandler
|
|||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
|
||||
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
|
||||
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Request
|
||||
from scrapy.http.response.text import TextResponse
|
||||
|
|
@ -33,7 +33,6 @@ from scrapy.spiders import Spider
|
|||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.test import get_crawler, skip_if_no_boto
|
||||
|
||||
from tests.mockserver import MockServer, ssl_context_factory, Echo
|
||||
from tests.spiders import SingleRequestSpider
|
||||
|
||||
|
|
@ -131,6 +130,7 @@ class ContentLengthHeaderResource(resource.Resource):
|
|||
A testing resource which renders itself as the value of the Content-Length
|
||||
header from the request.
|
||||
"""
|
||||
|
||||
def render(self, request):
|
||||
return request.requestHeaders.getRawHeaders(b"content-length")[0]
|
||||
|
||||
|
|
@ -142,6 +142,7 @@ class ChunkedResource(resource.Resource):
|
|||
request.write(b"chunked ")
|
||||
request.write(b"content\n")
|
||||
request.finish()
|
||||
|
||||
reactor.callLater(0, response)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
|
@ -155,6 +156,7 @@ class BrokenChunkedResource(resource.Resource):
|
|||
# Disable terminating chunk on finish.
|
||||
request.chunked = False
|
||||
closeConnection(request)
|
||||
|
||||
reactor.callLater(0, response)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
|
@ -186,6 +188,7 @@ class EmptyContentTypeHeaderResource(resource.Resource):
|
|||
A testing resource which renders itself as the value of request body
|
||||
without content-type header in response.
|
||||
"""
|
||||
|
||||
def render(self, request):
|
||||
request.setHeader("content-type", "")
|
||||
return request.content.read()
|
||||
|
|
@ -197,12 +200,12 @@ class LargeChunkedFileResource(resource.Resource):
|
|||
for i in range(1024):
|
||||
request.write(b"x" * 1024)
|
||||
request.finish()
|
||||
|
||||
reactor.callLater(0, response)
|
||||
return server.NOT_DONE_YET
|
||||
|
||||
|
||||
class HttpTestCase(unittest.TestCase):
|
||||
|
||||
scheme = 'http'
|
||||
download_handler_cls = HTTPDownloadHandler
|
||||
|
||||
|
|
@ -230,10 +233,12 @@ class HttpTestCase(unittest.TestCase):
|
|||
r.putChild(b"echo", Echo())
|
||||
self.site = server.Site(r, timeout=None)
|
||||
self.wrapper = WrappingFactory(self.site)
|
||||
self.host = 'localhost'
|
||||
self.host = u'localhost'
|
||||
if self.scheme == 'https':
|
||||
# Using WrappingFactory do not enable HTTP/2 failing all the
|
||||
# tests with H2DownloadHandler
|
||||
self.port = reactor.listenSSL(
|
||||
0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),
|
||||
0, self.site, ssl_context_factory(self.keyfile, self.certfile),
|
||||
interface=self.host)
|
||||
else:
|
||||
self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
|
||||
|
|
@ -330,6 +335,7 @@ class HttpTestCase(unittest.TestCase):
|
|||
https://github.com/kennethreitz/requests/issues/405
|
||||
https://bugs.python.org/issue14721
|
||||
"""
|
||||
|
||||
def _test(response):
|
||||
self.assertEqual(response.body, b'0')
|
||||
|
||||
|
|
@ -514,6 +520,30 @@ class Https11TestCase(Http11TestCase):
|
|||
yield download_handler.close()
|
||||
|
||||
|
||||
class Https2TestCase(Https11TestCase):
|
||||
scheme = 'https'
|
||||
download_handler_cls = H2DownloadHandler
|
||||
HTTP2_DATALOSS_SKIP_REASON = "Content-Length mismatch raises InvalidBodyLengthError"
|
||||
|
||||
def test_download_broken_content_cause_data_loss(self, url='broken'):
|
||||
raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_broken_chunked_content_cause_data_loss(self):
|
||||
raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_broken_content_allow_data_loss(self, url='broken'):
|
||||
raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_broken_chunked_content_allow_data_loss(self):
|
||||
raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
|
||||
raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
|
||||
raise unittest.SkipTest(self.HTTP2_DATALOSS_SKIP_REASON)
|
||||
|
||||
|
||||
class Https11WrongHostnameTestCase(Http11TestCase):
|
||||
scheme = 'https'
|
||||
|
||||
|
|
@ -526,6 +556,23 @@ class Https11WrongHostnameTestCase(Http11TestCase):
|
|||
certfile = 'keys/example-com.cert.pem'
|
||||
|
||||
|
||||
class Https2WrongHostnameTestCase(Https2TestCase):
|
||||
tls_log_message = (
|
||||
'SSL connection certificate: issuer "/C=XW/ST=XW/L=The '
|
||||
'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com", '
|
||||
'subject "/C=XW/ST=XW/L=The '
|
||||
'Internet/O=Scrapy/CN=www.example.com/emailAddress=test@example.com"'
|
||||
)
|
||||
|
||||
# above tests use a server certificate for "localhost",
|
||||
# client connection to "localhost" too.
|
||||
# here we test that even if the server certificate is for another domain,
|
||||
# "www.example.com" in this case,
|
||||
# the tests still pass
|
||||
keyfile = 'keys/example-com.key.pem'
|
||||
certfile = 'keys/example-com.cert.pem'
|
||||
|
||||
|
||||
class Https11InvalidDNSId(Https11TestCase):
|
||||
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
|
||||
|
||||
|
|
@ -534,6 +581,14 @@ class Https11InvalidDNSId(Https11TestCase):
|
|||
self.host = '127.0.0.1'
|
||||
|
||||
|
||||
class Https2InvalidDNSId(Https2TestCase):
|
||||
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
|
||||
|
||||
def setUp(self):
|
||||
super(Https2InvalidDNSId, self).setUp()
|
||||
self.host = '127.0.0.1'
|
||||
|
||||
|
||||
class Https11InvalidDNSPattern(Https11TestCase):
|
||||
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
|
||||
|
||||
|
|
@ -552,6 +607,24 @@ class Https11InvalidDNSPattern(Https11TestCase):
|
|||
super(Https11InvalidDNSPattern, self).setUp()
|
||||
|
||||
|
||||
class Https2InvalidDNSPattern(Https2TestCase):
|
||||
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
|
||||
|
||||
keyfile = 'keys/localhost.ip.key'
|
||||
certfile = 'keys/localhost.ip.crt'
|
||||
|
||||
def setUp(self):
|
||||
try:
|
||||
from service_identity.exceptions import CertificateError # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("cryptography lib is too old")
|
||||
self.tls_log_message = (
|
||||
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
|
||||
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
|
||||
)
|
||||
super(Https2InvalidDNSPattern, self).setUp()
|
||||
|
||||
|
||||
class Https11CustomCiphers(unittest.TestCase):
|
||||
scheme = 'https'
|
||||
download_handler_cls = HTTP11DownloadHandler
|
||||
|
|
@ -565,10 +638,9 @@ class Https11CustomCiphers(unittest.TestCase):
|
|||
FilePath(self.tmpname).child("file").setContent(b"0123456789")
|
||||
r = static.File(self.tmpname)
|
||||
self.site = server.Site(r, timeout=None)
|
||||
self.wrapper = WrappingFactory(self.site)
|
||||
self.host = 'localhost'
|
||||
self.port = reactor.listenSSL(
|
||||
0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
|
||||
0, self.site, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
|
||||
interface=self.host)
|
||||
self.portno = self.port.getHost().port
|
||||
crawler = get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
|
||||
|
|
@ -593,6 +665,11 @@ class Https11CustomCiphers(unittest.TestCase):
|
|||
return d
|
||||
|
||||
|
||||
class Https2CustomCiphers(Https11CustomCiphers):
|
||||
scheme = 'https'
|
||||
download_handler_cls = H2DownloadHandler
|
||||
|
||||
|
||||
class Http11MockServerTestCase(unittest.TestCase):
|
||||
"""HTTP 1.1 test case with MockServer"""
|
||||
|
||||
|
|
@ -644,6 +721,10 @@ class Http11MockServerTestCase(unittest.TestCase):
|
|||
self.assertTrue(reason, 'finished')
|
||||
|
||||
|
||||
class Http2MockServerTestCase(Http11MockServerTestCase):
|
||||
"""HTTP 2.0 test case with MockServer"""
|
||||
|
||||
|
||||
class UriResource(resource.Resource):
|
||||
"""Return the full uri that was requested"""
|
||||
|
||||
|
|
@ -734,6 +815,11 @@ class Http11ProxyTestCase(HttpProxyTestCase):
|
|||
self.assertIn(domain, timeout.osError)
|
||||
|
||||
|
||||
# TODO:
|
||||
class Http2ProxyTestCase(Http11ProxyTestCase):
|
||||
download_handler_cls = H2DownloadHandler
|
||||
|
||||
|
||||
class HttpDownloadHandlerMock:
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
|
|
@ -931,7 +1017,6 @@ class S3TestCase(unittest.TestCase):
|
|||
|
||||
|
||||
class BaseFTPTestCase(unittest.TestCase):
|
||||
|
||||
username = "scrapy"
|
||||
password = "passwd"
|
||||
req_meta = {"ftp_user": username, "ftp_password": password}
|
||||
|
|
@ -969,6 +1054,7 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
def _clean(data):
|
||||
self.download_handler.client.transport.loseConnection()
|
||||
return data
|
||||
|
||||
deferred.addCallback(_clean)
|
||||
if callback:
|
||||
deferred.addCallback(callback)
|
||||
|
|
@ -985,6 +1071,7 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
self.assertEqual(r.status, 200)
|
||||
self.assertEqual(r.body, b'I have the power!')
|
||||
self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
|
||||
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def test_ftp_download_path_with_spaces(self):
|
||||
|
|
@ -998,6 +1085,7 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
self.assertEqual(r.status, 200)
|
||||
self.assertEqual(r.body, b'Moooooooooo power!')
|
||||
self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']})
|
||||
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def test_ftp_download_notexist(self):
|
||||
|
|
@ -1007,6 +1095,7 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
|
||||
def _test(r):
|
||||
self.assertEqual(r.status, 404)
|
||||
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def test_ftp_local_filename(self):
|
||||
|
|
@ -1027,6 +1116,7 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
with open(local_fname, "rb") as f:
|
||||
self.assertEqual(f.read(), b"I have the power!")
|
||||
os.remove(local_fname)
|
||||
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
|
||||
|
|
@ -1043,11 +1133,11 @@ class FTPTestCase(BaseFTPTestCase):
|
|||
|
||||
def _test(r):
|
||||
self.assertEqual(r.type, ConnectionLost)
|
||||
|
||||
return self._add_test_callbacks(d, errback=_test)
|
||||
|
||||
|
||||
class AnonymousFTPTestCase(BaseFTPTestCase):
|
||||
|
||||
username = "anonymous"
|
||||
req_meta = {}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue