diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 670b44f3c..bb543433c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -620,8 +620,8 @@ handler (without replacement), place this in your ``settings.py``:: 'ftp': None, } -The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS` -as:: +The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update +:setting:`DOWNLOAD_HANDLERS` as follows:: DOWNLOAD_HANDLERS = { 'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler', @@ -706,11 +706,12 @@ Optionally, this can be set per-request basis by using the .. warning:: - This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` - is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In - case of data loss error the connection may be corrupted affecting other streams, - hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])`` - failure. + This setting is ignored by the + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` + download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss + error, the corresponding HTTP/2 connection may be corrupted, affecting other + requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])`` + failure is always raised for every request that was using that connection. .. setting:: DUPEFILTER_CLASS diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c0463cfc7..8dcba15ff 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,14 +1,13 @@ -from OpenSSL import SSL import warnings +from OpenSSL import SSL from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer -from scrapy.core.downloader.tls import openssl_methods -from scrapy.utils.misc import create_instance, load_object -from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS +from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions +from scrapy.utils.misc import create_instance, load_object @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index d6e5cd1c1..650af9778 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -6,15 +6,15 @@ from urllib.parse import urldefrag from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import URI, BrowserLikePolicyForHTTPS +from twisted.web.client import BrowserLikePolicyForHTTPS, URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes class H2DownloadHandler: @@ -88,29 +88,25 @@ class ScrapyH2Agent: if proxy: _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) scheme = _parse(request.url)[0] - proxy_host = str(proxy_host, 'utf-8') - omit_connect_timeout = b'noconnect' in proxy_params - if omit_connect_timeout: - warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " - "If you use Crawlera, it doesn't require this mode anymore, " - "so you should update scrapy-crawlera to 1.3.0+ " - "and remove '?noconnect' from the Crawlera URL.", - ScrapyDeprecationWarning) + proxy_host = proxy_host.decode() + omit_connect_tunnel = b'noconnect' in proxy_params + if omit_connect_tunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the " + "downloader handler. If you use Crawlera, it doesn't require this " + "mode anymore, so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.") - if scheme == b'https' and not omit_connect_timeout: - proxy_auth = request.headers.get(b'Proxy-Authorization', None) - proxy_conf = (proxy_host, proxy_port, proxy_auth) - - # TODO: Return TunnelingAgent instance - else: - return self._ProxyAgent( - reactor=reactor, - context_factory=self._context_factory, - proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), - connect_timeout=timeout, - bind_address=bind_address, - pool=self._pool - ) + if scheme == b'https' and not omit_connect_tunnel: + # ToDo + raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported') + return self._ProxyAgent( + reactor=reactor, + context_factory=self._context_factory, + proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) return self._Agent( reactor=reactor, diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 40ea07b63..1e136fbd5 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -192,10 +192,7 @@ class Stream: # a path component; these MUST include a ":path" pseudo-header field # with a value of '*' (refer RFC 7540 - Section 8.1.2.3) if not path: - if self._request.method == 'OPTIONS': - path = path or '*' - else: - path = path or '/' + path = '*' if self._request.method == 'OPTIONS' else '/' # Make sure pseudo-headers comes before all the other headers headers = [ diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 486614121..1cedf6b10 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -315,8 +315,8 @@ class HttpTestCase(unittest.TestCase): host = self.host + ':' + str(self.portno) def _test(response): - self.assertEqual(response.body, bytes(host, 'utf-8')) - self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8')) + self.assertEqual(response.body, to_bytes(host)) + self.assertEqual(request.headers.get('Host'), to_bytes(host)) request = Request(self.getURL('host'), headers={'Host': host}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -764,6 +764,7 @@ class UriResource(resource.Resource): class HttpProxyTestCase(unittest.TestCase): download_handler_cls = HTTPDownloadHandler + expected_http_proxy_request_body = b'http://example.com' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -786,10 +787,7 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, self.expected_http_proxy_request_body) http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -799,13 +797,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEqual(response.status, 200) self.assertEqual(response.url, request.url) - self.assertTrue( - response.body == b'http://example.com' # HTTP/1.x - or response.body == b'/' # HTTP/2 - ) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') - request = Request('http://example.com', meta={'proxy': http_proxy}) + request = Request('https://example.com', meta={'proxy': http_proxy}) with self.assertWarnsRegex(ScrapyDeprecationWarning, r'Using HTTPS proxies in the noconnect mode is deprecated'): return self.download_request(request, Spider('foo')).addCallback(_test) @@ -851,6 +846,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase): host = u'127.0.0.1' download_handler_cls = H2DownloadHandler + expected_http_proxy_request_body = b'/' def setUp(self): site = server.Site(UriResource(), timeout=None) @@ -866,6 +862,21 @@ class Https2ProxyTestCase(Http11ProxyTestCase): def getURL(self, path): return f"{self.scheme}://{self.host}:{self.portno}/{path}" + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex( + Warning, + r'Using HTTPS proxies in the noconnect mode is not supported by the ' + r'downloader handler.' + ): + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: