diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 5768d8f8e..073ef16bf 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,10 +1,12 @@ import warnings from OpenSSL import SSL +from twisted.internet._sslverify import _setAcceptableProtocols from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions from scrapy.utils.misc import create_instance, load_object @@ -84,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): The default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ - def creatorForNetloc(self, hostname, port): + def creatorForNetloc(self, hostname, port): # trustRoot set to platformTrust() will use the platform's root CAs. # # This means that a website like https://www.cacert.org will be rejected @@ -97,6 +99,24 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): ) +@implementer(IPolicyForHTTPS) +class AcceptableProtocolsContextFactory: + """Context factory to used to override the acceptable protocols + to set up the [OpenSSL.SSL.Context] for doing NPN and/or ALPN + negotiation. + """ + + def __init__(self, context_factory, acceptable_protocols): + verifyObject(IPolicyForHTTPS, context_factory) + self._wrapped_context_factory = context_factory + self._acceptable_protocols = acceptable_protocols + + def creatorForNetloc(self, hostname, port): + options = self._wrapped_context_factory.creatorForNetloc(hostname, port) + _setAcceptableProtocols(options._ctx, self._acceptable_protocols) + return options + + def load_context_factory_from_settings(settings, crawler): ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index ddd813cec..4be888bda 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,16 +1,15 @@ import warnings from time import time -from typing import Optional, Tuple +from typing import Optional from urllib.parse import urldefrag -from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.error import TimeoutError -from twisted.web.client import BrowserLikePolicyForHTTPS, URI +from twisted.web.client import URI from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool +from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -41,30 +40,6 @@ class H2DownloadHandler: self._pool.close_connections() -class ScrapyProxyH2Agent(H2Agent): - def __init__( - self, reactor: ReactorBase, - proxy_uri: URI, pool: H2ConnectionPool, - context_factory=BrowserLikePolicyForHTTPS(), - connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None - ) -> None: - super(ScrapyProxyH2Agent, self).__init__( - reactor=reactor, - pool=pool, - context_factory=context_factory, - connect_timeout=connect_timeout, - bind_address=bind_address - ) - self._proxy_uri = proxy_uri - - def get_endpoint(self, uri: URI): - return self.endpoint_factory.endpointForURI(self._proxy_uri) - - def get_key(self, uri: URI) -> Tuple: - """We use the proxy uri instead of uri obtained from request url""" - return "http-proxy", self._proxy_uri.host, self._proxy_uri.port - - class ScrapyH2Agent: _Agent = H2Agent _ProxyAgent = ScrapyProxyH2Agent diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index f829cc5f8..d950c6cfb 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -2,17 +2,14 @@ from collections import deque from typing import Deque, Dict, List, Tuple, Optional from twisted.internet import defer -from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOptions from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.error import SchemeNotSupported -from twisted.web.iweb import IPolicyForHTTPS -from zope.interface import implementer -from zope.interface.verify import verifyObject +from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings @@ -96,19 +93,6 @@ class H2ConnectionPool: conn.transport.abortConnection() -@implementer(IPolicyForHTTPS) -class AcceptableProtocolsContextFactory: - def __init__(self, context_factory, acceptable_protocols: List[bytes]) -> None: - verifyObject(IPolicyForHTTPS, context_factory) - self._wrapped_context_factory = context_factory - self._acceptable_protocols = acceptable_protocols - - def creatorForNetloc(self, hostname, port) -> ClientTLSOptions: - options = self._wrapped_context_factory.creatorForNetloc(hostname, port) - _setAcceptableProtocols(options._ctx, self._acceptable_protocols) - return options - - class H2Agent: def __init__( self, reactor: ReactorBase, pool: H2ConnectionPool, @@ -144,3 +128,27 @@ class H2Agent: d = self._pool.get_connection(key, uri, endpoint) d.addCallback(lambda conn: conn.request(request, spider)) return d + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + context_factory=BrowserLikePolicyForHTTPS(), + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + context_factory=context_factory, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + def get_endpoint(self, uri: URI): + return self.endpoint_factory.endpointForURI(self._proxy_uri) + + def get_key(self, uri: URI) -> Tuple: + """We use the proxy uri instead of uri obtained from request url""" + return "http-proxy", self._proxy_uri.host, self._proxy_uri.port