diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..c0463cfc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,8 +1,12 @@ from OpenSSL import SSL +import warnings + from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS from zope.interface.declarations import implementer +from scrapy.core.downloader.tls import openssl_methods +from scrapy.utils.misc import create_instance, load_object from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS @@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): trustRoot=platformTrust(), extraCertificateOptions={'method': self._ssl_method}, ) + + +def load_context_factory_from_settings(settings, crawler): + ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] + context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + # try method-aware context factory + try: + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + method=ssl_method, + ) + except TypeError: + # use context factory defaults + context_factory = create_instance( + objcls=context_factory_cls, + settings=settings, + crawler=crawler, + ) + msg = """ + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) + warnings.warn(msg) + + return context_factory diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..dac97ad29 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer from scrapy import signals -from scrapy.core.downloader.tls import openssl_methods +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes -from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import to_bytes, to_unicode @@ -43,29 +42,7 @@ class HTTP11DownloadHandler: self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False - self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - method=self._sslMethod, - ) - except TypeError: - # use context factory defaults - self._contextFactory = create_instance( - objcls=self._contextFactoryClass, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._contextFactory = load_context_factory_from_settings(settings, crawler) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index c8b401e80..81ea78e69 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -1,10 +1,19 @@ import warnings +from time import time +from typing import Optional, Tuple +from urllib.parse import urldefrag -from scrapy.core.downloader.tls import openssl_methods +from twisted.internet.base import ReactorBase +from twisted.internet.error import TimeoutError +from twisted.web.client import URI + +from scrapy.core.downloader.contextfactory import load_context_factory_from_settings +from scrapy.core.downloader.webclient import _parse from scrapy.core.http2.agent import H2Agent, H2ConnectionPool -from scrapy.http.request import Request +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.utils.misc import create_instance, load_object +from scrapy.spiders import Spider class H2DownloadHandler: @@ -13,44 +22,128 @@ class H2DownloadHandler: from twisted.internet import reactor self._pool = H2ConnectionPool(reactor, settings) - - self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] - self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - # try method-aware context factory - try: - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - method=self._ssl_method, - ) - except TypeError: - # use context factory defaults - self._context_factory = create_instance( - objcls=self._context_factory_cls, - settings=settings, - crawler=crawler, - ) - msg = """ - '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\ - Please upgrade your context factory class to handle them or ignore them.""" % ( - settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) - warnings.warn(msg) + self._context_factory = load_context_factory_from_settings(settings, crawler) + self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') + self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') @classmethod def from_crawler(cls, crawler): return cls(crawler.settings, crawler) - def download_request(self, request: Request, spider): + def download_request(self, request: Request, spider: Spider): + agent = ScrapyH2Agent( + context_factory=self._context_factory, + pool=self._pool, + maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + crawler=self._crawler + ) + return agent.download_request(request, spider) + + def close(self) -> None: + self._pool.close_connections() + + +class ScrapyProxyH2Agent(H2Agent): + def __init__( + self, reactor: ReactorBase, + proxy_uri: URI, pool: H2ConnectionPool, + connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None + ) -> None: + super(ScrapyProxyH2Agent, self).__init__( + reactor=reactor, + pool=pool, + connect_timeout=connect_timeout, + bind_address=bind_address + ) + self._proxy_uri = proxy_uri + + @staticmethod + def get_key(uri: URI) -> Tuple: + return "http-proxy", uri.host, uri.port + + +class ScrapyH2Agent: + _Agent = H2Agent + _ProxyAgent = ScrapyProxyH2Agent + + def __init__( + self, context_factory, + connect_timeout=10, + bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None, + maxsize: int = 0, warnsize: int = 0, + crawler=None + ) -> None: + self._context_factory = context_factory + self._connect_timeout = connect_timeout + self._bind_address = bind_address + self._pool = pool + self._maxsize = maxsize + self._warnsize = warnsize + self._crawler = crawler + + def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent: from twisted.internet import reactor + bind_address = request.meta.get('bindaddress') or self._bind_address + proxy = request.meta.get('proxy') + if proxy: + _, _, proxy_host, proxy_port, proxy_params = _parse(proxy) + scheme = _parse(request.url)[0] + proxy_host = str(proxy_host, 'utf-8') + omit_connect_timeout = b'noconnect' in proxy_params + if omit_connect_timeout: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) - agent = H2Agent(reactor, self._pool, self._context_factory) - d = agent.request(request) + if scheme == b'https' and not omit_connect_timeout: + proxy_auth = request.headers.get(b'Proxy-Authorization', None) + proxy_conf = (proxy_host, proxy_port, proxy_auth) - def print_result(result): - print(result) - return result + # TODO: Return TunnelingAgent instance + else: + return self._ProxyAgent( + reactor=reactor, + proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')), + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) - d.addCallback(print_result) + return self._Agent( + reactor=reactor, + context_factory=self._context_factory, + connect_timeout=timeout, + bind_address=bind_address, + pool=self._pool + ) + + def download_request(self, request: Request, spider: Spider): + from twisted.internet import reactor + timeout = request.meta.get('download_timeout') or self._connect_timeout + agent = self._get_agent(request, timeout) + + start_time = time() + d = agent.request(request, spider) + d.addCallback(self._cb_latency, request, start_time) + + timeout_cl = reactor.callLater(timeout, d.cancel) + d.addBoth(self._cb_timeout, request, timeout, timeout_cl) return d + + @staticmethod + def _cb_latency(response: Response, request: Request, start_time: float): + request.meta['download_latency'] = time() - start_time + return response + + @staticmethod + def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl): + if timeout_cl.active(): + timeout_cl.cancel() + return response + + url = urldefrag(request.url)[0] + raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.") diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7a8847c38..c7a49fd42 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption from twisted.internet.base import ReactorBase from twisted.internet.defer import Deferred from twisted.internet.endpoints import HostnameEndpoint +from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory +from twisted.web.error import SchemeNotSupported from twisted.web.iweb import IPolicyForHTTPS from zope.interface import implementer from zope.interface.verify import verifyObject @@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory from scrapy.http.request import Request from scrapy.settings import Settings +from scrapy.spiders import Spider class H2ConnectionPool: @@ -28,8 +31,7 @@ class H2ConnectionPool: # Save all requests that arrive before the connection is established self._pending_requests: Dict[Tuple, Deque[Deferred]] = {} - def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred: - key = (uri.scheme, uri.host, uri.port) + def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol @@ -84,6 +86,15 @@ class H2ConnectionPool: d = pending_requests.popleft() d.errback(errors) + def close_connections(self) -> None: + """Close all the HTTP/2 connections and remove them from pool + + Returns: + Deferred that fires when all connections have been closed + """ + for conn in self._connections.values(): + conn.transport.loseConnection() + @implementer(IPolicyForHTTPS) class H2WrappedContextFactory: @@ -111,9 +122,21 @@ class H2Agent: connect_timeout, bind_address ) - def request(self, request: Request) -> Deferred: + def _get_endpoint(self, uri: URI): + return self._endpoint_factory.endpointForURI(uri) + + @staticmethod + def get_key(uri: URI) -> Tuple: + return uri.scheme, uri.host, uri.port + + def request(self, request: Request, spider: Spider) -> Deferred: uri = URI.fromBytes(bytes(request.url, encoding='utf-8')) - endpoint = self._endpoint_factory.endpointForURI(uri) - d = self._pool.get_connection(uri, endpoint) - d.addCallback(lambda conn: conn.request(request)) + try: + endpoint = self._get_endpoint(uri) + except SchemeNotSupported: + return defer.fail(Failure()) + + key = self.get_key(uri) + d = self._pool.get_connection(key, uri, endpoint) + d.addCallback(lambda conn: conn.request(request, spider)) return d