RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat(wip): ScrapyH2Agent, ScrapyProxyH2Agent

This commit is contained in:
Aditya 2020-07-22 15:01:59 +05:30
parent 62ce842afc
commit 031bfc9c3b
4 changed files with 190 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scrapy/core/downloader/contextfactory.py
View File

@ -1,8 +1,12 @@
from OpenSSL import SSL
import warnings
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from scrapy.core.downloader.tls import openssl_methods
from scrapy.utils.misc import create_instance, load_object
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
@ -92,3 +96,31 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)
def load_context_factory_from_settings(settings, crawler):
ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
method=ssl_method,
)
except TypeError:
# use context factory defaults
context_factory = create_instance(
objcls=context_factory_cls,
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
return context_factory

27
scrapy/core/downloader/handlers/http11.py
View File

@ -20,12 +20,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.tls import openssl_methods
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_bytes, to_unicode
@ -43,29 +42,7 @@ class HTTP11DownloadHandler:
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
self._contextFactory = create_instance(
objcls=self._contextFactoryClass,
settings=settings,
crawler=crawler,
method=self._sslMethod,
)
except TypeError:
# use context factory defaults
self._contextFactory = create_instance(
objcls=self._contextFactoryClass,
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
self._contextFactory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')

161
scrapy/core/downloader/handlers/http2.py
View File

@ -1,10 +1,19 @@
import warnings
from time import time
from typing import Optional, Tuple
from urllib.parse import urldefrag
from scrapy.core.downloader.tls import openssl_methods
from twisted.internet.base import ReactorBase
from twisted.internet.error import TimeoutError
from twisted.web.client import URI
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
from scrapy.http.request import Request
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.utils.misc import create_instance, load_object
from scrapy.spiders import Spider
class H2DownloadHandler:
@ -13,44 +22,128 @@ class H2DownloadHandler:
from twisted.internet import reactor
self._pool = H2ConnectionPool(reactor, settings)
self._ssl_method = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
self._context_factory_cls = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
self._context_factory = create_instance(
objcls=self._context_factory_cls,
settings=settings,
crawler=crawler,
method=self._ssl_method,
)
except TypeError:
# use context factory defaults
self._context_factory = create_instance(
objcls=self._context_factory_cls,
settings=settings,
crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
self._context_factory = load_context_factory_from_settings(settings, crawler)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings, crawler)
def download_request(self, request: Request, spider):
def download_request(self, request: Request, spider: Spider):
agent = ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
crawler=self._crawler
)
return agent.download_request(request, spider)
def close(self) -> None:
self._pool.close_connections()
class ScrapyProxyH2Agent(H2Agent):
def __init__(
self, reactor: ReactorBase,
proxy_uri: URI, pool: H2ConnectionPool,
connect_timeout: Optional[float] = None, bind_address: Optional[bytes] = None
) -> None:
super(ScrapyProxyH2Agent, self).__init__(
reactor=reactor,
pool=pool,
connect_timeout=connect_timeout,
bind_address=bind_address
)
self._proxy_uri = proxy_uri
@staticmethod
def get_key(uri: URI) -> Tuple:
return "http-proxy", uri.host, uri.port
class ScrapyH2Agent:
_Agent = H2Agent
_ProxyAgent = ScrapyProxyH2Agent
def __init__(
self, context_factory,
connect_timeout=10,
bind_address: Optional[bytes] = None, pool: H2ConnectionPool = None,
maxsize: int = 0, warnsize: int = 0,
crawler=None
) -> None:
self._context_factory = context_factory
self._connect_timeout = connect_timeout
self._bind_address = bind_address
self._pool = pool
self._maxsize = maxsize
self._warnsize = warnsize
self._crawler = crawler
def _get_agent(self, request: Request, timeout: Optional[float]) -> H2Agent:
from twisted.internet import reactor
bind_address = request.meta.get('bindaddress') or self._bind_address
proxy = request.meta.get('proxy')
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = str(proxy_host, 'utf-8')
omit_connect_timeout = b'noconnect' in proxy_params
if omit_connect_timeout:
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Crawlera, it doesn't require this mode anymore, "
"so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.",
ScrapyDeprecationWarning)
agent = H2Agent(reactor, self._pool, self._context_factory)
d = agent.request(request)
if scheme == b'https' and not omit_connect_timeout:
proxy_auth = request.headers.get(b'Proxy-Authorization', None)
proxy_conf = (proxy_host, proxy_port, proxy_auth)
def print_result(result):
print(result)
return result
# TODO: Return TunnelingAgent instance
else:
return self._ProxyAgent(
reactor=reactor,
proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool
)
d.addCallback(print_result)
return self._Agent(
reactor=reactor,
context_factory=self._context_factory,
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool
)
def download_request(self, request: Request, spider: Spider):
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connect_timeout
agent = self._get_agent(request, timeout)
start_time = time()
d = agent.request(request, spider)
d.addCallback(self._cb_latency, request, start_time)
timeout_cl = reactor.callLater(timeout, d.cancel)
d.addBoth(self._cb_timeout, request, timeout, timeout_cl)
return d
@staticmethod
def _cb_latency(response: Response, request: Request, start_time: float):
request.meta['download_latency'] = time() - start_time
return response
@staticmethod
def _cb_timeout(response: Response, request: Request, timeout: float, timeout_cl):
if timeout_cl.active():
timeout_cl.cancel()
return response
url = urldefrag(request.url)[0]
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")

35
scrapy/core/http2/agent.py
View File

@ -6,7 +6,9 @@ from twisted.internet._sslverify import _setAcceptableProtocols, ClientTLSOption
from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
from twisted.internet.endpoints import HostnameEndpoint
from twisted.python.failure import Failure
from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory
from twisted.web.error import SchemeNotSupported
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface import implementer
from zope.interface.verify import verifyObject
@ -14,6 +16,7 @@ from zope.interface.verify import verifyObject
from scrapy.core.http2.protocol import H2ClientProtocol, H2ClientFactory
from scrapy.http.request import Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
class H2ConnectionPool:
@ -28,8 +31,7 @@ class H2ConnectionPool:
# Save all requests that arrive before the connection is established
self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
def get_connection(self, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
key = (uri.scheme, uri.host, uri.port)
def get_connection(self, key: Tuple, uri: URI, endpoint: HostnameEndpoint) -> Deferred:
if key in self._pending_requests:
# Received a request while connecting to remote
# Create a deferred which will fire with the H2ClientProtocol
@ -84,6 +86,15 @@ class H2ConnectionPool:
d = pending_requests.popleft()
d.errback(errors)
def close_connections(self) -> None:
"""Close all the HTTP/2 connections and remove them from pool
Returns:
Deferred that fires when all connections have been closed
"""
for conn in self._connections.values():
conn.transport.loseConnection()
@implementer(IPolicyForHTTPS)
class H2WrappedContextFactory:
@ -111,9 +122,21 @@ class H2Agent:
connect_timeout, bind_address
)
def request(self, request: Request) -> Deferred:
def _get_endpoint(self, uri: URI):
return self._endpoint_factory.endpointForURI(uri)
@staticmethod
def get_key(uri: URI) -> Tuple:
return uri.scheme, uri.host, uri.port
def request(self, request: Request, spider: Spider) -> Deferred:
uri = URI.fromBytes(bytes(request.url, encoding='utf-8'))
endpoint = self._endpoint_factory.endpointForURI(uri)
d = self._pool.get_connection(uri, endpoint)
d.addCallback(lambda conn: conn.request(request))
try:
endpoint = self._get_endpoint(uri)
except SchemeNotSupported:
return defer.fail(Failure())
key = self.get_key(uri)
d = self._pool.get_connection(key, uri, endpoint)
d.addCallback(lambda conn: conn.request(request, spider))
return d