RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: H2 docs, NotImplementedError for H2 Tunnel

This commit is contained in:
Aditya 2020-08-11 04:39:41 +05:30
parent e0c3019d90
commit c67d6dea31
5 changed files with 54 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/topics/settings.rst
View File

@ -620,8 +620,8 @@ handler (without replacement), place this in your ``settings.py``::
'ftp': None,
}
The default https handler uses HTTP/1.x, to use HTTP/2.0 update :setting:`DOWNLOAD_HANDLERS`
as::
The default HTTPS handler uses HTTP/1.1. To use HTTP/2 update
:setting:`DOWNLOAD_HANDLERS` as follows::
DOWNLOAD_HANDLERS = {
'https': 'scrapy.core.downloader.handlers.http2.H2DownloadHandler',
@ -706,11 +706,12 @@ Optionally, this can be set per-request basis by using the
.. warning::
This is ignored when :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
is set as ``https`` download handler in :setting:`DOWNLOAD_HANDLERS`. In
case of data loss error the connection may be corrupted affecting other streams,
hence all streams return with the ``ResponseFailed([InvalidBodyLengthError])``
failure.
This setting is ignored by the
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
download handler (see :setting:`DOWNLOAD_HANDLERS`). In case of a data loss
error, the corresponding HTTP/2 connection may be corrupted, affecting other
requests that use the same connection; hence, a ``ResponseFailed([InvalidBodyLengthError])``
failure is always raised for every request that was using that connection.
.. setting:: DUPEFILTER_CLASS

7
scrapy/core/downloader/contextfactory.py
View File

@ -1,14 +1,13 @@
from OpenSSL import SSL
import warnings
from OpenSSL import SSL
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
from scrapy.core.downloader.tls import openssl_methods
from scrapy.utils.misc import create_instance, load_object
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
from scrapy.core.downloader.tls import DEFAULT_CIPHERS, openssl_methods, ScrapyClientTLSOptions
from scrapy.utils.misc import create_instance, load_object
@implementer(IPolicyForHTTPS)

44
scrapy/core/downloader/handlers/http2.py
View File

@ -6,15 +6,15 @@ from urllib.parse import urldefrag
from twisted.internet.base import ReactorBase
from twisted.internet.defer import Deferred
from twisted.internet.error import TimeoutError
from twisted.web.client import URI, BrowserLikePolicyForHTTPS
from twisted.web.client import BrowserLikePolicyForHTTPS, URI
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.python import to_bytes
class H2DownloadHandler:
@ -88,29 +88,25 @@ class ScrapyH2Agent:
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = str(proxy_host, 'utf-8')
omit_connect_timeout = b'noconnect' in proxy_params
if omit_connect_timeout:
warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Crawlera, it doesn't require this mode anymore, "
"so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.",
ScrapyDeprecationWarning)
proxy_host = proxy_host.decode()
omit_connect_tunnel = b'noconnect' in proxy_params
if omit_connect_tunnel:
warnings.warn("Using HTTPS proxies in the noconnect mode is not supported by the "
"downloader handler. If you use Crawlera, it doesn't require this "
"mode anymore, so you should update scrapy-crawlera to 1.3.0+ "
"and remove '?noconnect' from the Crawlera URL.")
if scheme == b'https' and not omit_connect_timeout:
proxy_auth = request.headers.get(b'Proxy-Authorization', None)
proxy_conf = (proxy_host, proxy_port, proxy_auth)
# TODO: Return TunnelingAgent instance
else:
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(bytes(proxy, encoding='ascii')),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool
)
if scheme == b'https' and not omit_connect_tunnel:
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
return self._ProxyAgent(
reactor=reactor,
context_factory=self._context_factory,
proxy_uri=URI.fromBytes(to_bytes(proxy, encoding='ascii')),
connect_timeout=timeout,
bind_address=bind_address,
pool=self._pool
)
return self._Agent(
reactor=reactor,

5
scrapy/core/http2/stream.py
View File

@ -192,10 +192,7 @@ class Stream:
# a path component; these MUST include a ":path" pseudo-header field
# with a value of '*' (refer RFC 7540 - Section 8.1.2.3)
if not path:
if self._request.method == 'OPTIONS':
path = path or '*'
else:
path = path or '/'
path = '*' if self._request.method == 'OPTIONS' else '/'
# Make sure pseudo-headers comes before all the other headers
headers = [

33
tests/test_downloader_handlers.py
View File

@ -315,8 +315,8 @@ class HttpTestCase(unittest.TestCase):
host = self.host + ':' + str(self.portno)
def _test(response):
self.assertEqual(response.body, bytes(host, 'utf-8'))
self.assertEqual(request.headers.get('Host'), bytes(host, 'utf-8'))
self.assertEqual(response.body, to_bytes(host))
self.assertEqual(request.headers.get('Host'), to_bytes(host))
request = Request(self.getURL('host'), headers={'Host': host})
return self.download_request(request, Spider('foo')).addCallback(_test)
@ -764,6 +764,7 @@ class UriResource(resource.Resource):
class HttpProxyTestCase(unittest.TestCase):
download_handler_cls = HTTPDownloadHandler
expected_http_proxy_request_body = b'http://example.com'
def setUp(self):
site = server.Site(UriResource(), timeout=None)
@ -786,10 +787,7 @@ class HttpProxyTestCase(unittest.TestCase):
def _test(response):
self.assertEqual(response.status, 200)
self.assertEqual(response.url, request.url)
self.assertTrue(
response.body == b'http://example.com' # HTTP/1.x
or response.body == b'/' # HTTP/2
)
self.assertEqual(response.body, self.expected_http_proxy_request_body)
http_proxy = self.getURL('')
request = Request('http://example.com', meta={'proxy': http_proxy})
@ -799,13 +797,10 @@ class HttpProxyTestCase(unittest.TestCase):
def _test(response):
self.assertEqual(response.status, 200)
self.assertEqual(response.url, request.url)
self.assertTrue(
response.body == b'http://example.com' # HTTP/1.x
or response.body == b'/' # HTTP/2
)
self.assertEqual(response.body, b'https://example.com')
http_proxy = '%s?noconnect' % self.getURL('')
request = Request('http://example.com', meta={'proxy': http_proxy})
request = Request('https://example.com', meta={'proxy': http_proxy})
with self.assertWarnsRegex(ScrapyDeprecationWarning,
r'Using HTTPS proxies in the noconnect mode is deprecated'):
return self.download_request(request, Spider('foo')).addCallback(_test)
@ -851,6 +846,7 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
host = u'127.0.0.1'
download_handler_cls = H2DownloadHandler
expected_http_proxy_request_body = b'/'
def setUp(self):
site = server.Site(UriResource(), timeout=None)
@ -866,6 +862,21 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
def getURL(self, path):
return f"{self.scheme}://{self.host}:{self.portno}/{path}"
def test_download_with_proxy_https_noconnect(self):
def _test(response):
self.assertEqual(response.status, 200)
self.assertEqual(response.url, request.url)
self.assertEqual(response.body, b'/')
http_proxy = '%s?noconnect' % self.getURL('')
request = Request('https://example.com', meta={'proxy': http_proxy})
with self.assertWarnsRegex(
Warning,
r'Using HTTPS proxies in the noconnect mode is not supported by the '
r'downloader handler.'
):
return self.download_request(request, Spider('foo')).addCallback(_test)
class HttpDownloadHandlerMock: