mirror of https://github.com/scrapy/scrapy.git
Remove `noconnect` deprecate code
This commit is contained in:
parent
45c2bd7d9c
commit
e769532644
|
|
@ -3,7 +3,6 @@
|
|||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from time import time
|
||||
|
|
@ -22,7 +21,7 @@ from zope.interface import implementer
|
|||
from scrapy import signals
|
||||
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
|
||||
from scrapy.core.downloader.webclient import _parse
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
|
||||
from scrapy.exceptions import StopDownload
|
||||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
|
@ -279,17 +278,7 @@ class ScrapyAgent:
|
|||
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
|
||||
scheme = _parse(request.url)[0]
|
||||
proxyHost = to_unicode(proxyHost)
|
||||
omitConnectTunnel = b'noconnect' in proxyParams
|
||||
if omitConnectTunnel:
|
||||
warnings.warn(
|
||||
"Using HTTPS proxies in the noconnect mode is deprecated. "
|
||||
"If you use Zyte Smart Proxy Manager, it doesn't require "
|
||||
"this mode anymore, so you should update scrapy-crawlera "
|
||||
"to scrapy-zyte-smartproxy and remove '?noconnect' "
|
||||
"from the Zyte Smart Proxy Manager URL.",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
if scheme == b'https' and not omitConnectTunnel:
|
||||
if scheme == b'https':
|
||||
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
|
||||
proxyConf = (proxyHost, proxyPort, proxyAuth)
|
||||
return self._TunnelingAgent(
|
||||
|
|
@ -302,8 +291,6 @@ class ScrapyAgent:
|
|||
)
|
||||
else:
|
||||
proxyScheme = proxyScheme or b'http'
|
||||
proxyHost = to_bytes(proxyHost, encoding='ascii')
|
||||
proxyPort = to_bytes(str(proxyPort), encoding='ascii')
|
||||
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
|
||||
return self._ProxyAgent(
|
||||
reactor=reactor,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import warnings
|
||||
from time import time
|
||||
from typing import Optional, Type, TypeVar
|
||||
from urllib.parse import urldefrag
|
||||
|
|
@ -69,19 +68,8 @@ class ScrapyH2Agent:
|
|||
if proxy:
|
||||
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
|
||||
scheme = _parse(request.url)[0]
|
||||
proxy_host = proxy_host.decode()
|
||||
omit_connect_tunnel = b'noconnect' in proxy_params
|
||||
if omit_connect_tunnel:
|
||||
warnings.warn(
|
||||
"Using HTTPS proxies in the noconnect mode is not "
|
||||
"supported by the downloader handler. If you use Zyte "
|
||||
"Smart Proxy Manager, it doesn't require this mode "
|
||||
"anymore, so you should update scrapy-crawlera to "
|
||||
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
|
||||
"Zyte Smart Proxy Manager URL."
|
||||
)
|
||||
|
||||
if scheme == b'https' and not omit_connect_tunnel:
|
||||
if scheme == b'https':
|
||||
# ToDo
|
||||
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
|
||||
return self._ProxyAgent(
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
|
|||
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
|
||||
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
|
||||
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Headers, HtmlResponse, Request
|
||||
from scrapy.http.response.text import TextResponse
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
|
@ -757,18 +757,6 @@ class HttpProxyTestCase(unittest.TestCase):
|
|||
request = Request('http://example.com', meta={'proxy': http_proxy})
|
||||
return self.download_request(request, Spider('foo')).addCallback(_test)
|
||||
|
||||
def test_download_with_proxy_https_noconnect(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(response.url, request.url)
|
||||
self.assertEqual(response.body, b'https://example.com')
|
||||
|
||||
http_proxy = f'{self.getURL("")}?noconnect'
|
||||
request = Request('https://example.com', meta={'proxy': http_proxy})
|
||||
with self.assertWarnsRegex(ScrapyDeprecationWarning,
|
||||
r'Using HTTPS proxies in the noconnect mode is deprecated'):
|
||||
return self.download_request(request, Spider('foo')).addCallback(_test)
|
||||
|
||||
def test_download_without_proxy(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.status, 200)
|
||||
|
|
|
|||
|
|
@ -242,21 +242,6 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
|
|||
def getURL(self, path):
|
||||
return f"{self.scheme}://{self.host}:{self.portno}/{path}"
|
||||
|
||||
def test_download_with_proxy_https_noconnect(self):
|
||||
def _test(response):
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(response.url, request.url)
|
||||
self.assertEqual(response.body, b'/')
|
||||
|
||||
http_proxy = f"{self.getURL('')}?noconnect"
|
||||
request = Request('https://example.com', meta={'proxy': http_proxy})
|
||||
with self.assertWarnsRegex(
|
||||
Warning,
|
||||
r'Using HTTPS proxies in the noconnect mode is not supported by the '
|
||||
r'downloader handler.'
|
||||
):
|
||||
return self.download_request(request, Spider('foo')).addCallback(_test)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_download_with_proxy_https_timeout(self):
|
||||
with self.assertRaises(NotImplementedError):
|
||||
|
|
|
|||
Loading…
Reference in New Issue