Remove `noconnect` deprecate code

This commit is contained in:
Laerte Pereira 2022-11-23 08:30:11 -03:00
parent 45c2bd7d9c
commit e769532644
No known key found for this signature in database
GPG Key ID: BB69B5C731BB8A80
4 changed files with 4 additions and 56 deletions

View File

@ -3,7 +3,6 @@
import ipaddress
import logging
import re
import warnings
from contextlib import suppress
from io import BytesIO
from time import time
@ -22,7 +21,7 @@ from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.exceptions import StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.python import to_bytes, to_unicode
@ -279,17 +278,7 @@ class ScrapyAgent:
proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
if omitConnectTunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is deprecated. "
"If you use Zyte Smart Proxy Manager, it doesn't require "
"this mode anymore, so you should update scrapy-crawlera "
"to scrapy-zyte-smartproxy and remove '?noconnect' "
"from the Zyte Smart Proxy Manager URL.",
ScrapyDeprecationWarning,
)
if scheme == b'https' and not omitConnectTunnel:
if scheme == b'https':
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
return self._TunnelingAgent(
@ -302,8 +291,6 @@ class ScrapyAgent:
)
else:
proxyScheme = proxyScheme or b'http'
proxyHost = to_bytes(proxyHost, encoding='ascii')
proxyPort = to_bytes(str(proxyPort), encoding='ascii')
proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
return self._ProxyAgent(
reactor=reactor,

View File

@ -1,4 +1,3 @@
import warnings
from time import time
from typing import Optional, Type, TypeVar
from urllib.parse import urldefrag
@ -69,19 +68,8 @@ class ScrapyH2Agent:
if proxy:
_, _, proxy_host, proxy_port, proxy_params = _parse(proxy)
scheme = _parse(request.url)[0]
proxy_host = proxy_host.decode()
omit_connect_tunnel = b'noconnect' in proxy_params
if omit_connect_tunnel:
warnings.warn(
"Using HTTPS proxies in the noconnect mode is not "
"supported by the downloader handler. If you use Zyte "
"Smart Proxy Manager, it doesn't require this mode "
"anymore, so you should update scrapy-crawlera to "
"scrapy-zyte-smartproxy and remove '?noconnect' from the "
"Zyte Smart Proxy Manager URL."
)
if scheme == b'https' and not omit_connect_tunnel:
if scheme == b'https':
# ToDo
raise NotImplementedError('Tunneling via CONNECT method using HTTP/2.0 is not yet supported')
return self._ProxyAgent(

View File

@ -24,7 +24,7 @@ from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import NotConfigured
from scrapy.http import Headers, HtmlResponse, Request
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
@ -757,18 +757,6 @@ class HttpProxyTestCase(unittest.TestCase):
request = Request('http://example.com', meta={'proxy': http_proxy})
return self.download_request(request, Spider('foo')).addCallback(_test)
def test_download_with_proxy_https_noconnect(self):
def _test(response):
self.assertEqual(response.status, 200)
self.assertEqual(response.url, request.url)
self.assertEqual(response.body, b'https://example.com')
http_proxy = f'{self.getURL("")}?noconnect'
request = Request('https://example.com', meta={'proxy': http_proxy})
with self.assertWarnsRegex(ScrapyDeprecationWarning,
r'Using HTTPS proxies in the noconnect mode is deprecated'):
return self.download_request(request, Spider('foo')).addCallback(_test)
def test_download_without_proxy(self):
def _test(response):
self.assertEqual(response.status, 200)

View File

@ -242,21 +242,6 @@ class Https2ProxyTestCase(Http11ProxyTestCase):
def getURL(self, path):
return f"{self.scheme}://{self.host}:{self.portno}/{path}"
def test_download_with_proxy_https_noconnect(self):
def _test(response):
self.assertEqual(response.status, 200)
self.assertEqual(response.url, request.url)
self.assertEqual(response.body, b'/')
http_proxy = f"{self.getURL('')}?noconnect"
request = Request('https://example.com', meta={'proxy': http_proxy})
with self.assertWarnsRegex(
Warning,
r'Using HTTPS proxies in the noconnect mode is not supported by the '
r'downloader handler.'
):
return self.download_request(request, Spider('foo')).addCallback(_test)
@defer.inlineCallbacks
def test_download_with_proxy_https_timeout(self):
with self.assertRaises(NotImplementedError):