From 63546cbf3e02380819732441ad55d95725dca7c5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 27 Nov 2019 22:42:52 +0500 Subject: [PATCH 1/4] Deprecate the HTTPS proxy noconnect mode. --- scrapy/core/downloader/handlers/http11.py | 7 ++++++ tests/test_downloader_handlers.py | 10 --------- tests/test_proxy_connect.py | 26 ----------------------- 3 files changed, 7 insertions(+), 36 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7d917cb74..782eca89e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse @@ -285,6 +286,12 @@ class ScrapyAgent(object): scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams + if omitConnectTunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 60124b93f..45d4aa952 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -687,16 +687,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') - - http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 277455751..05d371b65 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -108,32 +108,6 @@ class ProxyConnectTestCase(TestCase): echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) - # The noconnect mode isn't supported by the current mitmproxy, it returns - # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all, - # and it's not clear what behavior is intended by Scrapy and by mitmproxy here. - # https://github.com/mitmproxy/mitmproxy/issues/848 may be related. - # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera, - # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related. - - @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') - @defer.inlineCallbacks - def test_https_noconnect(self): - proxy = os.environ['https_proxy'] - os.environ['https_proxy'] = proxy + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - - @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') - @defer.inlineCallbacks - def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, l) - def _assert_got_response_code(self, code, log): print(log) self.assertEqual(str(log).count('Crawled (%d)' % code), 1) From 2d92a39003a5a0b01b265b87cf722a61f121ca60 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 18 Dec 2019 12:07:08 +0500 Subject: [PATCH 2/4] Restore test_download_with_proxy_https_noconnect, check for a warning there. --- tests/test_downloader_handlers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45d4aa952..87ab4c9e5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -33,7 +33,7 @@ from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider @@ -687,6 +687,18 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'https://example.com') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex(ScrapyDeprecationWarning, + r'Using HTTPS proxies in the noconnect mode is deprecated'): + return self.download_request(request, Spider('foo')).addCallback(_test) + def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) From bb2ff13e4c7c80cfc7925f60eadc97dec0d69026 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 18 Dec 2019 15:39:08 +0500 Subject: [PATCH 3/4] Skip Http10ProxyTestCase.test_download_with_proxy_https_noconnect --- tests/test_downloader_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 87ab4c9e5..412a9c084 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -712,6 +712,8 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP10DownloadHandler + def test_download_with_proxy_https_noconnect(self): + raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler From ac302c3f615d339ed36000231ca3e1e1c347478a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 18 Dec 2019 15:43:05 +0500 Subject: [PATCH 4/4] Fix a flake8 problem. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 412a9c084..7412d7ebf 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -715,6 +715,7 @@ class Http10ProxyTestCase(HttpProxyTestCase): def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') + class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler