From beb7d80d6a8a82baf9cb8170758704b4ff5c63cf Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Nov 2019 22:47:35 +0500 Subject: [PATCH] Add a comment about the noconnect tests. --- tests/test_proxy_connect.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 2435999f9..277455751 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -80,16 +80,6 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') - @defer.inlineCallbacks - def test_https_noconnect(self): - proxy = os.environ['https_proxy'] - os.environ['https_proxy'] = proxy + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): @@ -118,6 +108,23 @@ class ProxyConnectTestCase(TestCase): echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) + # The noconnect mode isn't supported by the current mitmproxy, it returns + # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all, + # and it's not clear what behavior is intended by Scrapy and by mitmproxy here. + # https://github.com/mitmproxy/mitmproxy/issues/848 may be related. + # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera, + # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related. + + @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') + @defer.inlineCallbacks + def test_https_noconnect(self): + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) + self._assert_got_response_code(200, l) + @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') @defer.inlineCallbacks def test_https_noconnect_auth_error(self):