From b44bd6f8250579dc9ddc25d5e0f10d8b3790f5ea Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 20:51:03 +0500
Subject: [PATCH 01/11] Remove Python 2-only tests.

---
 conftest.py                             |  10 +-
 tests/py3-ignores.txt                   |   3 -
 tests/test_downloader_handlers.py       |  22 +--
 tests/test_linkextractors_deprecated.py | 233 ------------------------
 tests/test_proxy_connect.py             | 120 ------------
 tests/test_utils_python.py              |  27 ---
 tests/test_webclient.py                 |  20 --
 7 files changed, 14 insertions(+), 421 deletions(-)
 delete mode 100644 tests/test_linkextractors_deprecated.py
 delete mode 100644 tests/test_proxy_connect.py

diff --git a/conftest.py b/conftest.py
index 06d65ba1d..ede091e9f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,3 @@
-import six
 import pytest
 
 
@@ -8,11 +7,10 @@ collect_ignore = [
 ]
 
 
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+for line in open('tests/py3-ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 313e74ec9..45cf6fb92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,3 @@
-tests/test_linkextractors_deprecated.py
-tests/test_proxy_connect.py
-
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..4d3c4d4aa 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -633,18 +633,16 @@ class Http11MockServerTestCase(unittest.TestCase):
         # download_maxsize < 100, hence the CancelledError
         self.assertIsInstance(failure.value, defer.CancelledError)
 
-        if six.PY2:
-            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-            request = request.replace(url=self.mockserver.url('/xpayload'))
-            yield crawler.crawl(seed=request)
-            # download_maxsize = 50 is enough for the gzipped response
-            failure = crawler.spider.meta.get('failure')
-            self.assertTrue(failure == None)
-            reason = crawler.spider.meta['close_reason']
-            self.assertTrue(reason, 'finished')
-        else:
-            # See issue https://twistedmatrix.com/trac/ticket/8175
-            raise unittest.SkipTest("xpayload only enabled for PY2")
+        # See issue https://twistedmatrix.com/trac/ticket/8175
+        raise unittest.SkipTest("xpayload only enabled for PY2")
+        request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+        request = request.replace(url=self.mockserver.url('/xpayload'))
+        yield crawler.crawl(seed=request)
+        # download_maxsize = 50 is enough for the gzipped response
+        failure = crawler.spider.meta.get('failure')
+        self.assertTrue(failure == None)
+        reason = crawler.spider.meta['close_reason']
+        self.assertTrue(reason, 'finished')
 
 
 class UriResource(resource.Resource):
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
deleted file mode 100644
index 1366971be..000000000
--- a/tests/test_linkextractors_deprecated.py
+++ /dev/null
@@ -1,233 +0,0 @@
-# -*- coding: utf-8 -*-
-import unittest
-from scrapy.linkextractors.regex import RegexLinkExtractor
-from scrapy.http import HtmlResponse
-from scrapy.link import Link
-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
-from tests import get_testdata
-
-from tests.test_linkextractors import Base
-
-
-class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
-    # XXX: should we move some of these tests to base link extractor tests?
-
-    def test_basic(self):
-        html = """<html><head><title>Page title<title>
-        <body><p><a href="item/12.html">Item 12</a></p>
-        <p><a href="/about.html">About us</a></p>
-        <img src="/logo.png" alt="Company logo (not a link)" />
-        <p><a href="../othercat.html">Other category</a></p>
-        <p><a href="/">&gt;&gt;</a></p>
-        <p><a href="/" /></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
-                          Link(url='http://example.org/about.html', text='About us'),
-                          Link(url='http://example.org/othercat.html', text='Other category'),
-                          Link(url='http://example.org/', text='>>'),
-                          Link(url='http://example.org/', text='')])
-
-    def test_base_url(self):
-        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
-        <body><p><a href="item/12.html">Item 12</a></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
-
-        # base url is an absolute path and relative to host
-        html = """<html><head><title>Page title<title><base href="/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://example.org/item/12.html', text='Item 12')])
-
-        # base url has no scheme
-        html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
-
-    def test_link_text_wrong_encoding(self):
-        html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
-        response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
-        ])
-
-    def test_extraction_encoding(self):
-        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
-        response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
-        response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
-        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
-        response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response_utf8), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        self.assertEqual(lx.extract_links(response_noenc), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        # document encoding does not affect URL path component, only query part
-        # >>> u'sample_ñ.html'.encode('utf8')
-        # b'sample_\xc3\xb1.html'
-        # >>> u"sample_á.html".encode('utf8')
-        # b'sample_\xc3\xa1.html'
-        # >>> u"sample_ö.html".encode('utf8')
-        # b'sample_\xc3\xb6.html'
-        # >>> u"£32".encode('latin1')
-        # b'\xa332'
-        # >>> u"µ".encode('latin1')
-        # b'\xb5'
-        self.assertEqual(lx.extract_links(response_latin1), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
-            Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
-        ])
-
-    def test_matches(self):
-        url1 = 'http://lotsofstuff.com/stuff1/index'
-        url2 = 'http://evenmorestuff.com/uglystuff/index'
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), True)
-
-
-class HtmlParserLinkExtractorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-            Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-            Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
-        ])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-
-class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
-    extractor_cls = SgmlLinkExtractor
-    escapes_whitespace = True
-
-    def test_deny_extensions(self):
-        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
-        response = HtmlResponse("http://example.org/", body=html)
-        lx = SgmlLinkExtractor(deny_extensions="jpg")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
-        ])
-
-    def test_attrs_sgml(self):
-        html = """<html><area href="sample1.html"></area>
-        <a ref="sample2.html">sample text 2</a></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-        lx = SgmlLinkExtractor(attrs="href")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
-    def test_link_nofollow(self):
-        html = """
-        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
-        <a href="about.html">About us</a>
-        <a href="http://google.com/something" rel="external nofollow">Something</a>
-        """
-        response = HtmlResponse("http://example.org/page.html", body=html)
-        lx = SgmlLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
-            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
-            Link(url='http://google.com/something', text=u'Something', nofollow=True),
-        ])
-
-
-class RegexLinkExtractorTestCase(unittest.TestCase):
-    # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
-    # not to depend on SgmlLinkExractor. Its speed is also much worse
-    # than it should be.
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = RegexLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-    def test_html_base_href(self):
-        html = """
-        <html>
-            <head>
-                <base href="http://b.com/">
-            </head>
-            <body>
-                <a href="test.html"></a>
-            </body>
-        </html>
-        """
-        response = HtmlResponse("http://a.com/", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://b.com/test.html', text=u'', nofollow=False),
-        ])
-
-    @unittest.expectedFailure
-    def test_extraction(self):
-        # RegexLinkExtractor doesn't parse URLs with leading/trailing
-        # whitespaces correctly.
-        super(RegexLinkExtractorTestCase, self).test_extraction()
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
deleted file mode 100644
index ae1236bcb..000000000
--- a/tests/test_proxy_connect.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import json
-import os
-import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
-from threading import Thread
-from libmproxy import controller, proxy
-from netlib import http_auth
-from testfixtures import LogCapture
-
-from twisted.internet import defer
-from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
-from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
-from tests.mockserver import MockServer
-
-
-class HTTPSProxy(controller.Master, Thread):
-
-    def __init__(self):
-        password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
-        authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
-        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
-        server = proxy.ProxyServer(proxy.ProxyConfig(
-            authenticator = authenticator,
-            cacert = cert_path),
-            0)
-        self.server = server
-        Thread.__init__(self)
-        controller.Master.__init__(self, server)
-
-    def http_address(self):
-        return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
-
-
-def _wrong_credentials(proxy_url):
-    bad_auth_proxy = list(urlsplit(proxy_url))
-    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
-    return urlunsplit(bad_auth_proxy)
-
-class ProxyConnectTestCase(TestCase):
-
-    def setUp(self):
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
-        self._oldenv = os.environ.copy()
-
-        self._proxy = HTTPSProxy()
-        self._proxy.start()
-
-        # Wait for the proxy to start.
-        time.sleep(1.0)
-        os.environ['https_proxy'] = self._proxy.http_address()
-        os.environ['http_proxy'] = self._proxy.http_address()
-
-    def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
-        self._proxy.shutdown()
-        os.environ = self._oldenv
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_noconnect(self):
-        proxy = os.environ['https_proxy']
-        os.environ['https_proxy'] = proxy + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel_error(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        # The proxy returns a 407 error code but it does not reach the client;
-        # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_without_leak_proxy_authorization_header(self):
-        request = Request(self.mockserver.url("/echo", is_secure=True))
-        crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
-        self.assertTrue('Proxy-Authorization' not in echo['headers'])
-
-    @defer.inlineCallbacks
-    def test_https_noconnect_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
-
-    def _assert_got_response_code(self, code, log):
-        print(log)
-        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
-
-    def _assert_got_tunnel_error(self, log):
-        print(log)
-        self.assertIn('TunnelError', str(log))
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..6cb32cbdd 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase):
                 gc.collect()
         self.assertFalse(len(wk._weakdict))
 
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict(self):
-        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_tuples(self):
-        tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
-        d = dict(tuples)
-        d2 = stringify_dict(tuples, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_keys_only(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..7b015ff8d 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
-    def test_externalUnicodeInterference(self):
-        """
-        L{client._parse} should return C{str} for the scheme, host, and path
-        elements of its return tuple, even when passed an URL which has
-        previously been passed to L{urlparse} as a C{unicode} string.
-        """
-        if not six.PY2:
-            raise unittest.SkipTest(
-                "Applies only to Py2, as urls can be ONLY unicode on Py3")
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
-        self._parse(badInput)  # cache badInput in urlparse_cached
-        scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
-        self.assertTrue(isinstance(port, int))
-
-
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
 

From b0d6f4917d782f2396a702e90c36ffbc8bb42e7e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:17:03 +0500
Subject: [PATCH 02/11] Restore tests/test_proxy_connect.py and update it to
 modern mitmproxy.

---
 tests/requirements-py3.txt  |   1 +
 tests/test_proxy_connect.py | 134 ++++++++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 tests/test_proxy_connect.py

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dd5b23cc3..f27e45a54 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,5 +1,6 @@
 # Tests requirements
 jmespath
+mitmproxy
 pytest
 pytest-cov
 pytest-twisted
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
new file mode 100644
index 000000000..8142d9a41
--- /dev/null
+++ b/tests/test_proxy_connect.py
@@ -0,0 +1,134 @@
+import json
+import os
+import re
+from subprocess import Popen, PIPE
+import sys
+import time
+
+from six.moves.urllib.parse import urlsplit, urlunsplit
+from testfixtures import LogCapture
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.mockserver import MockServer
+
+
+class MitmProxy:
+    auth_user = 'scrapy'
+    auth_pass = 'scrapy'
+
+    def start(self):
+        from scrapy.utils.test import get_testenv
+        script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+        """
+        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+            'keys', 'mitmproxy-ca.pem')
+        self.proc = Popen([sys.executable,
+                           '-c', script,
+                           '--listen-host', '127.0.0.1',
+                           '--listen-port', '0',
+                           '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+                           '--certs', cert_path,
+                           '--ssl-insecure',
+                           ],
+                           stdout=PIPE, env=get_testenv())
+        line = self.proc.stdout.readline().decode('utf-8')
+        host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
+        address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
+        return address
+
+    def stop(self):
+        self.proc.kill()
+        self.proc.wait()
+        time.sleep(0.2)
+
+
+def _wrong_credentials(proxy_url):
+    bad_auth_proxy = list(urlsplit(proxy_url))
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+    return urlunsplit(bad_auth_proxy)
+
+
+class ProxyConnectTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self._oldenv = os.environ.copy()
+
+        self._proxy = MitmProxy()
+        proxy_url = self._proxy.start()
+        os.environ['https_proxy'] = proxy_url
+        os.environ['http_proxy'] = proxy_url
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+        self._proxy.stop()
+        os.environ = self._oldenv
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_noconnect(self):
+        proxy = os.environ['https_proxy']
+        os.environ['https_proxy'] = proxy + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel_error(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:99999/status?n=200")
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        # The proxy returns a 407 error code but it does not reach the client;
+        # he just sees a TunnelError.
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_without_leak_proxy_authorization_header(self):
+        request = Request(self.mockserver.url("/echo", is_secure=True))
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(seed=request)
+        self._assert_got_response_code(200, l)
+        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        self.assertTrue('Proxy-Authorization' not in echo['headers'])
+
+    @defer.inlineCallbacks
+    def test_https_noconnect_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(407, l)
+
+    def _assert_got_response_code(self, code, log):
+        print(log)
+        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
+
+    def _assert_got_tunnel_error(self, log):
+        print(log)
+        self.assertIn('TunnelError', str(log))

From 439e37fc7b2c192b4591d29dde74c9f589763cc8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:23:24 +0500
Subject: [PATCH 03/11] Mark failing proxy tests.

---
 tests/test_proxy_connect.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 8142d9a41..5e9470e39 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -5,6 +5,7 @@ from subprocess import Popen, PIPE
 import sys
 import time
 
+import pytest
 from six.moves.urllib.parse import urlsplit, urlunsplit
 from testfixtures import LogCapture
 
@@ -81,6 +82,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect(self):
         proxy = os.environ['https_proxy']
@@ -90,6 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='Python 3 fails this earlier')
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
@@ -117,6 +120,7 @@ class ProxyConnectTestCase(TestCase):
         echo = json.loads(crawler.spider.meta['responses'][0].body)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'

From 186f9d88acd9043cf5d8f21358c758c42e433105 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:24:06 +0500
Subject: [PATCH 04/11] Fix the skip message for test_download_gzip_response.

---
 tests/test_downloader_handlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4d3c4d4aa..9f74577a5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -634,7 +634,7 @@ class Http11MockServerTestCase(unittest.TestCase):
         self.assertIsInstance(failure.value, defer.CancelledError)
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
-        raise unittest.SkipTest("xpayload only enabled for PY2")
+        raise unittest.SkipTest("xpayload fails on PY3")
         request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
         request = request.replace(url=self.mockserver.url('/xpayload'))
         yield crawler.crawl(seed=request)

From 3ac4b430ae0165f25d8cd2fb1ac25ac9f307df8b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 15:20:28 +0500
Subject: [PATCH 05/11] Remove an unused six import.

---
 tests/test_downloader_handlers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 9f74577a5..e6856945c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,4 @@
 import os
-import six
 import shutil
 import tempfile
 import contextlib

From a5eb59b92d3311f702933753d85b15e6d698faa1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:14 +0500
Subject: [PATCH 06/11] Fix test_proxy_connect.py for py3.5.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 5e9470e39..f6381b5b1 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 5eb01b617d207aabc705038d8a0e3b46e358bdbb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:30 +0500
Subject: [PATCH 07/11] Use an older mitmproxy for py3.5.

---
 tests/requirements-py3.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index f27e45a54..c2b16bec6 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,7 @@
 # Tests requirements
 jmespath
-mitmproxy
+mitmproxy; python_version >= '3.6'
+mitmproxy==3.0.4; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted

From e0c5c724969ebc35970daaf0203969dc2eb00d56 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:19 +0500
Subject: [PATCH 08/11] Improve the
 test_https_tunnel_without_leak_proxy_authorization_header change.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index f6381b5b1..651576c2c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
+        echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 3c9963ab049e49a1d490d9505ad22ae9c1415421 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:38 +0500
Subject: [PATCH 09/11] Only xfail test_https_connect_tunnel_error on 3.6+.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 651576c2c..ec3f0716c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -92,7 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
-    @pytest.mark.xfail(reason='Python 3 fails this earlier')
+    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)

From 4b0cdf7f3ed94e81612cceacfab6d81d54117865 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:50:56 +0500
Subject: [PATCH 10/11] Use self.proc.communicate() after killing mitmdump.

---
 tests/test_proxy_connect.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ec3f0716c..69925f80c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -48,8 +48,7 @@ sys.exit(mitmdump())
 
     def stop(self):
         self.proc.kill()
-        self.proc.wait()
-        time.sleep(0.2)
+        self.proc.communicate()
 
 
 def _wrong_credentials(proxy_url):

From 350aa67c3dc8997ef9d3aac9ef3f596c83758e1c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:52:57 +0500
Subject: [PATCH 11/11] Rename tests/py3-ignores.txt to tests/ignores.txt.

---
 conftest.py                            | 2 +-
 tests/{py3-ignores.txt => ignores.txt} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{py3-ignores.txt => ignores.txt} (100%)

diff --git a/conftest.py b/conftest.py
index ede091e9f..7da4c4976 100644
--- a/conftest.py
+++ b/conftest.py
@@ -7,7 +7,7 @@ collect_ignore = [
 ]
 
 
-for line in open('tests/py3-ignores.txt'):
+for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
diff --git a/tests/py3-ignores.txt b/tests/ignores.txt
similarity index 100%
rename from tests/py3-ignores.txt
rename to tests/ignores.txt