From b44bd6f8250579dc9ddc25d5e0f10d8b3790f5ea Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 20:51:03 +0500
Subject: [PATCH 01/49] Remove Python 2-only tests.

---
 conftest.py                             |  10 +-
 tests/py3-ignores.txt                   |   3 -
 tests/test_downloader_handlers.py       |  22 +--
 tests/test_linkextractors_deprecated.py | 233 ------------------------
 tests/test_proxy_connect.py             | 120 ------------
 tests/test_utils_python.py              |  27 ---
 tests/test_webclient.py                 |  20 --
 7 files changed, 14 insertions(+), 421 deletions(-)
 delete mode 100644 tests/test_linkextractors_deprecated.py
 delete mode 100644 tests/test_proxy_connect.py

diff --git a/conftest.py b/conftest.py
index 06d65ba1d..ede091e9f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,3 @@
-import six
 import pytest
 
 
@@ -8,11 +7,10 @@ collect_ignore = [
 ]
 
 
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+for line in open('tests/py3-ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 313e74ec9..45cf6fb92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -1,6 +1,3 @@
-tests/test_linkextractors_deprecated.py
-tests/test_proxy_connect.py
-
 scrapy/linkextractors/sgml.py
 scrapy/linkextractors/regex.py
 scrapy/linkextractors/htmlparser.py
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..4d3c4d4aa 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -633,18 +633,16 @@ class Http11MockServerTestCase(unittest.TestCase):
         # download_maxsize < 100, hence the CancelledError
         self.assertIsInstance(failure.value, defer.CancelledError)
 
-        if six.PY2:
-            request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
-            request = request.replace(url=self.mockserver.url('/xpayload'))
-            yield crawler.crawl(seed=request)
-            # download_maxsize = 50 is enough for the gzipped response
-            failure = crawler.spider.meta.get('failure')
-            self.assertTrue(failure == None)
-            reason = crawler.spider.meta['close_reason']
-            self.assertTrue(reason, 'finished')
-        else:
-            # See issue https://twistedmatrix.com/trac/ticket/8175
-            raise unittest.SkipTest("xpayload only enabled for PY2")
+        # See issue https://twistedmatrix.com/trac/ticket/8175
+        raise unittest.SkipTest("xpayload only enabled for PY2")
+        request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
+        request = request.replace(url=self.mockserver.url('/xpayload'))
+        yield crawler.crawl(seed=request)
+        # download_maxsize = 50 is enough for the gzipped response
+        failure = crawler.spider.meta.get('failure')
+        self.assertTrue(failure == None)
+        reason = crawler.spider.meta['close_reason']
+        self.assertTrue(reason, 'finished')
 
 
 class UriResource(resource.Resource):
diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py
deleted file mode 100644
index 1366971be..000000000
--- a/tests/test_linkextractors_deprecated.py
+++ /dev/null
@@ -1,233 +0,0 @@
-# -*- coding: utf-8 -*-
-import unittest
-from scrapy.linkextractors.regex import RegexLinkExtractor
-from scrapy.http import HtmlResponse
-from scrapy.link import Link
-from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor
-from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
-from tests import get_testdata
-
-from tests.test_linkextractors import Base
-
-
-class BaseSgmlLinkExtractorTestCase(unittest.TestCase):
-    # XXX: should we move some of these tests to base link extractor tests?
-
-    def test_basic(self):
-        html = """<html><head><title>Page title<title>
-        <body><p><a href="item/12.html">Item 12</a></p>
-        <p><a href="/about.html">About us</a></p>
-        <img src="/logo.png" alt="Company logo (not a link)" />
-        <p><a href="../othercat.html">Other category</a></p>
-        <p><a href="/">&gt;&gt;</a></p>
-        <p><a href="/" /></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
-                          Link(url='http://example.org/about.html', text='About us'),
-                          Link(url='http://example.org/othercat.html', text='Other category'),
-                          Link(url='http://example.org/', text='>>'),
-                          Link(url='http://example.org/', text='')])
-
-    def test_base_url(self):
-        html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
-        <body><p><a href="item/12.html">Item 12</a></p>
-        </body></html>"""
-        response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
-        lx = BaseSgmlLinkExtractor()  # default: tag=a, attr=href
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
-
-        # base url is an absolute path and relative to host
-        html = """<html><head><title>Page title<title><base href="/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://example.org/item/12.html', text='Item 12')])
-
-        # base url has no scheme
-        html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
-        <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
-        response = HtmlResponse("https://example.org/somepage/index.html", body=html)
-        self.assertEqual(lx.extract_links(response),
-                         [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
-
-    def test_link_text_wrong_encoding(self):
-        html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
-        response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
-        ])
-
-    def test_extraction_encoding(self):
-        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
-        response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
-        response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
-        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
-        response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.extract_links(response_utf8), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        self.assertEqual(lx.extract_links(response_noenc), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
-        ])
-
-        # document encoding does not affect URL path component, only query part
-        # >>> u'sample_ñ.html'.encode('utf8')
-        # b'sample_\xc3\xb1.html'
-        # >>> u"sample_á.html".encode('utf8')
-        # b'sample_\xc3\xa1.html'
-        # >>> u"sample_ö.html".encode('utf8')
-        # b'sample_\xc3\xb6.html'
-        # >>> u"£32".encode('latin1')
-        # b'\xa332'
-        # >>> u"µ".encode('latin1')
-        # b'\xb5'
-        self.assertEqual(lx.extract_links(response_latin1), [
-            Link(url='http://example.com/sample_%C3%B1.html', text=''),
-            Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
-            Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
-        ])
-
-    def test_matches(self):
-        url1 = 'http://lotsofstuff.com/stuff1/index'
-        url2 = 'http://evenmorestuff.com/uglystuff/index'
-
-        lx = BaseSgmlLinkExtractor()
-        self.assertEqual(lx.matches(url1), True)
-        self.assertEqual(lx.matches(url2), True)
-
-
-class HtmlParserLinkExtractorTestCase(unittest.TestCase):
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response), [
-            Link(url='http://example.com/sample2.html', text=u'sample 2'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-            Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
-            Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-            Link(url='http://www.google.com/something', text=u''),
-            Link(url='http://example.com/innertag.html', text=u'inner tag'),
-            Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
-        ])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = HtmlParserLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-
-class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
-    extractor_cls = SgmlLinkExtractor
-    escapes_whitespace = True
-
-    def test_deny_extensions(self):
-        html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
-        response = HtmlResponse("http://example.org/", body=html)
-        lx = SgmlLinkExtractor(deny_extensions="jpg")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.org/page.html', text=u'asd'),
-        ])
-
-    def test_attrs_sgml(self):
-        html = """<html><area href="sample1.html"></area>
-        <a ref="sample2.html">sample text 2</a></html>"""
-        response = HtmlResponse("http://example.com/index.html", body=html)
-        lx = SgmlLinkExtractor(attrs="href")
-        self.assertEqual(lx.extract_links(response), [
-            Link(url='http://example.com/sample1.html', text=u''),
-        ])
-
-    def test_link_nofollow(self):
-        html = """
-        <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
-        <a href="about.html">About us</a>
-        <a href="http://google.com/something" rel="external nofollow">Something</a>
-        """
-        response = HtmlResponse("http://example.org/page.html", body=html)
-        lx = SgmlLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
-            Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
-            Link(url='http://google.com/something', text=u'Something', nofollow=True),
-        ])
-
-
-class RegexLinkExtractorTestCase(unittest.TestCase):
-    # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
-    # not to depend on SgmlLinkExractor. Its speed is also much worse
-    # than it should be.
-
-    def setUp(self):
-        body = get_testdata('link_extractor', 'sgml_linkextractor.html')
-        self.response = HtmlResponse(url='http://example.com/index', body=body)
-
-    def test_extraction(self):
-        # Default arguments
-        lx = RegexLinkExtractor()
-        self.assertEqual(lx.extract_links(self.response),
-                         [Link(url='http://example.com/sample2.html', text=u'sample 2'),
-                          Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
-                          Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
-                          Link(url='http://www.google.com/something', text=u''),
-                          Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
-    def test_link_wrong_href(self):
-        html = """
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
-            Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
-        ])
-
-    def test_html_base_href(self):
-        html = """
-        <html>
-            <head>
-                <base href="http://b.com/">
-            </head>
-            <body>
-                <a href="test.html"></a>
-            </body>
-        </html>
-        """
-        response = HtmlResponse("http://a.com/", body=html)
-        lx = RegexLinkExtractor()
-        self.assertEqual([link for link in lx.extract_links(response)], [
-            Link(url='http://b.com/test.html', text=u'', nofollow=False),
-        ])
-
-    @unittest.expectedFailure
-    def test_extraction(self):
-        # RegexLinkExtractor doesn't parse URLs with leading/trailing
-        # whitespaces correctly.
-        super(RegexLinkExtractorTestCase, self).test_extraction()
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
deleted file mode 100644
index ae1236bcb..000000000
--- a/tests/test_proxy_connect.py
+++ /dev/null
@@ -1,120 +0,0 @@
-import json
-import os
-import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
-from threading import Thread
-from libmproxy import controller, proxy
-from netlib import http_auth
-from testfixtures import LogCapture
-
-from twisted.internet import defer
-from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
-from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
-from tests.mockserver import MockServer
-
-
-class HTTPSProxy(controller.Master, Thread):
-
-    def __init__(self):
-        password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
-        authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
-        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
-        server = proxy.ProxyServer(proxy.ProxyConfig(
-            authenticator = authenticator,
-            cacert = cert_path),
-            0)
-        self.server = server
-        Thread.__init__(self)
-        controller.Master.__init__(self, server)
-
-    def http_address(self):
-        return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
-
-
-def _wrong_credentials(proxy_url):
-    bad_auth_proxy = list(urlsplit(proxy_url))
-    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
-    return urlunsplit(bad_auth_proxy)
-
-class ProxyConnectTestCase(TestCase):
-
-    def setUp(self):
-        self.mockserver = MockServer()
-        self.mockserver.__enter__()
-        self._oldenv = os.environ.copy()
-
-        self._proxy = HTTPSProxy()
-        self._proxy.start()
-
-        # Wait for the proxy to start.
-        time.sleep(1.0)
-        os.environ['https_proxy'] = self._proxy.http_address()
-        os.environ['http_proxy'] = self._proxy.http_address()
-
-    def tearDown(self):
-        self.mockserver.__exit__(None, None, None)
-        self._proxy.shutdown()
-        os.environ = self._oldenv
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_noconnect(self):
-        proxy = os.environ['https_proxy']
-        os.environ['https_proxy'] = proxy + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
-
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel_error(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        # The proxy returns a 407 error code but it does not reach the client;
-        # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
-
-    @defer.inlineCallbacks
-    def test_https_tunnel_without_leak_proxy_authorization_header(self):
-        request = Request(self.mockserver.url("/echo", is_secure=True))
-        crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
-        self.assertTrue('Proxy-Authorization' not in echo['headers'])
-
-    @defer.inlineCallbacks
-    def test_https_noconnect_auth_error(self):
-        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
-
-    def _assert_got_response_code(self, code, log):
-        print(log)
-        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
-
-    def _assert_got_tunnel_error(self, log):
-        print(log)
-        self.assertIn('TunnelError', str(log))
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..6cb32cbdd 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase):
                 gc.collect()
         self.assertFalse(len(wk._weakdict))
 
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict(self):
-        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_tuples(self):
-        tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
-        d = dict(tuples)
-        d2 = stringify_dict(tuples, keys_only=False)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
-    @unittest.skipUnless(six.PY2, "deprecated function")
-    def test_stringify_dict_keys_only(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
-        d2 = stringify_dict(d)
-        self.assertEqual(d, d2)
-        self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-
     def test_get_func_args(self):
         def f1(a, b, c):
             pass
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..7b015ff8d 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
-    def test_externalUnicodeInterference(self):
-        """
-        L{client._parse} should return C{str} for the scheme, host, and path
-        elements of its return tuple, even when passed an URL which has
-        previously been passed to L{urlparse} as a C{unicode} string.
-        """
-        if not six.PY2:
-            raise unittest.SkipTest(
-                "Applies only to Py2, as urls can be ONLY unicode on Py3")
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
-        self._parse(badInput)  # cache badInput in urlparse_cached
-        scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
-        self.assertTrue(isinstance(port, int))
-
-
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
 

From b0d6f4917d782f2396a702e90c36ffbc8bb42e7e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:17:03 +0500
Subject: [PATCH 02/49] Restore tests/test_proxy_connect.py and update it to
 modern mitmproxy.

---
 tests/requirements-py3.txt  |   1 +
 tests/test_proxy_connect.py | 134 ++++++++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 tests/test_proxy_connect.py

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dd5b23cc3..f27e45a54 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,5 +1,6 @@
 # Tests requirements
 jmespath
+mitmproxy
 pytest
 pytest-cov
 pytest-twisted
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
new file mode 100644
index 000000000..8142d9a41
--- /dev/null
+++ b/tests/test_proxy_connect.py
@@ -0,0 +1,134 @@
+import json
+import os
+import re
+from subprocess import Popen, PIPE
+import sys
+import time
+
+from six.moves.urllib.parse import urlsplit, urlunsplit
+from testfixtures import LogCapture
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.utils.test import get_crawler
+from scrapy.http import Request
+from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.mockserver import MockServer
+
+
+class MitmProxy:
+    auth_user = 'scrapy'
+    auth_pass = 'scrapy'
+
+    def start(self):
+        from scrapy.utils.test import get_testenv
+        script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+        """
+        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+            'keys', 'mitmproxy-ca.pem')
+        self.proc = Popen([sys.executable,
+                           '-c', script,
+                           '--listen-host', '127.0.0.1',
+                           '--listen-port', '0',
+                           '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+                           '--certs', cert_path,
+                           '--ssl-insecure',
+                           ],
+                           stdout=PIPE, env=get_testenv())
+        line = self.proc.stdout.readline().decode('utf-8')
+        host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
+        address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
+        return address
+
+    def stop(self):
+        self.proc.kill()
+        self.proc.wait()
+        time.sleep(0.2)
+
+
+def _wrong_credentials(proxy_url):
+    bad_auth_proxy = list(urlsplit(proxy_url))
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
+    return urlunsplit(bad_auth_proxy)
+
+
+class ProxyConnectTestCase(TestCase):
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self._oldenv = os.environ.copy()
+
+        self._proxy = MitmProxy()
+        proxy_url = self._proxy.start()
+        os.environ['https_proxy'] = proxy_url
+        os.environ['http_proxy'] = proxy_url
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+        self._proxy.stop()
+        os.environ = self._oldenv
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_noconnect(self):
+        proxy = os.environ['https_proxy']
+        os.environ['https_proxy'] = proxy + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(200, l)
+
+    @defer.inlineCallbacks
+    def test_https_connect_tunnel_error(self):
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl("https://localhost:99999/status?n=200")
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        # The proxy returns a 407 error code but it does not reach the client;
+        # he just sees a TunnelError.
+        self._assert_got_tunnel_error(l)
+
+    @defer.inlineCallbacks
+    def test_https_tunnel_without_leak_proxy_authorization_header(self):
+        request = Request(self.mockserver.url("/echo", is_secure=True))
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(seed=request)
+        self._assert_got_response_code(200, l)
+        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        self.assertTrue('Proxy-Authorization' not in echo['headers'])
+
+    @defer.inlineCallbacks
+    def test_https_noconnect_auth_error(self):
+        os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
+        crawler = get_crawler(SimpleSpider)
+        with LogCapture() as l:
+            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
+        self._assert_got_response_code(407, l)
+
+    def _assert_got_response_code(self, code, log):
+        print(log)
+        self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
+
+    def _assert_got_tunnel_error(self, log):
+        print(log)
+        self.assertIn('TunnelError', str(log))

From 439e37fc7b2c192b4591d29dde74c9f589763cc8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:23:24 +0500
Subject: [PATCH 03/49] Mark failing proxy tests.

---
 tests/test_proxy_connect.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 8142d9a41..5e9470e39 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -5,6 +5,7 @@ from subprocess import Popen, PIPE
 import sys
 import time
 
+import pytest
 from six.moves.urllib.parse import urlsplit, urlunsplit
 from testfixtures import LogCapture
 
@@ -81,6 +82,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect(self):
         proxy = os.environ['https_proxy']
@@ -90,6 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
+    @pytest.mark.xfail(reason='Python 3 fails this earlier')
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
@@ -117,6 +120,7 @@ class ProxyConnectTestCase(TestCase):
         echo = json.loads(crawler.spider.meta['responses'][0].body)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
+    @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'

From 186f9d88acd9043cf5d8f21358c758c42e433105 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 3 Sep 2019 15:24:06 +0500
Subject: [PATCH 04/49] Fix the skip message for test_download_gzip_response.

---
 tests/test_downloader_handlers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4d3c4d4aa..9f74577a5 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -634,7 +634,7 @@ class Http11MockServerTestCase(unittest.TestCase):
         self.assertIsInstance(failure.value, defer.CancelledError)
 
         # See issue https://twistedmatrix.com/trac/ticket/8175
-        raise unittest.SkipTest("xpayload only enabled for PY2")
+        raise unittest.SkipTest("xpayload fails on PY3")
         request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
         request = request.replace(url=self.mockserver.url('/xpayload'))
         yield crawler.crawl(seed=request)

From bbd9f4be90e734df5c290aeb399e8e512f834224 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Jul 2019 22:27:29 +0500
Subject: [PATCH 05/49] Remove six.PY2 and six.PY3 conditionals.

---
 docs/topics/downloader-middleware.rst |  6 +++---
 scrapy/_monkeypatches.py              | 10 ----------
 scrapy/commands/fetch.py              |  5 ++---
 scrapy/crawler.py                     | 11 -----------
 scrapy/exporters.py                   |  2 +-
 scrapy/extensions/feedexport.py       |  3 +--
 scrapy/http/request/form.py           |  3 +--
 scrapy/http/response/text.py          |  3 ---
 scrapy/item.py                        |  8 +-------
 scrapy/link.py                        | 15 ++-------------
 scrapy/mail.py                        |  9 ++-------
 scrapy/settings/__init__.py           |  8 +-------
 scrapy/settings/default_settings.py   |  4 +---
 scrapy/utils/boto.py                  | 10 +---------
 scrapy/utils/conf.py                  |  5 +----
 scrapy/utils/datatypes.py             | 20 +++++++-------------
 scrapy/utils/gz.py                    | 13 +++----------
 scrapy/utils/iterators.py             |  6 +-----
 scrapy/utils/python.py                | 10 ++--------
 tests/__init__.py                     | 15 ---------------
 tests/test_http_request.py            | 11 +++--------
 tests/test_http_response.py           |  3 +--
 tests/test_item.py                    |  8 ++------
 tests/test_link.py                    | 13 ++-----------
 tests/test_middleware.py              | 21 ++++++---------------
 tests/test_request_cb_kwargs.py       |  8 +-------
 tests/test_settings/__init__.py       |  8 +-------
 tests/test_utils_datatypes.py         |  7 +------
 tests/test_utils_python.py            |  7 +++----
 29 files changed, 50 insertions(+), 202 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8048e1c86..366b95510 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the anydbm_ module, but you can change it with the
+    By default, it uses the dbm_ module, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
 
 .. versionadded:: 0.13
 
-Default: ``'anydbm'``
+Default: ``'dbm'``
 
 The database module to use in the :ref:`DBM storage backend
 <httpcache-storage-dbm>`. This setting is specific to the DBM backend.
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html
+.. _dbm: https://docs.python.org/3/library/dbm.html
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index b68099cad..1f8067b35 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,16 +1,6 @@
-import six
 from six.moves import copyreg
 
 
-if six.PY2:
-    from urlparse import urlparse
-    
-    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
-    if urlparse('s3://bucket/key?key=value').query != 'key=value':
-        from urlparse import uses_query
-        uses_query.append('s3')
-
-
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
 import twisted.persisted.styles  # NOQA
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 7d4840529..d45133e0e 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, six
+import sys
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
@@ -45,8 +45,7 @@ class Command(ScrapyCommand):
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
-        bytes_writer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ded3c082b..19b998e0d 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -88,20 +88,9 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
-            # In Python 2 reraising an exception after yield discards
-            # the original traceback (see https://bugs.python.org/issue7563),
-            # so sys.exc_info() workaround is used.
-            # This workaround also works in Python 3, but it is not needed,
-            # and it is slower, so in Python 3 we use native `raise`.
-            if six.PY2:
-                exc_info = sys.exc_info()
-
             self.crawling = False
             if self.engine is not None:
                 yield self.engine.close()
-
-            if six.PY2:
-                six.reraise(*exc_info)
             raise
 
     def _create_spider(self, *args, **kwargs):
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 6fc87ed18..5d1f1ad8f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -216,7 +216,7 @@ class CsvItemExporter(BaseItemExporter):
             write_through=True,
             encoding=self.encoding,
             newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
-        ) if six.PY3 else file
+        )
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 6fb6397b1..07ffd3476 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,6 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-import six
 from six.moves.urllib.parse import urlparse, unquote
 from ftplib import FTP
 
@@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
 
     def __init__(self, uri, _stdout=None):
         if not _stdout:
-            _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+            _stdout = sys.stdout.buffer
         self._stdout = _stdout
 
     def open(self, spider):
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 3ce8fc48e..b6feede07 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
-        raise ValueError('No <form> element found with %s' % encoded)
+        raise ValueError('No <form> element found with %s' % formxpath)
 
     # If we get here, it means that either formname was None
     # or invalid
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 339913d4e..a8010877c 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -32,9 +32,6 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            if six.PY2 and self.encoding is None:
-                raise TypeError("Cannot convert unicode url - %s "
-                                "has no encoding" % type(self).__name__)
             self._url = to_native_str(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
diff --git a/scrapy/item.py b/scrapy/item.py
index 73b8f54b0..32f9b2ebb 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,8 +4,8 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-import collections
 from abc import ABCMeta
+from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
 from warnings import warn
@@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 class BaseItem(object_ref):
     """Base class for all scraped items.
 
diff --git a/scrapy/link.py b/scrapy/link.py
index 2c8301680..a175b8afd 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,12 +4,6 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
-import warnings
-import six
-
-from scrapy.utils.python import to_bytes
-
-
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
@@ -17,13 +11,8 @@ class Link(object):
 
     def __init__(self, url, text='', fragment='', nofollow=False):
         if not isinstance(url, str):
-            if six.PY2:
-                warnings.warn("Link urls must be str objects. "
-                              "Assuming utf-8 encoding (which could be wrong)")
-                url = to_bytes(url, encoding='utf8')
-            else:
-                got = url.__class__.__name__
-                raise TypeError("Link urls must be str objects, got %s" % got)
+            got = url.__class__.__name__
+            raise TypeError("Link urls must be str objects, got %s" % got)
         self.url = url
         self.text = text
         self.fragment = fragment
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 5b944e1c4..746468e25 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -9,18 +9,13 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-import six
 
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
-if six.PY2:
-    from email.MIMENonMultipart import MIMENonMultipart
-    from email import Encoders
-else:
-    from email.mime.nonmultipart import MIMENonMultipart
-    from email import encoders as Encoders
+from email.mime.nonmultipart import MIMENonMultipart
+from email import encoders as Encoders
 
 from twisted.internet import defer, reactor, ssl
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index f28c7940d..c871e86e0 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,19 +1,13 @@
 import six
 import json
 import copy
-import collections
+from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
 from scrapy.settings import default_settings
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 9c22999cb..5c9678c01 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -17,8 +17,6 @@ import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
 
-import six
-
 AJAXCRAWL_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
@@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
+HTTPCACHE_DBM_MODULE = 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 421ab2f7e..c8fc911bb 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,6 @@
 """Boto/botocore helpers"""
 
 from __future__ import absolute_import
-import six
 
 from scrapy.exceptions import NotConfigured
 
@@ -11,11 +10,4 @@ def is_botocore():
         import botocore
         return True
     except ImportError:
-        if six.PY2:
-            try:
-                import boto
-                return False
-            except ImportError:
-                raise NotConfigured('missing botocore or boto library')
-        else:
-            raise NotConfigured('missing botocore library')
+        raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index fb7ca3310..561bb72fc 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,13 +1,10 @@
+from configparser import ConfigParser
 import os
 import sys
 import numbers
 from operator import itemgetter
 
 import six
-if six.PY2:
-    from ConfigParser import SafeConfigParser as ConfigParser
-else:
-    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index df2b99c28..6e9de47f3 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
+from collections.abc import Mapping
 import warnings
 
 import six
@@ -14,12 +15,6 @@ import six
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-if six.PY2:
-    Mapping = collections.Mapping
-else:
-    Mapping = collections.abc.Mapping
-
-
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -252,13 +247,12 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
-        if not six.PY2:
-            warnings.warn(
-                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-                "of collections.ChainMap (introduced in Python 3.3)",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+            "of collections.ChainMap (introduced in Python 3.3)",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.dicts = dicts
 
     def __getitem__(self, key):
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index b3fb16b1e..9984492f0 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -6,7 +6,6 @@ except ImportError:
     from io import BytesIO
 from gzip import GzipFile
 
-import six
 import re
 
 from scrapy.utils.decorators import deprecated
@@ -17,14 +16,8 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
-# - scrapy does not support Python 3.2
-# - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY2:
-    def read1(gzf, size=-1):
-        return gzf.read(size)
-else:
-    def read1(gzf, size=-1):
-        return gzf.read1(size)
+def read1(gzf, size=-1):
+    return gzf.read1(size)
 
 
 def gunzip(data):
@@ -37,7 +30,7 @@ def gunzip(data):
     chunk = b'.'
     while chunk:
         try:
-            chunk = read1(f, 8196)
+            chunk = f.read1(8196)
             output_list.append(chunk)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index a12e14005..dbc1e0d20 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -102,11 +102,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def row_to_unicode(row_):
         return [to_unicode(field, encoding) for field in row_]
 
-    # Python 3 csv reader input object needs to return strings
-    if six.PY3:
-        lines = StringIO(_body_or_str(obj, unicode=True))
-    else:
-        lines = BytesIO(_body_or_str(obj, unicode=False))
+    lines = StringIO(_body_or_str(obj, unicode=True))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index c6140f885..5009aab81 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -113,10 +113,7 @@ def to_bytes(text, encoding=None, errors='strict'):
 def to_native_str(text, encoding=None, errors='strict'):
     """ Return str representation of ``text``
     (bytes in Python 2.x and unicode in Python 3.x). """
-    if six.PY2:
-        return to_bytes(text, encoding, errors)
-    else:
-        return to_unicode(text, encoding, errors)
+    return to_unicode(text, encoding, errors)
 
 
 def re_rsearch(pattern, text, chunk_size=1024):
@@ -189,7 +186,7 @@ def _getargspec_py23(func):
     """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
                                                         defaults)
 
-    Identical to inspect.getargspec() in python2, but uses
+    Was identical to inspect.getargspec() in python2, but uses
     inspect.getfullargspec() for python3 behind the scenes to avoid
     DeprecationWarning.
 
@@ -199,9 +196,6 @@ def _getargspec_py23(func):
     >>> _getargspec_py23(f)
     ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
     """
-    if six.PY2:
-        return inspect.getargspec(func)
-
     return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
 
 
diff --git a/tests/__init__.py b/tests/__init__.py
index 9c9e35c35..a54367f8c 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -35,18 +35,3 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
-
-
-# FIXME: delete after dropping py2 support
-# Monkey patch the unittest module to prevent the
-# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
-import six
-if six.PY2:
-    import unittest
-    import twisted.trial.unittest
-    if not getattr(unittest.TestCase, 'assertRegex', None):
-        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
-    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
-        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
-    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
-        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 16d7a1cb8..828902b99 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,13 +3,12 @@ import cgi
 import unittest
 import re
 import json
+from urllib.parse import unquote_to_bytes
 import warnings
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
-if six.PY3:
-    from urllib.parse import unquote_to_bytes
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
@@ -1064,8 +1063,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs, {})
 
         xpath = u"//form[@name='\u03b1']"
-        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegex(ValueError, re.escape(encoded),
+        self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
 
@@ -1208,10 +1206,7 @@ def _qs(req, encoding='utf-8', to_unicode=False):
         qs = req.body
     else:
         qs = req.url.partition('?')[2]
-    if six.PY2:
-        uqs = unquote(to_native_str(qs, encoding))
-    elif six.PY3:
-        uqs = unquote_to_bytes(qs)
+    uqs = unquote_to_bytes(qs)
     if to_unicode:
         uqs = uqs.decode(encoding)
     return parse_qs(uqs, True)
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index cd5c3486e..d6e77d6b8 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the consturctor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
-        if not six.PY2:
-            self.assertRaises(TypeError, self.response_class, b"http://example.com")
+        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
diff --git a/tests/test_item.py b/tests/test_item.py
index 947566686..0ad278701 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase):
         i['number'] = 123
         itemrepr = repr(i)
 
-        if six.PY2:
-            self.assertEqual(itemrepr,
-                             "{'name': u'John Doe', 'number': 123}")
-        else:
-            self.assertEqual(itemrepr,
-                             "{'name': 'John Doe', 'number': 123}")
+        self.assertEqual(itemrepr,
+                         "{'name': 'John Doe', 'number': 123}")
 
         i2 = eval(itemrepr)
         self.assertEqual(i2['name'], 'John Doe')
diff --git a/tests/test_link.py b/tests/test_link.py
index 955430b37..5e2ce5eeb 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,6 +1,4 @@
 import unittest
-import warnings
-import six
 
 from scrapy.link import Link
 
@@ -46,12 +44,5 @@ class LinkTest(unittest.TestCase):
         self._assert_same_links(l1, l2)
 
     def test_non_str_url_py2(self):
-        if six.PY2:
-            with warnings.catch_warnings(record=True) as w:
-                link = Link(u"http://www.example.com/\xa3")
-                self.assertIsInstance(link.url, str)
-                self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            assert len(w) == 1, "warning not issued"
-        else:
-            with self.assertRaises(TypeError):
-                Link(b"http://www.example.com/\xc2\xa3")
+        with self.assertRaises(TypeError):
+            Link(b"http://www.example.com/\xc2\xa3")
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index aea0be825..af9b43d61 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -3,7 +3,6 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
-import six
 
 class M1(object):
 
@@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        if six.PY2:
-            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.im_class for x in mwman.methods['process']],
-                [M1, M3])
-        else:
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
-                [M1, M3])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+            [M1, M2])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+            [M2, M1])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+            [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index c9943faa8..a5cdc0de0 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,7 +1,6 @@
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
         self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 1dbacbea3..08286ff02 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -60,9 +60,6 @@ class SettingsAttributeTest(unittest.TestCase):
 
 class BaseSettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = BaseSettings()
 
@@ -152,7 +149,7 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+        self.assertCountEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))
 
         for key in six.iterkeys(ctrl_attributes):
@@ -343,9 +340,6 @@ class BaseSettingsTest(unittest.TestCase):
 
 class SettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = Settings()
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 535095b8d..47877f555 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,12 +1,7 @@
+from collections.abc import Mapping, MutableMapping
 import copy
 import unittest
 
-import six
-if six.PY2:
-    from collections import Mapping, MutableMapping
-else:
-    from collections.abc import Mapping, MutableMapping
-
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 
 
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..096aa50b7 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
-            stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
-                get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, True), ['list'])
             self.assertEqual(
-                get_func_args(operator.itemgetter(2), stripself), ['obj'])
+                get_func_args(operator.itemgetter(2), True), ['obj'])
 
 
     def test_without_none_values(self):

From de7789e52df5d8dc8595570d36a60ae92b4fcb50 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 20:56:26 +0500
Subject: [PATCH 06/49] Remove unneeded and unused code from XmlItemExporter.

---
 scrapy/exporters.py | 22 ++++------------------
 1 file changed, 4 insertions(+), 18 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 5d1f1ad8f..40567f53b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -143,11 +143,11 @@ class XmlItemExporter(BaseItemExporter):
 
     def _beautify_newline(self, new_item=False):
         if self.indent is not None and (self.indent > 0 or new_item):
-            self._xg_characters('\n')
+            self.xg.characters('\n')
 
     def _beautify_indent(self, depth=1):
         if self.indent:
-            self._xg_characters(' ' * self.indent * depth)
+            self.xg.characters(' ' * self.indent * depth)
 
     def start_exporting(self):
         self.xg.startDocument()
@@ -182,26 +182,12 @@ class XmlItemExporter(BaseItemExporter):
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, six.text_type):
-            self._xg_characters(serialized_value)
+            self.xg.characters(serialized_value)
         else:
-            self._xg_characters(str(serialized_value))
+            self.xg.characters(str(serialized_value))
         self.xg.endElement(name)
         self._beautify_newline()
 
-    # Workaround for https://bugs.python.org/issue17606
-    # Before Python 2.7.4 xml.sax.saxutils required bytes;
-    # since 2.7.4 it requires unicode. The bug is likely to be
-    # fixed in 2.7.6, but 2.7.6 will still support unicode,
-    # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
-    if sys.version_info[:3] >= (2, 7, 4):
-        def _xg_characters(self, serialized_value):
-            if not isinstance(serialized_value, six.text_type):
-                serialized_value = serialized_value.decode(self.encoding)
-            return self.xg.characters(serialized_value)
-    else:  # pragma: no cover
-        def _xg_characters(self, serialized_value):
-            return self.xg.characters(serialized_value)
-
 
 class CsvItemExporter(BaseItemExporter):
 

From c2898fdcf91cbbfa9d801196924960d3a3764b93 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:06:52 +0500
Subject: [PATCH 07/49] Deprecate scrapy.utils.gz.read1.

---
 scrapy/utils/gz.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 9984492f0..dc8316d8c 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -16,6 +16,7 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
+@deprecated('GzipFile.read1')
 def read1(gzf, size=-1):
     return gzf.read1(size)
 

From cea2f5e244f30591bae667abf2b2689e566ab343 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:09:49 +0500
Subject: [PATCH 08/49] Remove cStringIO imports.

---
 scrapy/downloadermiddlewares/decompression.py | 6 +-----
 scrapy/mail.py                                | 6 +-----
 scrapy/pipelines/files.py                     | 7 +------
 scrapy/pipelines/images.py                    | 6 +-----
 scrapy/utils/gz.py                            | 9 ++-------
 scrapy/utils/iterators.py                     | 6 +-----
 tests/test_cmdline/__init__.py                | 5 +----
 tests/test_pipeline_media.py                  | 6 ++----
 8 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 49313cc04..e2d73f347 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
+from io import BytesIO
 import zipfile
 import tarfile
 import logging
@@ -11,11 +12,6 @@ from tempfile import mktemp
 
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from scrapy.responsetypes import responsetypes
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 746468e25..d24de2212 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,13 +3,9 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+from io import BytesIO
 import logging
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index cc3d10b63..8d74c5011 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import mimetypes
 import os
 import os.path
@@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index fa4d12ad1..e77cef4ff 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from PIL import Image
 
 from scrapy.utils.misc import md5sum
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index dc8316d8c..f41e62fe3 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,12 +1,7 @@
-import struct
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from gzip import GzipFile
-
+from io import BytesIO
 import re
+import struct
 
 from scrapy.utils.decorators import deprecated
 
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index dbc1e0d20..9693ba768 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,11 +1,7 @@
 import re
 import csv
-import logging
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from io import StringIO
+import logging
 import six
 
 from scrapy.http import TextResponse, Response
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 68dfb1cca..56cfe642a 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,3 +1,4 @@
+from io import StringIO
 import json
 import os
 import pstats
@@ -7,10 +8,6 @@ from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
 
 from scrapy.utils.test import get_testenv
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 28e39cefa..ad2618ec9 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -144,10 +144,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
         # The Failure should encapsulate a FileException ...
         self.assertEqual(failure.value, file_exc)
-        # ... and if we're running on Python 3 ...
-        if sys.version_info.major >= 3:
-            # ... it should have the returnValue exception set as its context
-            self.assertEqual(failure.value.__context__, def_gen_return_exc)
+        # ... and it should have the returnValue exception set as its context
+        self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
         fp = request_fingerprint(request)

From 5b70b051a6cec699c3cb14c3ac9cf530671a8487 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:13:01 +0500
Subject: [PATCH 09/49] Some text function messages cleanup, deprecate
 to_native_str.

---
 scrapy/http/response/__init__.py | 2 +-
 scrapy/utils/python.py           | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index b0a526b72..a81404afb 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -88,7 +88,7 @@ class Response(object_ref):
     @property
     def text(self):
         """For subclasses of TextResponse, this will return the body
-        as text (unicode object in Python 2 and str in Python 3)
+        as str
         """
         raise AttributeError("Response content isn't text")
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5009aab81..974abaeb1 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'):
     if isinstance(text, six.text_type):
         return text
     if not isinstance(text, (bytes, six.text_type)):
-        raise TypeError('to_unicode must receive a bytes, str or unicode '
+        raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
@@ -103,16 +103,16 @@ def to_bytes(text, encoding=None, errors='strict'):
     if isinstance(text, bytes):
         return text
     if not isinstance(text, six.string_types):
-        raise TypeError('to_bytes must receive a unicode, str or bytes '
+        raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
     return text.encode(encoding, errors)
 
 
+@deprecated('to_unicode')
 def to_native_str(text, encoding=None, errors='strict'):
-    """ Return str representation of ``text``
-    (bytes in Python 2.x and unicode in Python 3.x). """
+    """ Return str representation of ``text``. """
     return to_unicode(text, encoding, errors)
 
 

From 3ac4b430ae0165f25d8cd2fb1ac25ac9f307df8b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 15:20:28 +0500
Subject: [PATCH 10/49] Remove an unused six import.

---
 tests/test_downloader_handlers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 9f74577a5..e6856945c 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,4 @@
 import os
-import six
 import shutil
 import tempfile
 import contextlib

From 75b1d051d99cda17558a210326e77046eec851a7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:27:52 +0500
Subject: [PATCH 11/49] Simplify some more imports.

---
 scrapy/downloadermiddlewares/httpproxy.py    | 5 +----
 scrapy/loader/processors.py                  | 5 +----
 tests/__init__.py                            | 5 -----
 tests/test_downloader_handlers.py            | 5 +----
 tests/test_downloadermiddleware.py           | 3 ++-
 tests/test_downloadermiddleware_robotstxt.py | 4 +++-
 tests/test_extension_telnet.py               | 5 -----
 tests/test_feedexport.py                     | 2 +-
 tests/test_http_request.py                   | 3 +--
 tests/test_item.py                           | 2 +-
 tests/test_pipeline_files.py                 | 6 +-----
 tests/test_settings/__init__.py              | 3 +--
 tests/test_spider.py                         | 3 +--
 tests/test_spidermiddleware.py               | 3 ++-
 tests/test_stats.py                          | 6 +-----
 tests/test_utils_deprecate.py                | 3 +--
 tests/test_utils_misc/__init__.py            | 2 +-
 tests/test_utils_trackref.py                 | 2 +-
 18 files changed, 20 insertions(+), 47 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2c35d1b90..2212d9688 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,10 +1,7 @@
 import base64
 from six.moves.urllib.parse import unquote, urlunparse
 from six.moves.urllib.request import getproxies, proxy_bypass
-try:
-    from urllib2 import _parse_proxy
-except ImportError:
-    from urllib.request import _parse_proxy
+from urllib.request import _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index 2acdc8093..02c625acc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-try:
-    from collections import ChainMap
-except ImportError:
-    from scrapy.utils.datatypes import MergeDict as ChainMap
+from collections import ChainMap
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.loader.common import wrap_loader_context
diff --git a/tests/__init__.py b/tests/__init__.py
index a54367f8c..12ce79fa9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
     os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
                                                  os.environ['COV_CORE_CONFIG'])
 
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                              'sample_data')
 
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 109469503..6090998d4 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -2,11 +2,8 @@ import os
 import six
 import shutil
 import tempfile
+from unittest import mock
 import contextlib
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 03564e748..6b9a5bee8 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
 from scrapy.utils.python import to_bytes
-from tests import mock
 
 
 class ManagerTestCase(TestCase):
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index fbc46cba4..8266bf35f 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+
+from unittest import mock
+
 from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
@@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
-from tests import mock
 from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 4f389e5cb..875ceb83c 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -1,8 +1,3 @@
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 from twisted.trial import unittest
 from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e1436fbe5..7431f921f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ from io import BytesIO
 import tempfile
 import shutil
 import string
+from unittest import mock
 from six.moves.urllib.parse import urljoin, urlparse, quote
 from six.moves.urllib.request import pathname2url
 
@@ -15,7 +16,6 @@ from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.crawler import CrawlerRunner
 from scrapy.settings import Settings
-from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 828902b99..effb9e53b 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,6 +3,7 @@ import cgi
 import unittest
 import re
 import json
+from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
@@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
 
-from tests import mock
-
 
 class RequestTest(unittest.TestCase):
 
diff --git a/tests/test_item.py b/tests/test_item.py
index 0ad278701..d98c63ddd 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,12 +1,12 @@
 import sys
 import unittest
+from unittest import mock
 from warnings import catch_warnings
 
 import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
-from tests import mock
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index cb8f8da18..bd40e4103 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,10 +1,9 @@
 import os
 import random
 import time
-import hashlib
-import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from unittest import mock
 from six.moves.urllib.parse import urlparse
 from six import BytesIO
 
@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
-from tests import mock
-
 
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 08286ff02..32e65bed5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,10 +1,9 @@
 import six
 import unittest
-import warnings
+from unittest import mock
 
 from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
                              SETTINGS_PRIORITIES, get_settings_priority)
-from tests import mock
 from . import default_settings
 
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 2220b8ffc..b913a56b7 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,5 +1,6 @@
 import gzip
 import inspect
+from unittest import mock
 import warnings
 from io import BytesIO
 
@@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler
 
-from tests import mock
-
 
 class SpiderTest(unittest.TestCase):
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 832fd3330..55d665e79 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -6,7 +8,6 @@ from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
-from tests import mock
 
 
 class SpiderMiddlewareTestCase(TestCase):
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2033dbe07..2bbbb9e2c 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,6 @@
 from datetime import datetime
 import unittest
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from unittest import mock
 
 from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 3e7236fb1..ce04e7f29 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -2,12 +2,11 @@
 from __future__ import absolute_import
 import inspect
 import unittest
+from unittest import mock
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import create_deprecated_class, update_classpath
 
-from tests import mock
-
 
 class MyWarning(UserWarning):
     pass
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..de9da9104 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,11 +1,11 @@
 import sys
 import os
 import unittest
+from unittest import mock
 
 from scrapy.item import Item, Field
 from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
-from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index c6072fc0d..480a717e7 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,7 @@
 import six
 import unittest
+from unittest import mock
 from scrapy.utils import trackref
-from tests import mock
 
 
 class Foo(trackref.object_ref):

From 397e8835564614608647c855c62c36232020f078 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 20 Aug 2019 21:35:13 +0500
Subject: [PATCH 12/49] Replace to_native_str calls with to_unicode.

---
 scrapy/core/downloader/handlers/http11.py |  2 +-
 scrapy/downloadermiddlewares/cookies.py   |  6 +++---
 scrapy/downloadermiddlewares/robotstxt.py |  3 ---
 scrapy/exporters.py                       |  4 ++--
 scrapy/http/cookies.py                    | 10 +++++-----
 scrapy/http/response/text.py              |  8 ++++----
 scrapy/linkextractors/lxmlhtml.py         |  4 ++--
 scrapy/responsetypes.py                   |  6 +++---
 scrapy/robotstxt.py                       |  8 ++++----
 scrapy/spidermiddlewares/referer.py       |  5 ++---
 scrapy/utils/reqser.py                    |  4 ++--
 scrapy/utils/request.py                   |  4 ++--
 scrapy/utils/response.py                  |  4 ++--
 scrapy/utils/ssl.py                       |  4 ++--
 tests/test_command_parse.py               |  5 ++---
 tests/test_commands.py                    |  7 ++-----
 tests/test_feedexport.py                  |  7 +++----
 tests/test_http_request.py                |  7 +++----
 tests/test_http_response.py               |  6 +++---
 tests/test_robotstxt_interface.py         |  1 -
 20 files changed, 47 insertions(+), 58 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 91b45a8fc..7d917cb74 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
     r"""
     Return binary content of a CONNECT request.
 
-    >>> from scrapy.utils.python import to_native_str as s
+    >>> from scrapy.utils.python import to_unicode as s
     >>> s(tunnel_request_data("example.com", 8080))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 321c0171b..0d2b9900c 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -6,7 +6,7 @@ from collections import defaultdict
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in request.headers.getlist('Cookie')]
             if cl:
                 cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in response.headers.getlist('Set-Cookie')]
             if cl:
                 cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6a5dfb79c..251706c50 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-import sys
-import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
 from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 40567f53b..f276c28e8 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -12,7 +12,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 import warnings
@@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter):
     def _build_row(self, values):
         for s in values:
             try:
-                yield to_native_str(s, self.encoding)
+                yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4e8056750..4532c3ab7 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
     CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
 )
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default),
+        return to_unicode(self.request.headers.get(name, default),
                              errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k, errors='replace'),
-             [to_native_str(x, errors='replace') for x in v])
+            (to_unicode(k, errors='replace'),
+             [to_unicode(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -189,7 +189,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v, errors='replace')
+        return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index a8010877c..37f450e54 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace
 from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
 
 
 class TextResponse(Response):
@@ -32,7 +32,7 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            self._url = to_native_str(url, self.encoding)
+            self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
@@ -81,11 +81,11 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b'Content-Type', b'')
-        return http_content_type_encoding(to_native_str(content_type))
+        return http_content_type_encoding(to_unicode(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+            content_type = to_unicode(self.headers.get(b'Content-Type', b''))
             benc, ubody = html_to_unicode(content_type, self.body,
                     auto_detect_fun=self._auto_detect_fun,
                     default_encoding=self._DEFAULT_ENCODING)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 8f6f93a44..890c019c8 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_native_str(url, encoding=response_encoding)
+            url = to_unicode(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4a2d5bf52..de62276c8 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -10,7 +10,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
 class ResponseTypes(object):
@@ -55,12 +55,12 @@ class ResponseTypes(object):
         header """
         if content_encoding:
             return Response
-        mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+        mimetype = to_unicode(content_type).split(';')[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition,
+            filename = to_unicode(content_disposition,
                 encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 189f165d1..95a8c09b8 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -3,14 +3,14 @@ import logging
 from abc import ABCMeta, abstractmethod
 from six import with_metaclass
 
-from scrapy.utils.python import to_native_str, to_unicode
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
-            robotstxt_body = to_native_str(robotstxt_body)
+            robotstxt_body = to_unicode(robotstxt_body)
         else:
             robotstxt_body = robotstxt_body.decode('utf-8')
     except UnicodeDecodeError:
@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
         return o
 
     def allowed(self, url, user_agent):
-        user_agent = to_native_str(user_agent)
-        url = to_native_str(url)
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
 
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1ddfb37f4..c76e4d5a2 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
 
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
             if isinstance(resp_or_url, Response):
                 policy_header = resp_or_url.headers.get('Referrer-Policy')
                 if policy_header is not None:
-                    policy_name = to_native_str(policy_header.decode('latin1'))
+                    policy_name = to_unicode(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index c7ea7b425..495564ac0 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
 import six
 
 from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 
 
@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
         eb = _get_method(spider, eb)
     request_cls = load_object(d['_class']) if '_class' in d else Request
     return request_cls(
-        url=to_native_str(d['url']),
+        url=to_unicode(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index fb5af66a2..63d0ae772 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -9,7 +9,7 @@ import weakref
 from six.moves.urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 from w3lib.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
@@ -97,4 +97,4 @@ def referer_str(request):
     referrer = request.headers.get('Referer')
     if referrer is None:
         return referrer
-    return to_native_str(referrer, errors='replace')
+    return to_unicode(referrer, errors='replace')
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c3236afd4..feab07431 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -8,7 +8,7 @@ import webbrowser
 import tempfile
 
 from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
@@ -36,7 +36,7 @@ def response_status_message(status):
     """Return status code plus status text descriptive message
     """
     message = http.RESPONSES.get(int(status), "Unknown Status")
-    return '%s %s' % (status, to_native_str(message))
+    return '%s %s' % (status, to_unicode(message))
 
 
 def response_httprepr(response):
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 02aed60ee..6e81b33ff 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -3,7 +3,7 @@
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
 
 
 def ffi_buf_to_string(buf):
-    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+    return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
 def x509name_to_string(x509name):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 62d5d76b4..b134beb88 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,17 +1,16 @@
 import os
 from os.path import join, abspath
-from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from tests.test_commands import CommandTest
 
 
 def _textmode(bstr):
     """Normalize input the same as writing to a file
     and reading from it in text mode"""
-    return to_native_str(bstr).replace(os.linesep, '\n')
+    return to_unicode(bstr).replace(os.linesep, '\n')
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b8445ae6c..536379170 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -10,13 +10,10 @@ from contextlib import contextmanager
 from threading import Timer
 
 from twisted.trial import unittest
-from twisted.internet import defer
 
 import scrapy
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
-from scrapy.utils.testsite import SiteTest
-from scrapy.utils.testproc import ProcessTest
 from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        return p, to_native_str(stdout), to_native_str(stderr)
+        return p, to_unicode(stdout), to_unicode(stderr)
 
 
 class StartprojectTest(ProjectTest):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 7431f921f..abe2ab557 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
-from scrapy.utils.python import to_native_str
-from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_unicode
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -456,7 +455,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'csv'})
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(to_native_str(data).splitlines())
+        reader = csv.DictReader(to_unicode(data).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -470,7 +469,7 @@ class FeedExportTest(unittest.TestCase):
         settings = settings or {}
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index effb9e53b..807265981 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -7,12 +7,11 @@ from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
-import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
@@ -349,8 +348,8 @@ class FormRequestTest(RequestTest):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
-        first = to_native_str(first).split("&")
-        second = to_native_str(second).split("&")
+        first = to_unicode(first).split("&")
+        second = to_unicode(second).split("&")
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_empty_formdata(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index d6e77d6b8..80bf51647 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.exceptions import NotSupported
 from scrapy.link import Link
 from tests import get_testdata
@@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest):
         assert isinstance(resp.url, str)
 
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9aaab560a..cd7480e33 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,6 +1,5 @@
 # coding=utf-8
 from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
 
 
 def reppy_available():

From 7299e91b1f1b231095e2a7ddbce5ad980ae7da8a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 28 Aug 2019 16:29:53 +0500
Subject: [PATCH 13/49] Remove Py2-only code that checks sys.version_info.

---
 tests/test_utils_reqser.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 11ac56897..92cd16de7 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -80,8 +80,6 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_mixin_private_callback_serialization(self):
-        if sys.version_info[0] < 3:
-            return
         r = Request("http://www.example.com",
                     callback=self.spider._TestSpiderMixin__mixin_callback,
                     errback=self.spider.handle_error)
@@ -119,9 +117,8 @@ class RequestSerializationTest(unittest.TestCase):
     def test_private_name_mangling(self):
         self._assert_mangles_to(
             self.spider, '_TestSpider__parse_item_private')
-        if sys.version_info[0] >= 3:
-            self._assert_mangles_to(
-                self.spider, '_TestSpiderMixin__mixin_callback')
+        self._assert_mangles_to(
+            self.spider, '_TestSpiderMixin__mixin_callback')
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)

From 864123132a99dda61feb407a59367c6c06133517 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 22:55:58 +0500
Subject: [PATCH 14/49] Fix a duplicate ref name in docs.

---
 docs/topics/downloader-middleware.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 366b95510..e93645077 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the dbm_ module, but you can change it with the
+    By default, it uses the `dbm module`_, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _dbm: https://docs.python.org/3/library/dbm.html
+.. _dbm module: https://docs.python.org/3/library/dbm.html

From a5eb59b92d3311f702933753d85b15e6d698faa1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:14 +0500
Subject: [PATCH 15/49] Fix test_proxy_connect.py for py3.5.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 5e9470e39..f6381b5b1 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body)
+        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 5eb01b617d207aabc705038d8a0e3b46e358bdbb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 31 Oct 2019 23:21:30 +0500
Subject: [PATCH 16/49] Use an older mitmproxy for py3.5.

---
 tests/requirements-py3.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index f27e45a54..c2b16bec6 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,6 +1,7 @@
 # Tests requirements
 jmespath
-mitmproxy
+mitmproxy; python_version >= '3.6'
+mitmproxy==3.0.4; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted

From e0c5c724969ebc35970daaf0203969dc2eb00d56 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:19 +0500
Subject: [PATCH 17/49] Improve the
 test_https_tunnel_without_leak_proxy_authorization_header change.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index f6381b5b1..651576c2c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase):
         with LogCapture() as l:
             yield crawler.crawl(seed=request)
         self._assert_got_response_code(200, l)
-        echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8'))
+        echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')

From 3c9963ab049e49a1d490d9505ad22ae9c1415421 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:46:38 +0500
Subject: [PATCH 18/49] Only xfail test_https_connect_tunnel_error on 3.6+.

---
 tests/test_proxy_connect.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 651576c2c..ec3f0716c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -92,7 +92,7 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, l)
 
-    @pytest.mark.xfail(reason='Python 3 fails this earlier')
+    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)

From 4b0cdf7f3ed94e81612cceacfab6d81d54117865 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:50:56 +0500
Subject: [PATCH 19/49] Use self.proc.communicate() after killing mitmdump.

---
 tests/test_proxy_connect.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ec3f0716c..69925f80c 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -48,8 +48,7 @@ sys.exit(mitmdump())
 
     def stop(self):
         self.proc.kill()
-        self.proc.wait()
-        time.sleep(0.2)
+        self.proc.communicate()
 
 
 def _wrong_credentials(proxy_url):

From 350aa67c3dc8997ef9d3aac9ef3f596c83758e1c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 19:52:57 +0500
Subject: [PATCH 20/49] Rename tests/py3-ignores.txt to tests/ignores.txt.

---
 conftest.py                            | 2 +-
 tests/{py3-ignores.txt => ignores.txt} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename tests/{py3-ignores.txt => ignores.txt} (100%)

diff --git a/conftest.py b/conftest.py
index ede091e9f..7da4c4976 100644
--- a/conftest.py
+++ b/conftest.py
@@ -7,7 +7,7 @@ collect_ignore = [
 ]
 
 
-for line in open('tests/py3-ignores.txt'):
+for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':
         collect_ignore.append(file_path)
diff --git a/tests/py3-ignores.txt b/tests/ignores.txt
similarity index 100%
rename from tests/py3-ignores.txt
rename to tests/ignores.txt

From 48b8ac60099c3751ad1f595a9a629be26ffb34cd Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 1 Nov 2019 20:05:37 +0500
Subject: [PATCH 21/49] Improve the dbm module ref.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/downloader-middleware.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index e93645077..ae6d41809 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the `dbm module`_, but you can change it with the
+    By default, it uses the :mod:`dbm`, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _dbm module: https://docs.python.org/3/library/dbm.html

From 415526d922b6c70bfd5872c8d67d69fcce0ee1fb Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:25:15 -0300
Subject: [PATCH 22/49] Remove __future__ imports

---
 extras/qps-bench-server.py                | 1 -
 scrapy/cmdline.py                         | 1 -
 scrapy/commands/check.py                  | 1 -
 scrapy/commands/fetch.py                  | 1 -
 scrapy/commands/genspider.py              | 1 -
 scrapy/commands/list.py                   | 1 -
 scrapy/commands/parse.py                  | 1 -
 scrapy/commands/settings.py               | 1 -
 scrapy/commands/startproject.py           | 1 -
 scrapy/commands/version.py                | 2 --
 scrapy/core/downloader/__init__.py        | 1 -
 scrapy/core/downloader/handlers/http.py   | 1 -
 scrapy/downloadermiddlewares/ajaxcrawl.py | 1 -
 scrapy/dupefilters.py                     | 1 -
 scrapy/extensions/httpcache.py            | 2 --
 scrapy/pipelines/media.py                 | 2 --
 scrapy/responsetypes.py                   | 1 -
 scrapy/shell.py                           | 2 --
 scrapy/signalmanager.py                   | 1 -
 scrapy/spiderloader.py                    | 1 -
 scrapy/utils/boto.py                      | 2 --
 scrapy/utils/display.py                   | 1 -
 scrapy/utils/engine.py                    | 1 -
 scrapy/utils/ossignal.py                  | 2 --
 scrapy/utils/request.py                   | 1 -
 scrapy/utils/test.py                      | 1 -
 scrapy/utils/testproc.py                  | 1 -
 scrapy/utils/testsite.py                  | 1 -
 scrapy/utils/trackref.py                  | 1 -
 29 files changed, 35 deletions(-)

diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 3bef20bf3..da7a0022b 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 from time import time
 from collections import deque
 from twisted.web.server import Site, NOT_DONE_YET
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..69e917004 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys
 import os
 import optparse
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index ab73e85e7..ac2a95c9b 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import time
 import sys
 from collections import defaultdict
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index d45133e0e..95f6f7b9a 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys
 from w3lib.url import is_url
 
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d5498bb5c..adb01fa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import shutil
 import string
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..60686f109 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from scrapy.commands import ScrapyCommand
 
 class Command(ScrapyCommand):
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ef8acd29c..ff6f1d8cd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 import logging
 
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bee52f06a..a34338715 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 
 from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 67337c26e..34df25cef 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import re
 import os
 import string
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 577365c3b..494855500 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.versions import scrapy_components_versions
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 949dacbc8..f5f8be6e8 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import random
 import warnings
 from time import time
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index ac4b867c3..6111e132a 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,3 +1,2 @@
-from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 72715dba7..78b802673 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 import re
 import logging
 
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 0bcdd3495..f8802eb7d 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import logging
 
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index f3fabf710..b1ed0b9f8 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import gzip
 import logging
 import os
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 95dca9a3f..c174addf9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import functools
 import logging
 from collections import defaultdict
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index de62276c8..b64fbbd42 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -2,7 +2,6 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
-from __future__ import absolute_import
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 80b625633..a649d555f 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -3,8 +3,6 @@
 See documentation in docs/topics/shell.rst
 
 """
-from __future__ import print_function
-
 import os
 import signal
 import warnings
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 296d27ed8..d474f1806 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from pydispatch import dispatcher
 from scrapy.utils import signal as _signal
 
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 7478faa78..3beca4060 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 from collections import defaultdict
 import traceback
 import warnings
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index c8fc911bb..46816b54d 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,5 @@
 """Boto/botocore helpers"""
 
-from __future__ import absolute_import
-
 from scrapy.exceptions import NotConfigured
 
 
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index f6a6c4645..536de6b88 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,7 +2,6 @@
 pprint and pformat wrappers with colorization support
 """
 
-from __future__ import print_function
 import sys
 from pprint import pformat as pformat_
 
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 11dd36d91..36ef8626a 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -1,6 +1,5 @@
 """Some debugging functions for working with the Scrapy engine"""
 
-from __future__ import print_function
 from time import time # used in global tests code
 
 def get_engine_status(engine):
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index f87d5a803..7a7aec9be 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,5 +1,3 @@
-
-from __future__ import absolute_import
 import signal
 
 from twisted.internet import reactor
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 63d0ae772..45f1ef17e 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -3,7 +3,6 @@ This module provides some useful functions for working with
 scrapy.http.Request objects
 """
 
-from __future__ import print_function
 import hashlib
 import weakref
 from six.moves.urllib.parse import urlunparse
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..febaa4dcc 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,7 +2,6 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
 import os
 
 from importlib import import_module
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index f268e91ff..0f15cf60a 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import sys
 import os
 
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index e50a989b3..05d06d53b 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 from six.moves.urllib.parse import urljoin
 
 from twisted.internet import reactor
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index eed14c5a1..78389e464 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,7 +9,6 @@ and no performance penalty at all when disabled (as object_ref becomes just an
 alias to object in that case).
 """
 
-from __future__ import print_function
 import weakref
 from time import time
 from operator import itemgetter

From c0bfaef37abe029c98072e7b1a1bf346ab79016c Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:27:04 -0300
Subject: [PATCH 23/49] Remove __future__ imports from tests

---
 tests/mockserver.py                          | 1 -
 tests/test_downloadermiddleware_httpcache.py | 1 -
 tests/test_downloadermiddleware_robotstxt.py | 2 --
 tests/test_engine.py                         | 1 -
 tests/test_exporters.py                      | 1 -
 tests/test_feedexport.py                     | 1 -
 tests/test_pipeline_media.py                 | 2 --
 tests/test_utils_deprecate.py                | 1 -
 tests/test_utils_log.py                      | 1 -
 tests/test_utils_request.py                  | 1 -
 10 files changed, 12 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 77908284b..8aff7b525 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys, time, random, os, json
 from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 950664ffe..db0843b57 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import time
 import tempfile
 import shutil
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 8266bf35f..a1645ed96 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
-
 from unittest import mock
 
 from twisted.internet import reactor, error
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 30150391a..d5b911a40 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,7 +10,6 @@ module with the ``runserver`` argument::
     python test_engine.py runserver
 """
 
-from __future__ import print_function
 import sys, os, re
 from six.moves.urllib.parse import urlparse
 
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 0046c5666..7880c5bf8 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import re
 import json
 import marshal
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index abe2ab557..b13b16b41 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import os
 import csv
 import json
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ad2618ec9..70f11466b 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import sys
 
 from testfixtures import LogCapture
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index ce04e7f29..159ef8f25 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 import inspect
 import unittest
 from unittest import mock
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 742e04803..2c23f3616 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import print_function
 import sys
 import logging
 import unittest
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 625a32048..446497086 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import unittest
 from scrapy.http import Request
 from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \

From df00389c16fc8e74c5ab7737e5bffcbde6cc011b Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:48:11 -0300
Subject: [PATCH 24/49] Remove six.moves occurrences

---
 scrapy/_monkeypatches.py                  |  2 +-
 scrapy/commands/bench.py                  |  3 +--
 scrapy/core/downloader/handlers/ftp.py    |  2 +-
 scrapy/core/downloader/handlers/http11.py |  4 ++--
 scrapy/core/downloader/handlers/s3.py     |  2 +-
 scrapy/core/downloader/webclient.py       |  2 +-
 scrapy/downloadermiddlewares/httpproxy.py |  5 ++---
 scrapy/downloadermiddlewares/redirect.py  |  3 ++-
 scrapy/exporters.py                       |  7 ++++---
 scrapy/extensions/feedexport.py           |  2 +-
 scrapy/extensions/httpcache.py            |  2 +-
 scrapy/extensions/spiderstate.py          |  3 ++-
 scrapy/http/cookies.py                    |  5 ++---
 scrapy/http/request/form.py               |  4 ++--
 scrapy/http/request/rpc.py                |  2 +-
 scrapy/http/response/__init__.py          |  2 +-
 scrapy/http/response/text.py              |  4 ++--
 scrapy/linkextractors/__init__.py         |  2 +-
 scrapy/linkextractors/htmlparser.py       |  6 +++---
 scrapy/linkextractors/lxmlhtml.py         |  4 ++--
 scrapy/linkextractors/regex.py            |  3 ++-
 scrapy/linkextractors/sgml.py             |  4 ++--
 scrapy/mail.py                            | 14 +++++++-------
 scrapy/pipelines/files.py                 | 10 +++++-----
 scrapy/robotstxt.py                       |  2 +-
 scrapy/spidermiddlewares/referer.py       |  2 +-
 scrapy/squeues.py                         |  2 +-
 scrapy/utils/benchserver.py               |  3 ++-
 scrapy/utils/curl.py                      |  4 ++--
 scrapy/utils/httpobj.py                   |  2 +-
 scrapy/utils/project.py                   |  6 +++---
 scrapy/utils/python.py                    |  2 +-
 scrapy/utils/request.py                   |  6 +++---
 scrapy/utils/sitemap.py                   |  3 ++-
 scrapy/utils/testsite.py                  |  2 +-
 scrapy/utils/url.py                       |  2 +-
 36 files changed, 68 insertions(+), 65 deletions(-)

diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 1f8067b35..f74f89bda 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves import copyreg
+import copyreg
 
 
 # Undo what Twisted's perspective broker adds to pickle register
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 90c8d56a2..7bbe362e7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,8 +1,7 @@
 import sys
 import time
 import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 import scrapy
 from scrapy.commands import ScrapyCommand
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 806a537d4..2116a5a44 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..63dedc19b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -2,10 +2,10 @@
 
 import re
 import logging
+import warnings
 from io import BytesIO
 from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
 
 from zope.interface import implementer
 from twisted.internet import defer, reactor, protocol
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..85b733228 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3a5890ed0..9699da109 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,5 @@
 from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
 
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 2212d9688..814ce78fe 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,7 +1,6 @@
 import base64
-from six.moves.urllib.parse import unquote, urlunparse
-from six.moves.urllib.request import getproxies, proxy_bypass
-from urllib.request import _parse_proxy
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b73f864dd..77cb5aa94 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
 import logging
-from six.moves.urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin, urlparse
 
 from w3lib.url import safe_url_string
 
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f276c28e8..d9531e67c 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -7,15 +7,16 @@ import io
 import sys
 import pprint
 import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+from import pickle
 from xml.sax.saxutils import XMLGenerator
 
+import six
+
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 07ffd3476..0b854e6f3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,7 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-from six.moves.urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote
 from ftplib import FTP
 
 from zope.interface import Interface, implementer
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index b1ed0b9f8..b98ec218f 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,13 +1,13 @@
 import gzip
 import logging
 import os
+import pickle
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
 from warnings import warn
 from weakref import WeakKeyDictionary
 
-from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
 
 from scrapy.exceptions import ScrapyDeprecationWarning
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2220cbd8f..2c8e46914 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,10 +1,11 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
+
 class SpiderState(object):
     """Store and load spider state during a scraping job"""
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 4532c3ab7..c39de0b52 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,7 +1,6 @@
 import time
-from six.moves.http_cookiejar import (
-    CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b6feede07..d2bcf77b7 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
 
 import lxml.html
+import six
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index bd09f7534..811d3ad6b 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
 
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index a81404afb..64e9c6c20 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from scrapy.http.request import Request
 from scrapy.http.headers import Headers
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 37f450e54..69bcba577 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,10 +5,10 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 import parsel
+import six
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index ebf3cd7d8..594c7264d 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
 For more info see docs/topics/link-extractors.rst
 """
 import re
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 from parsel.csstranslator import HTMLTranslator
 from w3lib.url import canonicalize_url
 
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 27978a8a1..623732cee 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -2,10 +2,10 @@
 HTMLParser-based link extractor
 """
 import warnings
-import six
-from six.moves.html_parser import HTMLParser
-from six.moves.urllib.parse import urljoin
+from html.parser import HTMLParser
+from urllib.parse import urljoin
 
+import six
 from w3lib.url import safe_url_string
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 890c019c8..5b7e709de 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,9 +1,9 @@
 """
 Link extractor based on lxml.html
 """
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
+import six
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index e689b4727..f96db256b 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -1,11 +1,12 @@
 import re
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
 from .sgml import SgmlLinkExtractor
 
+
 linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
         re.DOTALL | re.IGNORECASE)
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 8940a4d77..98bed15e9 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -1,11 +1,11 @@
 """
 SGMLParser-based Link extractors
 """
-import six
-from six.moves.urllib.parse import urljoin
 import warnings
+from urllib.parse import urljoin
 from sgmllib import SGMLParser
 
+import six
 from w3lib.url import safe_url_string, canonicalize_url
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/mail.py b/scrapy/mail.py
index d24de2212..891bb5e09 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,21 +3,21 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
-from io import BytesIO
 import logging
-
-from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-from email.mime.nonmultipart import MIMENonMultipart
 from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from io import BytesIO
 
 from twisted.internet import defer, reactor, ssl
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8d74c5011..432d4c182 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,17 +5,17 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
-from io import BytesIO
+import logging
 import mimetypes
 import os
 import os.path
 import time
-import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
 from collections import defaultdict
-import six
+from email.utils import parsedate_tz, mktime_tz
+from io import BytesIO
+from urllib.parse import urlparse
 
+import six
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 95a8c09b8..397924110 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -53,7 +53,7 @@ class RobotParser(with_metaclass(ABCMeta)):
 
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
-        from six.moves.urllib_robotparser import RobotFileParser
+        from urllib.robotparser import RobotFileParser
         self.spider = spider
         robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp = RobotFileParser()
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index c76e4d5a2..dce2b3598 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,8 +2,8 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import urlparse
 import warnings
+from urllib.parse import urlparse
 
 from w3lib.url import safe_url_string
 
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 30cc926e5..d5d3be67e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,7 +3,7 @@ Scheduler queues
 """
 
 import marshal
-from six.moves import cPickle as pickle
+import pickle
 
 from queuelib import queue
 
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 5bbda6e27..cdbe21942 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -1,5 +1,6 @@
 import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
 from twisted.web.server import Site
 from twisted.web.resource import Resource
 from twisted.internet import reactor
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index b3fd0a497..a0a47e473 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -1,9 +1,9 @@
 import argparse
 import warnings
 from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
 
-from six.moves.http_cookies import SimpleCookie
-from six.moves.urllib.parse import urlparse
 from six import string_types, iteritems
 from w3lib.http import basic_auth_header
 
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index b4c929b0e..54ffe086d 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,8 +1,8 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
 import weakref
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 
 _urlparse_cache = weakref.WeakKeyDictionary()
 def urlparse_cached(request_or_response):
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 1cbda141a..f28c2eaa1 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,5 +1,5 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 import warnings
 
 from importlib import import_module
@@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 974abaeb1..cb8ac3c8e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -65,7 +65,7 @@ def is_listlike(x):
     True
     >>> is_listlike((x for x in range(3)))
     True
-    >>> is_listlike(six.moves.xrange(5))
+    >>> is_listlike(range(5))
     True
     """
     return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 45f1ef17e..c5e877acd 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -5,13 +5,13 @@ scrapy.http.Request objects
 
 import hashlib
 import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_unicode
-
 from w3lib.url import canonicalize_url
+
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 _fingerprint_cache = weakref.WeakKeyDictionary()
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 4742b3e13..2f10cf4de 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
+from urllib.parse import urljoin
+
 import lxml.etree
-from six.moves.urllib.parse import urljoin
 
 
 class Sitemap(object):
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 05d06d53b..6f5c21624 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from twisted.internet import reactor
 from twisted.web import server, resource, static, util
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index b3a4be007..4b48868fe 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code

From 5ab0f189ce33f235b847f96b66a243d808b5ce1f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:01:09 -0300
Subject: [PATCH 25/49] Remove six.moves occurrences from tests

---
 tests/mockserver.py                    | 8 ++++++--
 tests/spiders.py                       | 2 +-
 tests/test_crawl.py                    | 2 +-
 tests/test_engine.py                   | 6 ++++--
 tests/test_exporters.py                | 2 +-
 tests/test_feedexport.py               | 6 +++---
 tests/test_http_cookies.py             | 2 +-
 tests/test_http_request.py             | 8 +++-----
 tests/test_pipeline_files.py           | 4 ++--
 tests/test_proxy_connect.py            | 6 +++---
 tests/test_spidermiddleware_offsite.py | 9 ++++-----
 tests/test_spidermiddleware_referer.py | 2 +-
 tests/test_urlparse_monkeypatches.py   | 2 +-
 tests/test_utils_datatypes.py          | 8 --------
 tests/test_utils_defer.py              | 6 ++----
 tests/test_utils_httpobj.py            | 3 ++-
 tests/test_utils_response.py           | 3 ++-
 tests/test_utils_url.py                | 3 ++-
 18 files changed, 39 insertions(+), 43 deletions(-)

diff --git a/tests/mockserver.py b/tests/mockserver.py
index 8aff7b525..15b1b24f5 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,6 +1,10 @@
-import sys, time, random, os, json
-from six.moves.urllib.parse import urlencode
+import json
+import os
+import random
+import sys
+import time
 from subprocess import Popen, PIPE
+from urllib.parse import urlencode
 
 from OpenSSL import SSL
 from twisted.web.server import Site, NOT_DONE_YET
diff --git a/tests/spiders.py b/tests/spiders.py
index 7816bf7c7..72a428c50 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking
 """
 
 import time
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 from scrapy.spiders import Spider
 from scrapy.http import Request
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..3307899b7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -140,7 +140,7 @@ class CrawlTestCase(TestCase):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from six.moves.urllib.parse import urlencode
+        from urllib.parse import urlencode
         query = urlencode({'raw': '''\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1
diff --git a/tests/test_engine.py b/tests/test_engine.py
index d5b911a40..002c4e6bb 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,8 +10,10 @@ module with the ``runserver`` argument::
     python test_engine.py runserver
 """
 
-import sys, os, re
-from six.moves.urllib.parse import urlparse
+import os
+import re
+import sys
+from urllib.parse import urlparse
 
 from twisted.internet import reactor, defer
 from twisted.web import server, static, util
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 7880c5bf8..f151a1285 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,11 +1,11 @@
 import re
 import json
 import marshal
+import pickle
 import tempfile
 import unittest
 from io import BytesIO
 from datetime import datetime
-from six.moves import cPickle as pickle
 
 import lxml.etree
 import six
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index b13b16b41..7a71669c3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2,13 +2,13 @@ import os
 import csv
 import json
 import warnings
-from io import BytesIO
 import tempfile
 import shutil
 import string
+from io import BytesIO
 from unittest import mock
-from six.moves.urllib.parse import urljoin, urlparse, quote
-from six.moves.urllib.request import pathname2url
+from urllib.parse import urljoin, urlparse, quote
+from urllib.request import pathname2url
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 0a9ed500a..45ddb42ba 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 from unittest import TestCase
 
 from scrapy.http import Request, Response
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 807265981..62d2847d7 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -3,12 +3,10 @@ import cgi
 import unittest
 import re
 import json
-from unittest import mock
-from urllib.parse import unquote_to_bytes
+import xmlrpc.client as xmlrpclib
 import warnings
-
-from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse, parse_qs, unquote
+from unittest import mock
+from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_unicode
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index bd40e4103..dede4bf12 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -4,9 +4,9 @@ import time
 from tempfile import mkdtemp
 from shutil import rmtree
 from unittest import mock
-from six.moves.urllib.parse import urlparse
-from six import BytesIO
+from urllib.parse import urlparse
 
+from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ae1236bcb..2b62a378d 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,15 +1,15 @@
 import json
 import os
 import time
-
-from six.moves.urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urlsplit, urlunsplit
 from threading import Thread
+
 from libmproxy import controller, proxy
 from netlib import http_auth
 from testfixtures import LogCapture
-
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
+
 from scrapy.utils.test import get_crawler
 from scrapy.http import Request
 from tests.spiders import SimpleSpider, SingleRequestSpider
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7e4af0d4c..5833591af 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,13 +1,12 @@
 from unittest import TestCase
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+import warnings
 
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
-from scrapy.spidermiddlewares.offsite import URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
 from scrapy.utils.test import get_crawler
-import warnings
+
 
 class TestOffsiteMiddleware(TestCase):
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 21439c20e..23c38d9b1 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 from unittest import TestCase
 import warnings
 
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 22e39821c..bea0cf3e5 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 import unittest
 
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 47877f555..8782e4c53 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase):
         self.assertIn(20, d)
         self.assertNotIn(15, d)
 
-    def test_six_range(self):
-        import six.moves
-        seq = six.moves.range(10**3, 10**6)
-        d = SequenceExclude(seq)
-        self.assertIn(10**2, d)
-        self.assertIn(10**7, d)
-        self.assertNotIn(10**4, d)
-
     def test_range_step(self):
         seq = range(10, 20, 3)
         d = SequenceExclude(seq)
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 003bb9b02..0d8c46657 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,8 +5,6 @@ from twisted.python.failure import Failure
 from scrapy.utils.defer import mustbe_deferred, process_chain, \
     process_chain_both, process_parallel, iter_errback
 
-from six.moves import xrange
-
 
 class MustbeDeferredTest(unittest.TestCase):
     def test_success_function(self):
@@ -83,7 +81,7 @@ class IterErrbackTest(unittest.TestCase):
 
     def test_iter_errback_good(self):
         def itergood():
-            for x in xrange(10):
+            for x in range(10):
                 yield x
 
         errors = []
@@ -93,7 +91,7 @@ class IterErrbackTest(unittest.TestCase):
 
     def test_iter_errback_bad(self):
         def iterbad():
-            for x in xrange(10):
+            for x in range(10):
                 if x == 5:
                     a = 1/0
                 yield x
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 4f9f7a370..cf8ad1f23 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,9 +1,10 @@
 import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 
+
 class HttpobjUtilsTest(unittest.TestCase):
 
     def test_urlparse_cached(self):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index bea4dade3..6ebf290c0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,12 +1,13 @@
 import os
 import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 from scrapy.http import Response, TextResponse, HtmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (response_httprepr, open_in_browser,
                                    get_meta_refresh, get_base_url, response_status_message)
 
+
 __doctests__ = ['scrapy.utils.response']
 
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index e6588055c..a8e37d7b8 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 import unittest
+from urllib.parse import urlparse
 
 import six
-from six.moves.urllib.parse import urlparse
 
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
                               add_http_if_no_scheme, guess_scheme,
                               parse_url, strip_url)
 
+
 __doctests__ = ['scrapy.utils.url']
 
 

From 1aba5136939ff26503a59ca9fb9320a030aa77de Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:26:44 -0300
Subject: [PATCH 26/49] Remove six.iter* occurrences

---
 scrapy/core/downloader/__init__.py            |  3 +--
 scrapy/core/downloader/handlers/__init__.py   |  5 +++--
 scrapy/downloadermiddlewares/cookies.py       |  8 +++++---
 scrapy/downloadermiddlewares/decompression.py |  7 +++----
 scrapy/exporters.py                           |  6 +++---
 scrapy/extensions/memdebug.py                 |  3 +--
 scrapy/http/request/form.py                   |  3 +--
 scrapy/item.py                                |  2 +-
 scrapy/loader/__init__.py                     |  6 ++----
 scrapy/pipelines/files.py                     | 11 +++++------
 scrapy/pipelines/images.py                    |  3 +--
 scrapy/responsetypes.py                       |  3 +--
 scrapy/settings/__init__.py                   |  8 ++++----
 scrapy/utils/conf.py                          | 12 +++++-------
 scrapy/utils/datatypes.py                     |  7 ++-----
 scrapy/utils/python.py                        |  4 ++--
 scrapy/utils/spider.py                        |  5 ++---
 scrapy/utils/trackref.py                      | 13 ++++++-------
 tests/test_settings/__init__.py               | 10 +++++-----
 tests/test_webclient.py                       |  2 +-
 20 files changed, 54 insertions(+), 67 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index f5f8be6e8..73d84664f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -4,7 +4,6 @@ from time import time
 from datetime import datetime
 from collections import deque
 
-import six
 from twisted.internet import reactor, defer, task
 
 from scrapy.utils.defer import mustbe_deferred
@@ -189,7 +188,7 @@ class Downloader(object):
 
     def close(self):
         self._slot_gc_loop.stop()
-        for slot in six.itervalues(self.slots):
+        for slot in self.slots.values():
             slot.close()
 
     def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..39a0b1f51 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,8 +1,9 @@
 """Download handlers for different schemes"""
 
 import logging
+
 from twisted.internet import defer
-import six
+
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
         self._notconfigured = {}  # remembers failed handlers
         handlers = without_none_values(
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
-        for scheme, clspath in six.iteritems(handlers):
+        for scheme, clspath in handlers.items():
             self._schemes[scheme] = clspath
             self._load_handler(scheme, skip_lazy=True)
 
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 0d2b9900c..9deba40d6 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,5 +1,4 @@
 import os
-import six
 import logging
 from collections import defaultdict
 
@@ -8,6 +7,7 @@ from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +82,10 @@ class CookiesMiddleware(object):
 
     def _get_request_cookies(self, jar, request):
         if isinstance(request.cookies, dict):
-            cookie_list = [{'name': k, 'value': v} for k, v in \
-                    six.iteritems(request.cookies)]
+            cookie_list = [
+                {'name': k, 'value': v}
+                for k, v in request.cookies.items()
+            ]
         else:
             cookie_list = request.cookies
 
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index e2d73f347..dfd64c35f 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
-from io import BytesIO
 import zipfile
 import tarfile
 import logging
+from io import BytesIO
 from tempfile import mktemp
 
-import six
-
 from scrapy.responsetypes import responsetypes
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
         if not response.body:
             return response
 
-        for fmt, func in six.iteritems(self._formats):
+        for fmt, func in self._formats.items():
             new_response = func(response)
             if new_response:
                 logger.debug('Decompressed response with format: %(responsefmt)s',
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index d9531e67c..19af2d6e4 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -62,9 +62,9 @@ class BaseItemExporter(object):
             include_empty = self.export_empty_fields
         if self.fields_to_export is None:
             if include_empty and not isinstance(item, dict):
-                field_iter = six.iterkeys(item.fields)
+                field_iter = item.fields.keys()
             else:
-                field_iter = six.iterkeys(item)
+                field_iter = item.keys()
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -326,7 +326,7 @@ class PythonItemExporter(BaseItemExporter):
         return value
 
     def _serialize_dict(self, value):
-        for key, val in six.iteritems(value):
+        for key, val in value.items():
             key = to_bytes(key) if self.binary else key
             yield key, self._serialize_value(val)
 
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 263d8ce4c..892aa8a86 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
 """
 
 import gc
-import six
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
     def spider_closed(self, spider, reason):
         gc.collect()
         self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
-        for cls, wdict in six.iteritems(live_refs):
+        for cls, wdict in live_refs.items():
             if not wdict:
                 continue
             self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index d2bcf77b7..af02c8484 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst
 from urllib.parse import urljoin, urlencode
 
 import lxml.html
-import six
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
     xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/item.py b/scrapy/item.py
index 32f9b2ebb..4e0f0ac44 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -78,7 +78,7 @@ class DictItem(MutableMapping, BaseItem):
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case
-            for k, v in six.iteritems(dict(*args, **kwargs)):
+            for k, v in dict(*args, **kwargs).items():
                 self[k] = v
 
     def __getitem__(self, key):
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 60fd6d222..fe01a856f 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -5,8 +5,6 @@ See documentation in docs/topics/loaders.rst
 """
 from collections import defaultdict
 
-import six
-
 from scrapy.item import Item
 from scrapy.loader.common import wrap_loader_context
 from scrapy.loader.processors import Identity
@@ -72,7 +70,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._add_value(k, v)
         else:
             self._add_value(field_name, value)
@@ -82,7 +80,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._replace_value(k, v)
         else:
             self._replace_value(field_name, value)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 432d4c182..6d55c8980 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -8,14 +8,12 @@ import hashlib
 import logging
 import mimetypes
 import os
-import os.path
 import time
 from collections import defaultdict
 from email.utils import parsedate_tz, mktime_tz
 from io import BytesIO
 from urllib.parse import urlparse
 
-import six
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -153,14 +152,14 @@ class S3FilesStore(object):
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
-                Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+                Metadata={k: str(v) for k, v in (meta or {}).items()},
                 ACL=self.POLICY,
                 **extra)
         else:
             b = self._get_boto_bucket()
             k = b.new_key(key_name)
             if meta:
-                for metakey, metavalue in six.iteritems(meta):
+                for metakey, metavalue in meta.items():
                     k.set_metadata(metakey, str(metavalue))
             h = self.HEADERS.copy()
             if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
             'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
         })
         extra = {}
-        for key, value in six.iteritems(headers):
+        for key, value in headers.items():
             try:
                 kwarg = mapping[key]
             except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
     def persist_file(self, path, buf, info, meta=None, headers=None):
         blob = self.bucket.blob(self.prefix + path)
         blob.cache_control = self.CACHE_CONTROL
-        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e77cef4ff..e9c6b759c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
 import functools
 import hashlib
 from io import BytesIO
-import six
 
 from PIL import Image
 
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
         image, buf = self.convert_image(orig_image)
         yield path, image, buf
 
-        for thumb_id, size in six.iteritems(self.thumbs):
+        for thumb_id, size in self.thumbs.items():
             thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index b64fbbd42..91d309147 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -5,7 +5,6 @@ based on different criteria.
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
-import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
@@ -36,7 +35,7 @@ class ResponseTypes(object):
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types').decode('utf8')
         self.mimetypes.readfp(StringIO(mimedata))
-        for mimetype, cls in six.iteritems(self.CLASSES):
+        for mimetype, cls in self.CLASSES.items():
             self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype):
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c871e86e0..d53b28895 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -317,10 +317,10 @@ class BaseSettings(MutableMapping):
             values = json.loads(values)
         if values is not None:
             if isinstance(values, BaseSettings):
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, values.getpriority(name))
             else:
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, priority)
 
     def delete(self, name, priority='project'):
@@ -377,7 +377,7 @@ class BaseSettings(MutableMapping):
 
     def _to_dict(self):
         return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
-                for k, v in six.iteritems(self)}
+                for k, v in self.items()}
 
     def copy_to_dict(self):
         """
@@ -445,7 +445,7 @@ class Settings(BaseSettings):
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
-        for name, val in six.iteritems(self):
+        for name, val in self.items():
             if isinstance(val, dict):
                 self.set(name, BaseSettings(val, 'default'), 'default')
         self.update(values, priority)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 561bb72fc..7a15e77ff 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,11 +1,9 @@
-from configparser import ConfigParser
 import os
 import sys
 import numbers
+from configparser import ConfigParser
 from operator import itemgetter
 
-import six
-
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
     def _map_keys(compdict):
         if isinstance(compdict, BaseSettings):
             compbs = BaseSettings()
-            for k, v in six.iteritems(compdict):
+            for k, v in compdict.items():
                 prio = compdict.getpriority(k)
                 if compbs.getpriority(convert(k)) == prio:
                     raise ValueError('Some paths in {!r} convert to the same '
@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
             return compbs
         else:
             _check_components(compdict)
-            return {convert(k): v for k, v in six.iteritems(compdict)}
+            return {convert(k): v for k, v in compdict.items()}
 
     def _validate_values(compdict):
         """Fail if a value in the components dict is not a real number or None."""
-        for name, value in six.iteritems(compdict):
+        for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
                 raise ValueError('Invalid value {} for component {}, please provide ' \
                                  'a real number or None instead'.format(value, name))
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
 
     _validate_values(compdict)
     compdict = without_none_values(_map_keys(compdict))
-    return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+    return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
 
 
 def arglist_to_dict(arglist):
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 6e9de47f3..56d4d1b8e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -7,11 +7,8 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
-from collections.abc import Mapping
 import warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -151,7 +148,7 @@ class MultiValueDict(dict):
                         self.setlistdefault(key, []).append(value)
                 except TypeError:
                     raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in six.iteritems(kwargs):
+        for key, value in kwargs.items():
             self.setlistdefault(key, []).append(value)
 
 
@@ -226,7 +223,7 @@ class CaselessDict(dict):
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     def update(self, seq):
-        seq = seq.items() if isinstance(seq, Mapping) else seq
+        seq = seq.items() if isinstance(seq, collections.abc.Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super(CaselessDict, self).update(iseq)
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index cb8ac3c8e..845c19fb9 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -300,7 +300,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
-    for k, v in six.iteritems(dict(dct_or_tuples)):
+    for k, v in dict(dct_or_tuples).items():
         k = k.encode(encoding) if isinstance(k, six.text_type) else k
         if not keys_only:
             v = v.encode(encoding) if isinstance(v, six.text_type) else v
@@ -345,7 +345,7 @@ def without_none_values(iterable):
     value ``None`` have been removed.
     """
     try:
-        return {k: v for k, v in six.iteritems(iterable) if v is not None}
+        return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
         return type(iterable)((v for v in iterable if v is not None))
 
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 94b24f67e..48ad5041e 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,11 +1,10 @@
 import logging
 import inspect
 
-import six
-
 from scrapy.spiders import Spider
 from scrapy.utils.misc import  arg_to_iter
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
     # singleton in scrapy.spider.spiders
     from scrapy.spiders import Spider
 
-    for obj in six.itervalues(vars(module)):
+    for obj in vars(module).values():
         if inspect.isclass(obj) and \
            issubclass(obj, Spider) and \
            obj.__module__ == module.__name__ and \
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 78389e464..4842b95df 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -13,7 +13,6 @@ import weakref
 from time import time
 from operator import itemgetter
 from collections import defaultdict
-import six
 
 
 NoneType = type(None)
@@ -36,13 +35,13 @@ def format_live_refs(ignore=NoneType):
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
-    for cls, wdict in sorted(six.iteritems(live_refs),
+    for cls, wdict in sorted(live_refs.items(),
                              key=lambda x: x[0].__name__):
         if not wdict:
             continue
         if issubclass(cls, ignore):
             continue
-        oldest = min(six.itervalues(wdict))
+        oldest = min(wdict.values())
         s += "%-30s %6d   oldest: %ds ago\n" % (
             cls.__name__, len(wdict), now - oldest
         )
@@ -56,15 +55,15 @@ def print_live_refs(*a, **kw):
 
 def get_oldest(class_name):
     """Get the oldest object for a specific class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             if not wdict:
                 break
-            return min(six.iteritems(wdict), key=itemgetter(1))[0]
+            return min(wdict.items(), key=itemgetter(1))[0]
 
 
 def iter_all(class_name):
     """Iterate over all objects of the same class by its class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
-            return six.iterkeys(wdict)
+            return wdict.keys()
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 32e65bed5..d5cbef6f5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -10,7 +10,7 @@ from . import default_settings
 class SettingsGlobalFuncsTest(unittest.TestCase):
 
     def test_get_settings_priority(self):
-        for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+        for prio_str, prio_num in SETTINGS_PRIORITIES.items():
             self.assertEqual(get_settings_priority(prio_str), prio_num)
         self.assertEqual(get_settings_priority(99), 99)
 
@@ -148,10 +148,10 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertCountEqual(six.iterkeys(self.settings.attributes),
-                              six.iterkeys(ctrl_attributes))
+        self.assertCountEqual(self.settings.attributes.keys(),
+                              ctrl_attributes.keys())
 
-        for key in six.iterkeys(ctrl_attributes):
+        for key in ctrl_attributes.keys():
             attr = self.settings.attributes[key]
             ctrl_attr = ctrl_attributes[key]
             self.assertEqual(attr.value, ctrl_attr.value)
@@ -227,7 +227,7 @@ class BaseSettingsTest(unittest.TestCase):
         }
         settings = self.settings
         settings.attributes = {key: SettingsAttribute(value, 0) for key, value
-                               in six.iteritems(test_configuration)}
+                               in test_configuration.items()}
 
         self.assertTrue(settings.getbool('TEST_ENABLED1'))
         self.assertTrue(settings.getbool('TEST_ENABLED2'))
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..0c04e7114 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -318,7 +318,7 @@ class WebClientTestCase(unittest.TestCase):
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
             # anything.
-            connected = list(six.iterkeys(self.wrapper.protocols))
+            connected = list(self.wrapper.protocols.keys())
             # There might be nothing here if the server managed to already see
             # that the connection was lost.
             if connected:

From 68bf192172e548da49613064086a6e02cf0bae54 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:32:07 -0300
Subject: [PATCH 27/49] Fix bad import

---
 scrapy/exporters.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 19af2d6e4..2ff089ce9 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,7 +8,7 @@ import sys
 import pprint
 import marshal
 import warnings
-from import pickle
+import pickle
 from xml.sax.saxutils import XMLGenerator
 
 import six

From ce8e515fa8960d0229132ac33fc5f8a6e43bbd6f Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:36:25 -0300
Subject: [PATCH 28/49] Remove six type wrappers

---
 scrapy/crawler.py                   |  2 +-
 scrapy/exporters.py                 |  4 ++--
 scrapy/http/headers.py              |  6 +++---
 scrapy/http/request/__init__.py     |  2 +-
 scrapy/http/response/text.py        |  6 +++---
 scrapy/linkextractors/htmlparser.py |  2 +-
 scrapy/linkextractors/lxmlhtml.py   |  2 +-
 scrapy/linkextractors/sgml.py       |  2 +-
 scrapy/settings/__init__.py         | 10 +++++-----
 scrapy/spiders/crawl.py             |  2 +-
 scrapy/spiders/sitemap.py           |  4 ++--
 scrapy/utils/iterators.py           |  6 +++---
 scrapy/utils/misc.py                |  6 +++---
 scrapy/utils/python.py              | 14 +++++++-------
 14 files changed, 34 insertions(+), 34 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 19b998e0d..b88df8f64 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -205,7 +205,7 @@ class CrawlerRunner(object):
         return self._create_crawler(crawler_or_spidercls)
 
     def _create_crawler(self, spidercls):
-        if isinstance(spidercls, six.string_types):
+        if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings)
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2ff089ce9..e45317491 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -182,7 +182,7 @@ class XmlItemExporter(BaseItemExporter):
             for value in serialized_value:
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
-        elif isinstance(serialized_value, six.text_type):
+        elif isinstance(serialized_value, str):
             self.xg.characters(serialized_value)
         else:
             self.xg.characters(str(serialized_value))
@@ -321,7 +321,7 @@ class PythonItemExporter(BaseItemExporter):
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
         encode_func = to_bytes if self.binary else to_unicode
-        if isinstance(value, (six.text_type, bytes)):
+        if isinstance(value, (str, bytes)):
             return encode_func(value, encoding=self.encoding)
         return value
 
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 62507eb19..5bbe7d72a 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -19,7 +19,7 @@ class Headers(CaselessDict):
         """Normalize values to bytes"""
         if value is None:
             value = []
-        elif isinstance(value, (six.text_type, bytes)):
+        elif isinstance(value, (str, bytes)):
             value = [value]
         elif not hasattr(value, '__iter__'):
             value = [value]
@@ -29,10 +29,10 @@ class Headers(CaselessDict):
     def _tobytes(self, x):
         if isinstance(x, bytes):
             return x
-        elif isinstance(x, six.text_type):
+        elif isinstance(x, str):
             return x.encode(self.encoding)
         elif isinstance(x, int):
-            return six.text_type(x).encode(self.encoding)
+            return str(x).encode(self.encoding)
         else:
             raise TypeError('Unsupported value type: {}'.format(type(x)))
 
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index d09eaf849..ff7a44545 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -60,7 +60,7 @@ class Request(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if not isinstance(url, six.string_types):
+        if not isinstance(url, str):
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
 
         s = safe_url_string(url, self.encoding)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 69bcba577..65400803f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -31,14 +31,14 @@ class TextResponse(Response):
         super(TextResponse, self).__init__(*args, **kwargs)
 
     def _set_url(self, url):
-        if isinstance(url, six.text_type):
+        if isinstance(url, str):
             self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
     def _set_body(self, body):
         self._body = b''  # used by encoding detection
-        if isinstance(body, six.text_type):
+        if isinstance(body, str):
             if self._encoding is None:
                 raise TypeError('Cannot convert unicode body - %s has no encoding' %
                     type(self).__name__)
@@ -158,7 +158,7 @@ class TextResponse(Response):
 
 def _url_from_selector(sel):
     # type: (parsel.Selector) -> str
-    if isinstance(sel.root, six.string_types):
+    if isinstance(sel.root, str):
         # e.g. ::attr(href) result
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 623732cee..2fec35799 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -42,7 +42,7 @@ class HtmlParserLinkExtractor(HTMLParser):
         ret = []
         base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in links:
-            if isinstance(link.url, six.text_type):
+            if isinstance(link.url, str):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 5b7e709de..496ad3053 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()")
 
 
 def _nons(tag):
-    if isinstance(tag, six.string_types):
+    if isinstance(tag, str):
         if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
             return tag.split('}')[-1]
     return tag
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 98bed15e9..a9dffcad3 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -49,7 +49,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if base_url is None:
             base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in self.links:
-            if isinstance(link.url, six.text_type):
+            if isinstance(link.url, str):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index d53b28895..c88c9c0e2 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -23,7 +23,7 @@ def get_settings_priority(priority):
     :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
     numerical value, or directly returns a given numerical priority.
     """
-    if isinstance(priority, six.string_types):
+    if isinstance(priority, str):
         return SETTINGS_PRIORITIES[priority]
     else:
         return priority
@@ -173,7 +173,7 @@ class BaseSettings(MutableMapping):
         :type default: any
         """
         value = self.get(name, default or [])
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = value.split(',')
         return list(value)
 
@@ -194,7 +194,7 @@ class BaseSettings(MutableMapping):
         :type default: any
         """
         value = self.get(name, default or {})
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = json.loads(value)
         return dict(value)
 
@@ -284,7 +284,7 @@ class BaseSettings(MutableMapping):
         :type priority: string or int
         """
         self._assert_mutability()
-        if isinstance(module, six.string_types):
+        if isinstance(module, str):
             module = import_module(module)
         for key in dir(module):
             if key.isupper():
@@ -313,7 +313,7 @@ class BaseSettings(MutableMapping):
         :type priority: string or int
         """
         self._assert_mutability()
-        if isinstance(values, six.string_types):
+        if isinstance(values, str):
             values = json.loads(values)
         if values is not None:
             if isinstance(values, BaseSettings):
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 03000ce54..59e2c5661 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -25,7 +25,7 @@ def _identity(request, response):
 def _get_method(method, spider):
     if callable(method):
         return method
-    elif isinstance(method, six.string_types):
+    elif isinstance(method, str):
         return getattr(spider, method, None)
 
 
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 534c45c70..2917daf57 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -22,7 +22,7 @@ class SitemapSpider(Spider):
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
-            if isinstance(c, six.string_types):
+            if isinstance(c, str):
                 c = getattr(self, c)
             self._cbs.append((regex(r), c))
         self._follow = [regex(x) for x in self.sitemap_follow]
@@ -86,7 +86,7 @@ class SitemapSpider(Spider):
 
 
 def regex(x):
-    if isinstance(x, six.string_types):
+    if isinstance(x, str):
         return re.compile(x)
     return x
 
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9693ba768..10481fe8d 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -60,7 +60,7 @@ class _StreamReader(object):
             self._text, self.encoding = obj.body, obj.encoding
         else:
             self._text, self.encoding = obj, 'utf-8'
-        self._is_unicode = isinstance(self._text, six.text_type)
+        self._is_unicode = isinstance(self._text, str)
 
     def read(self, n=65535):
         self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    expected_types = (Response, six.text_type, six.binary_type)
+    expected_types = (Response, str, bytes)
     assert isinstance(obj, expected_types), \
         "obj must be %s, not %s" % (
             " or ".join(t.__name__ for t in expected_types),
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
             return obj.text
         else:
             return obj.body.decode('utf-8')
-    elif isinstance(obj, six.text_type):
+    elif isinstance(obj, str):
         return obj if unicode else obj.encode('utf-8')
     else:
         return obj.decode('utf-8') if unicode else obj
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index f638adb25..9a44f3576 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -13,7 +13,7 @@ from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
 
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
 
 
 def arg_to_iter(arg):
@@ -83,7 +83,7 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex doesn't contain any group the entire regex matching is returned
     """
 
-    if isinstance(regex, six.string_types):
+    if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)
 
     try:
@@ -92,7 +92,7 @@ def extract_regex(regex, text, encoding='utf-8'):
         strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
-    if isinstance(text, six.text_type):
+    if isinstance(text, str):
         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
     else:
         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 845c19fb9..18fee1964 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -68,7 +68,7 @@ def is_listlike(x):
     >>> is_listlike(range(5))
     True
     """
-    return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+    return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
 def unique(list_, key=lambda x: x):
@@ -87,9 +87,9 @@ def unique(list_, key=lambda x: x):
 def to_unicode(text, encoding=None, errors='strict'):
     """Return the unicode representation of a bytes object ``text``. If
     ``text`` is already an unicode object, return it as-is."""
-    if isinstance(text, six.text_type):
+    if isinstance(text, str):
         return text
-    if not isinstance(text, (bytes, six.text_type)):
+    if not isinstance(text, (bytes, str)):
         raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
@@ -102,7 +102,7 @@ def to_bytes(text, encoding=None, errors='strict'):
     is already a bytes object, return it as-is."""
     if isinstance(text, bytes):
         return text
-    if not isinstance(text, six.string_types):
+    if not isinstance(text, str):
         raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
@@ -138,7 +138,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
-    if isinstance(pattern, six.string_types):
+    if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
     for chunk, offset in _chunk_iter():
@@ -301,9 +301,9 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     """
     d = {}
     for k, v in dict(dct_or_tuples).items():
-        k = k.encode(encoding) if isinstance(k, six.text_type) else k
+        k = k.encode(encoding) if isinstance(k, str) else k
         if not keys_only:
-            v = v.encode(encoding) if isinstance(v, six.text_type) else v
+            v = v.encode(encoding) if isinstance(v, str) else v
         d[k] = v
     return d
 

From 54a786b102a3862c94fcdb123a26b54a39892697 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 00:58:47 -0300
Subject: [PATCH 29/49] Remove six imports

---
 scrapy/crawler.py                   | 6 +++---
 scrapy/exporters.py                 | 2 --
 scrapy/http/headers.py              | 4 ----
 scrapy/http/request/__init__.py     | 1 -
 scrapy/http/response/text.py        | 1 -
 scrapy/linkextractors/htmlparser.py | 1 -
 scrapy/linkextractors/lxmlhtml.py   | 1 -
 scrapy/linkextractors/sgml.py       | 1 -
 scrapy/settings/__init__.py         | 1 -
 scrapy/spiders/crawl.py             | 2 --
 scrapy/spiders/sitemap.py           | 1 -
 scrapy/utils/curl.py                | 3 +--
 scrapy/utils/iterators.py           | 6 +++---
 scrapy/utils/misc.py                | 1 -
 tests/test_http_headers.py          | 3 ---
 tests/test_http_request.py          | 2 +-
 16 files changed, 8 insertions(+), 28 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index b88df8f64..4d7d9bac4 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,9 +1,8 @@
-import six
-import signal
 import logging
+import signal
+import sys
 import warnings
 
-import sys
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
@@ -22,6 +21,7 @@ from scrapy.utils.log import (
     get_scrapy_root_handler, install_scrapy_root_handler)
 from scrapy import signals
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index e45317491..f2999daea 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -11,8 +11,6 @@ import warnings
 import pickle
 from xml.sax.saxutils import XMLGenerator
 
-import six
-
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 5bbe7d72a..860a5c9c6 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,4 +1,3 @@
-import six
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
@@ -68,9 +67,6 @@ class Headers(CaselessDict):
         self[key] = lst
 
     def items(self):
-        return list(self.iteritems())
-
-    def iteritems(self):
         return ((k, self.getlist(k)) for k in self.keys())
 
     def values(self):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index ff7a44545..1e4a9c166 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,7 +4,6 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-import six
 from w3lib.url import safe_url_string
 
 from scrapy.http.headers import Headers
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 65400803f..1079fd6e8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst
 from urllib.parse import urljoin
 
 import parsel
-import six
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 2fec35799..0425d4340 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -5,7 +5,6 @@ import warnings
 from html.parser import HTMLParser
 from urllib.parse import urljoin
 
-import six
 from w3lib.url import safe_url_string
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 496ad3053..cb55e805a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -3,7 +3,6 @@ Link extractor based on lxml.html
 """
 from urllib.parse import urljoin
 
-import six
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index a9dffcad3..2ba6bca45 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -5,7 +5,6 @@ import warnings
 from urllib.parse import urljoin
 from sgmllib import SGMLParser
 
-import six
 from w3lib.url import safe_url_string, canonicalize_url
 from w3lib.html import strip_html5_whitespace
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c88c9c0e2..b6133619c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,4 +1,3 @@
-import six
 import json
 import copy
 from collections.abc import MutableMapping
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 59e2c5661..a5eb1a518 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst
 import copy
 import warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 2917daf57..d368c7108 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,6 +1,5 @@
 import re
 import logging
-import six
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index a0a47e473..16639356e 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -4,7 +4,6 @@ from shlex import split
 from http.cookies import SimpleCookie
 from urllib.parse import urlparse
 
-from six import string_types, iteritems
 from w3lib.http import basic_auth_header
 
 
@@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
         name = name.strip()
         val = val.strip()
         if name.title() == 'Cookie':
-            for name, morsel in iteritems(SimpleCookie(val)):
+            for name, morsel in SimpleCookie(val).items():
                 cookies[name] = morsel.value
         else:
             headers.append((name, val))
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 10481fe8d..3c0cb68c3 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,13 +1,13 @@
-import re
 import csv
-from io import StringIO
 import logging
-import six
+import re
+from io import StringIO
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 9a44f3576..0de3f18b3 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -6,7 +6,6 @@ from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
-import six
 from w3lib.html import replace_entities
 
 from scrapy.utils.python import flatten, to_unicode
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 69d906fbf..50763d8f7 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -85,9 +85,6 @@ class HeadersTest(unittest.TestCase):
         self.assertSortedEqual(h.items(),
                                [(b'X-Forwarded-For', [b'ip1', b'ip2']),
                                 (b'Content-Type', [b'text/html'])])
-        self.assertSortedEqual(h.iteritems(),
-                               [(b'X-Forwarded-For', [b'ip1', b'ip2']),
-                                (b'Content-Type', [b'text/html'])])
         self.assertSortedEqual(h.values(), [b'ip2', b'text/html'])
 
     def test_update(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 62d2847d7..988c8a811 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -62,7 +62,7 @@ class RequestTest(unittest.TestCase):
         # headers must not be unicode
         h = Headers({'key1': u'val1', u'key2': 'val2'})
         h[u'newkey'] = u'newval'
-        for k, v in h.iteritems():
+        for k, v in h.items():
             self.assertIsInstance(k, bytes)
             for s in v:
                 self.assertIsInstance(s, bytes)

From ac62524824c590e9dd2d323bf418f71a683ecbf4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:00:54 -0300
Subject: [PATCH 30/49] Remove six.get_method_*

---
 scrapy/core/downloader/middleware.py | 6 +++---
 scrapy/core/spidermw.py              | 4 ++--
 scrapy/utils/reqser.py               | 4 ++--
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 7a6a4dfac..72432558a 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -38,7 +38,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
+                                         (method.__self__.__class__.__name__, response.__class__.__name__))
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -53,7 +53,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, response=response, spider=spider)
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, type(response)))
+                                         (method.__self__.__class__.__name__, type(response)))
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -65,7 +65,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, exception=exception, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, type(response)))
+                                         (method.__self__.__class__.__name__, type(response)))
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b5f9837ff..57436d2f6 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -37,8 +37,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f:'%s.%s' % (
-                six.get_method_self(f).__class__.__name__,
-                six.get_method_function(f).__name__)
+                f.__self__.__class__.__name__,
+                f.__func__.__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 495564ac0..e961ffca9 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -87,12 +87,12 @@ def _mangle_private_name(obj, func, name):
 def _find_method(obj, func):
     if obj:
         try:
-            func_self = six.get_method_self(func)
+            func_self = func.__self__
         except AttributeError:  # func has no __self__
             pass
         else:
             if func_self is obj:
-                name = six.get_method_function(func).__name__
+                name = func.__func__.__name__
                 if _is_private_method(name):
                     return _mangle_private_name(obj, func, name)
                 return name

From 5d8abdde59e7501e8bda25b27c8926fda66b9af1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:01:10 -0300
Subject: [PATCH 31/49] Remove six.text_type from tests

---
 tests/test_exporters.py       |  2 +-
 tests/test_http_response.py   |  8 ++++----
 tests/test_loader.py          | 14 +++++++-------
 tests/test_logformatter.py    |  4 ++--
 tests/test_toplevel.py        |  2 +-
 tests/test_utils_iterators.py |  4 ++--
 tests/test_utils_python.py    | 14 +++++++-------
 7 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index f151a1285..8433fa4db 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -79,7 +79,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
         ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
         _, name = list(ie._get_serialized_fields(self.i))[0]
-        assert isinstance(name, six.text_type)
+        assert isinstance(name, str)
         self.assertEqual(name, u'John\xa3')
 
     def test_field_custom_serializer(self):
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 80bf51647..1d121cc83 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(r4.flags, [])
 
     def _assert_response_values(self, response, encoding, body):
-        if isinstance(body, six.text_type):
+        if isinstance(body, str):
             body_unicode = body
             body_bytes = body.encode(encoding)
         else:
@@ -110,7 +110,7 @@ class BaseResponseTest(unittest.TestCase):
             body_bytes = body
 
         assert isinstance(response.body, bytes)
-        assert isinstance(response.text, six.text_type)
+        assert isinstance(response.text, str)
         self._assert_response_encoding(response, encoding)
         self.assertEqual(response.body, body_bytes)
         self.assertEqual(response.body_as_unicode(), body_unicode)
@@ -220,11 +220,11 @@ class TextResponseTest(BaseResponseTest):
         r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
 
         # check body_as_unicode
-        self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
+        self.assertTrue(isinstance(r1.body_as_unicode(), str))
         self.assertEqual(r1.body_as_unicode(), unicode_string)
 
         # check response.text
-        self.assertTrue(isinstance(r1.text, six.text_type))
+        self.assertTrue(isinstance(r1.text, str))
         self.assertEqual(r1.text, unicode_string)
 
     def test_encoding(self):
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 4a4264a2a..f1cf0114f 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -157,7 +157,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
         self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
         self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
 
@@ -258,7 +258,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
+            name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
 
         il = ChildItemLoader()
         il.add_value('name', u'marta')
@@ -266,7 +266,7 @@ class BasicItemLoaderTest(unittest.TestCase):
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
 
         il = ChildDefaultedItemLoader()
         il.add_value('name', u'marta')
@@ -689,7 +689,7 @@ class ProcessorsTest(unittest.TestCase):
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
         self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
         self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assertIsInstance(proc(['hello', 'world']), six.text_type)
+        self.assertIsInstance(proc(['hello', 'world']), str)
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
@@ -704,12 +704,12 @@ class ProcessorsTest(unittest.TestCase):
     def test_mapcompose(self):
         def filter_world(x):
             return None if x == 'world' else x
-        proc = MapCompose(filter_world, six.text_type.upper)
+        proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
-        proc = MapCompose(filter_world, six.text_type.upper)
+        proc = MapCompose(filter_world, str.upper)
         self.assertEqual(proc(None), [])
-        proc = MapCompose(filter_world, six.text_type.upper)
+        proc = MapCompose(filter_world, str.upper)
         self.assertRaises(ValueError, proc, [1])
         proc = MapCompose(filter_world, lambda x: x + 1)
         self.assertRaises(ValueError, proc, 'hello')
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index eb9c4a561..ca90de9d2 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -59,7 +59,7 @@ class LoggingContribTest(unittest.TestCase):
         logkws = self.formatter.dropped(item, exception, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
-        assert all(isinstance(x, six.text_type) for x in lines)
+        assert all(isinstance(x, str) for x in lines)
         self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
 
     def test_scraped(self):
@@ -69,7 +69,7 @@ class LoggingContribTest(unittest.TestCase):
         logkws = self.formatter.scraped(item, response, self.spider)
         logline = logkws['msg'] % logkws['args']
         lines = logline.splitlines()
-        assert all(isinstance(x, six.text_type) for x in lines)
+        assert all(isinstance(x, str) for x in lines)
         self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
 
 
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 91bbe43bc..6d305249a 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -6,7 +6,7 @@ import scrapy
 class ToplevelTestCase(TestCase):
 
     def test_version(self):
-        self.assertIs(type(scrapy.__version__), six.text_type)
+        self.assertIs(type(scrapy.__version__), str)
 
     def test_version_info(self):
         self.assertIs(type(scrapy.version_info), tuple)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 2d845697e..0910bd560 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -255,8 +255,8 @@ class UtilsCsvTestCase(unittest.TestCase):
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
         for result_row in result:
-            self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys())))
-            self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values())))
+            self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
+            self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
         body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 096aa50b7..bea431969 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -169,8 +169,8 @@ class UtilsPythonTestCase(unittest.TestCase):
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
         self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.keys()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.values()))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_tuples(self):
@@ -179,8 +179,8 @@ class UtilsPythonTestCase(unittest.TestCase):
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
         self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.keys()), d2.keys())
+        self.assertFalse(any(isinstance(x, str) for x in d2.values()))
 
     @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_keys_only(self):
@@ -188,7 +188,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
         self.assertIsNot(d, d2)  # shouldn't modify in place
-        self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.assertFalse(any(isinstance(x, str) for x in d2.keys()))
 
     def test_get_func_args(self):
         def f1(a, b, c):
@@ -227,12 +227,12 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         if platform.python_implementation() == 'CPython':
             # TODO: how do we fix this to return the actual argument names?
-            self.assertEqual(get_func_args(six.text_type.split), [])
+            self.assertEqual(get_func_args(str.split), [])
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
             self.assertEqual(
-                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
+                get_func_args(str.split, True), ['sep', 'maxsplit'])
             self.assertEqual(get_func_args(" ".join, True), ['list'])
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), True), ['obj'])

From eaeaa40b991bcb2113ba63146fbe4b2dc9e93016 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:08:08 -0300
Subject: [PATCH 32/49] Remove six.PY* checks

---
 conftest.py             | 12 +++++-------
 tests/test_webclient.py | 20 --------------------
 2 files changed, 5 insertions(+), 27 deletions(-)

diff --git a/conftest.py b/conftest.py
index 06d65ba1d..5e6a42977 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,4 +1,3 @@
-import six
 import pytest
 
 
@@ -7,12 +6,11 @@ collect_ignore = [
     "scrapy/utils/testsite.py",
 ]
 
-
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+# FIXME: fix or delete these tests
+for line in open('tests/py3-ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 0c04e7114..608cfe597 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase):
                 to_bytes(x) if not isinstance(x, int) else x for x in test)
             self.assertEqual(client._parse(url), test, url)
 
-    def test_externalUnicodeInterference(self):
-        """
-        L{client._parse} should return C{str} for the scheme, host, and path
-        elements of its return tuple, even when passed an URL which has
-        previously been passed to L{urlparse} as a C{unicode} string.
-        """
-        if not six.PY2:
-            raise unittest.SkipTest(
-                "Applies only to Py2, as urls can be ONLY unicode on Py3")
-        badInput = u'http://example.com/path'
-        goodInput = badInput.encode('ascii')
-        self._parse(badInput)  # cache badInput in urlparse_cached
-        scheme, netloc, host, port, path = self._parse(goodInput)
-        self.assertTrue(isinstance(scheme, str))
-        self.assertTrue(isinstance(netloc, str))
-        self.assertTrue(isinstance(host, str))
-        self.assertTrue(isinstance(path, str))
-        self.assertTrue(isinstance(port, int))
-
-
 
 class ScrapyHTTPPageGetterTests(unittest.TestCase):
 

From d72444b9c8db94dd5567e763ee53b2c70a423c9a Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 01:11:23 -0300
Subject: [PATCH 33/49] Remove more six imports

---
 scrapy/core/downloader/middleware.py | 2 --
 scrapy/core/spidermw.py              | 1 -
 scrapy/utils/reqser.py               | 2 --
 tests/test_exporters.py              | 1 -
 tests/test_http_response.py          | 1 -
 tests/test_loader.py                 | 2 --
 tests/test_logformatter.py           | 1 -
 tests/test_toplevel.py               | 2 +-
 tests/test_utils_iterators.py        | 3 ++-
 tests/test_utils_reqser.py           | 2 --
 tests/test_utils_url.py              | 2 --
 tests/test_webclient.py              | 1 -
 12 files changed, 3 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 72432558a..38608a429 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,6 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-import six
-
 from twisted.internet import defer
 
 from scrapy.exceptions import _InvalidOutput
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 57436d2f6..e4c6df8c7 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import chain, islice
 
-import six
 from twisted.python.failure import Failure
 from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index e961ffca9..749bbc387 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,8 +1,6 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
-import six
-
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 8433fa4db..5d1f5c182 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -8,7 +8,6 @@ from io import BytesIO
 from datetime import datetime
 
 import lxml.etree
-import six
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 1d121cc83..ee7177ceb 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 import unittest
 
-import six
 from w3lib.encoding import resolve_encoding
 
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
diff --git a/tests/test_loader.py b/tests/test_loader.py
index f1cf0114f..69ded4d50 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,8 +1,6 @@
 from functools import partial
 import unittest
 
-import six
-
 from scrapy.http import HtmlResponse
 from scrapy.item import Item, Field
 from scrapy.loader import ItemLoader
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index ca90de9d2..5dc077c5b 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -3,7 +3,6 @@ import unittest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase as TwistedTestCase
-import six
 
 from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import DropItem
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 6d305249a..fdc5df166 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,5 +1,5 @@
 from unittest import TestCase
-import six
+
 import scrapy
 
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 0910bd560..4d69edb31 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 import os
-import six
+
 from twisted.trial import unittest
 
 from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
 from scrapy.http import XmlResponse, TextResponse, Response
 from tests import get_testdata
 
+
 FOOBAR_NL = u"foo\nbar"
 
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 92cd16de7..b5729b086 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -2,8 +2,6 @@
 import unittest
 import sys
 
-import six
-
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
 from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index a8e37d7b8..c5fdc752b 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -2,8 +2,6 @@
 import unittest
 from urllib.parse import urlparse
 
-import six
-
 from scrapy.spiders import Spider
 from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
                               add_http_if_no_scheme, guess_scheme,
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 608cfe597..746367b41 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,7 +3,6 @@ from twisted.internet import defer
 Tests borrowed from the twisted.web.client tests.
 """
 import os
-import six
 import shutil
 
 import OpenSSL.SSL

From e461570f991cf70d17a784aa628496521c241873 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 2 Nov 2019 23:13:54 -0300
Subject: [PATCH 34/49] Remove protego from requirements file

---
 requirements-py3.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2c98e6f6d..cd183a525 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -2,7 +2,6 @@ parsel>=1.5.0
 PyDispatcher>=2.0.5
 Twisted>=17.9.0
 w3lib>=1.17.0
-protego>=0.1.15
 
 pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
 queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"

From 7f3cb05d8e20048eb3e37c5e83bcf495d28c6064 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:03:02 -0300
Subject: [PATCH 35/49] Remove metaclass-related six code

---
 scrapy/item.py     | 5 +----
 tests/test_item.py | 4 +---
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index 4e0f0ac44..1d39b48b2 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -10,8 +10,6 @@ from copy import deepcopy
 from pprint import pformat
 from warnings import warn
 
-import six
-
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
@@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
         return deepcopy(self)
 
 
-@six.add_metaclass(ItemMeta)
-class Item(DictItem):
+class Item(DictItem, metaclass=ItemMeta):
     pass
diff --git a/tests/test_item.py b/tests/test_item.py
index d98c63ddd..0da8fa1ac 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -3,8 +3,6 @@ import unittest
 from unittest import mock
 from warnings import catch_warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
 
@@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase):
 class ItemMetaClassCellRegression(unittest.TestCase):
 
     def test_item_meta_classcell_regression(self):
-        class MyItem(six.with_metaclass(ItemMeta, Item)):
+        class MyItem(Item, metaclass=ItemMeta):
             def __init__(self, *args, **kwargs):
                 # This call to super() trigger the __classcell__ propagation
                 # requirement. When not done properly raises an error:

From 586b25d27e1641433d505338901fa9a1409ccdd4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:10:37 -0300
Subject: [PATCH 36/49] Remove six types

---
 scrapy/downloadermiddlewares/ajaxcrawl.py | 3 +--
 scrapy/utils/python.py                    | 3 +--
 tests/test_utils_trackref.py              | 7 ++++---
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 78b802673..c618e9ffc 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -2,7 +2,6 @@
 import re
 import logging
 
-import six
 from w3lib import html
 
 from scrapy.exceptions import NotConfigured
@@ -66,7 +65,7 @@ class AjaxCrawlMiddleware(object):
 
 
 # XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
+_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
 def _has_ajaxcrawlable_meta(text):
     """
     >>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment"  content="!"/></head><body></body></html>')
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 18fee1964..d32ee5a3a 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,7 +7,6 @@ import re
 import inspect
 import weakref
 import errno
-import six
 from functools import partial, wraps
 from itertools import chain
 import sys
@@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
     return new_method
 
 
-_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
 _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 @deprecated("scrapy.utils.python.binary_is_text")
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 480a717e7..16e02f919 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,6 +1,7 @@
-import six
 import unittest
+from io import StringIO
 from unittest import mock
+
 from scrapy.utils import trackref
 
 
@@ -38,12 +39,12 @@ Live References
 Bar                                 1   oldest: 0s ago
 ''')
 
-    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    @mock.patch('sys.stdout', new_callable=StringIO)
     def test_print_live_refs_empty(self, stdout):
         trackref.print_live_refs()
         self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
 
-    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    @mock.patch('sys.stdout', new_callable=StringIO)
     def test_print_live_refs_with_objects(self, stdout):
         o1 = Foo()  # NOQA
         trackref.print_live_refs()

From 5797aefd4c561666c2c047494b7424d77eebb469 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:18:35 -0300
Subject: [PATCH 37/49] Remove six.assertCountEqual

---
 tests/test_cmdline/__init__.py  |  7 +++----
 tests/test_settings/__init__.py | 14 ++++++--------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 56cfe642a..909ea90e0 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,13 +1,12 @@
-from io import StringIO
 import json
 import os
 import pstats
 import shutil
-import six
-from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
+from io import StringIO
+from subprocess import Popen, PIPE
 
 from scrapy.utils.test import get_testenv
 
@@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase):
         for char in ("'", "<", ">", 'u"'):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
-        six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
+        self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
         self.assertEqual(200, settingsdict[EXT_PATH])
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index d5cbef6f5..fda44653a 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,4 +1,3 @@
-import six
 import unittest
 from unittest import mock
 
@@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase):
         new_dict = {'three': 11, 'four': 21}
         attribute.set(new_dict, 10)
         self.assertIsInstance(attribute.value, BaseSettings)
-        six.assertCountEqual(self, attribute.value, new_dict)
-        six.assertCountEqual(self, original_settings, original_dict)
+        self.assertCountEqual(attribute.value, new_dict)
+        self.assertCountEqual(original_settings, original_dict)
 
         new_settings = BaseSettings({'five': 12}, 0)
         attribute.set(new_settings, 0)  # Insufficient priority
-        six.assertCountEqual(self, attribute.value, new_dict)
+        self.assertCountEqual(attribute.value, new_dict)
         attribute.set(new_settings, 10)
-        six.assertCountEqual(self, attribute.value, new_settings)
+        self.assertCountEqual(attribute.value, new_settings)
 
     def test_repr(self):
         self.assertEqual(repr(self.attribute),
@@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase):
                           'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
                           'HASNOBASE': BaseSettings({3: 3000}, 'default')})
         s['TEST'].set(2, 200, 'cmdline')
-        six.assertCountEqual(self, s.getwithbase('TEST'),
-                             {1: 1, 2: 200, 3: 30})
-        six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+        self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
+        self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
         self.assertEqual(s.getwithbase('NONEXISTENT'), {})
 
     def test_maxpriority(self):

From 00b793dc59a32d13c89ac5c0d54985677b228801 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:26:38 -0300
Subject: [PATCH 38/49] Remove elluding six occurrences

---
 scrapy/robotstxt.py          | 6 ++++--
 tests/test_contracts.py      | 5 ++---
 tests/test_pipeline_files.py | 2 +-
 tests/test_utils_curl.py     | 7 ++-----
 4 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 397924110..0a9af3a62 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,12 +1,13 @@
 import sys
 import logging
 from abc import ABCMeta, abstractmethod
-from six import with_metaclass
 
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
+
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
@@ -23,7 +24,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         robotstxt_body = ''
     return robotstxt_body
 
-class RobotParser(with_metaclass(ABCMeta)):
+
+class RobotParser(metaclass=ABCMeta):
     @classmethod
     @abstractmethod
     def from_crawler(cls, crawler, robotstxt_body):
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b2e358700..582e3d052 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,6 +1,5 @@
 from unittest import TextTestResult
 
-from six import get_unbound_function
 from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
@@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
         with MockServer() as mockserver:
             contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
 
-            get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
-            get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
+            TestSameUrlSpider.parse_first.__doc__ = contract_doc
+            TestSameUrlSpider.parse_second.__doc__ = contract_doc
 
             crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
             yield crawler.crawl()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index dede4bf12..52f2b554e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,12 +1,12 @@
 import os
 import random
 import time
+from io import BytesIO
 from tempfile import mkdtemp
 from shutil import rmtree
 from unittest import mock
 from urllib.parse import urlparse
 
-from six import BytesIO
 from twisted.trial import unittest
 from twisted.internet import defer
 
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index c5655df7e..50e1bfd5f 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,7 +1,6 @@
 import unittest
 import warnings
 
-from six import assertRaisesRegex
 from w3lib.http import basic_auth_header
 
 from scrapy import Request
@@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
         self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
     def test_too_few_arguments_error(self):
-        assertRaisesRegex(
-            self,
+        self.assertRaisesRegex(
             ValueError,
             r"too few arguments|the following arguments are required:\s*url",
             lambda: curl_to_request_kwargs("curl"),
@@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
             self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
 
         # case 2: ignore_unknown_options=False (raise exception):
-        assertRaisesRegex(
-            self,
+        self.assertRaisesRegex(
             ValueError,
             "Unrecognized options:.*--bar.*--baz",
             lambda: curl_to_request_kwargs(

From 0c4e5b68ea0e077033fde5da28c6dc4fdd859d92 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sun, 3 Nov 2019 12:30:34 -0300
Subject: [PATCH 39/49] Remove six from requirements and setup files

---
 requirements-py3.txt | 1 -
 setup.py             | 1 -
 2 files changed, 2 deletions(-)

diff --git a/requirements-py3.txt b/requirements-py3.txt
index cd183a525..28c649e28 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -13,5 +13,4 @@ cryptography>=2.0  # Earlier versions would fail to install
 cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
-six>=1.10.0
 zope.interface>=4.1.3
diff --git a/setup.py b/setup.py
index 8f5f14f0d..85d797f88 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,6 @@ setup(
         'pyOpenSSL>=16.2.0',
         'queuelib>=1.4.2',
         'service_identity>=16.0.0',
-        'six>=1.10.0',
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',

From fe31695ba0266deaa94222fc01885b7270af4294 Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Tue, 5 Nov 2019 15:36:19 -0300
Subject: [PATCH 40/49] Remove unused import (urllib.parse.unquote)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 tests/test_http_request.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 988c8a811..05cac617c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -6,7 +6,7 @@ import json
 import xmlrpc.client as xmlrpclib
 import warnings
 from unittest import mock
-from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse
+from urllib.parse import parse_qs, unquote_to_bytes, urlparse
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_unicode

From 44f19df3119d553aa5c002321bd901a424c4bb2c Mon Sep 17 00:00:00 2001
From: elacuesta <elacuesta@users.noreply.github.com>
Date: Fri, 8 Nov 2019 11:32:50 -0300
Subject: [PATCH 41/49] [test] Update mitmproxy version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-Authored-By: Adrián Chaves <adrian@chaves.io>
---
 tests/requirements-py3.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index c2b16bec6..7abb66b9c 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,7 @@
 # Tests requirements
 jmespath
 mitmproxy; python_version >= '3.6'
-mitmproxy==3.0.4; python_version < '3.6'
+mitmproxy<4.0.0; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted

From 6cde428af43a5c8268208c6e4e239ab5ce507af4 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 Nov 2019 12:26:40 -0300
Subject: [PATCH 42/49] Remove deprecated MergeDict class

---
 scrapy/utils/datatypes.py | 59 ---------------------------------------
 1 file changed, 59 deletions(-)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 56d4d1b8e..e194a7613 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -235,65 +235,6 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class MergeDict(object):
-    """
-    A simple class for creating new "virtual" dictionaries that actually look
-    up values in more than one dictionary, passed in the constructor.
-
-    If a key appears in more than one of the given dictionaries, only the
-    first occurrence will be used.
-    """
-    def __init__(self, *dicts):
-        warnings.warn(
-            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-            "of collections.ChainMap (introduced in Python 3.3)",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        self.dicts = dicts
-
-    def __getitem__(self, key):
-        for dict_ in self.dicts:
-            try:
-                return dict_[key]
-            except KeyError:
-                pass
-        raise KeyError
-
-    def __copy__(self):
-        return self.__class__(*self.dicts)
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def getlist(self, key):
-        for dict_ in self.dicts:
-            if key in dict_.keys():
-                return dict_.getlist(key)
-        return []
-
-    def items(self):
-        item_list = []
-        for dict_ in self.dicts:
-            item_list.extend(dict_.items())
-        return item_list
-
-    def has_key(self, key):
-        for dict_ in self.dicts:
-            if key in dict_:
-                return True
-        return False
-
-    __contains__ = has_key
-
-    def copy(self):
-        """Returns a copy of this object."""
-        return self.__copy__()
-
-
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 

From b6bbb2819707a2202c87abd6b3dba6af13a7cc85 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 8 Nov 2019 22:13:03 -0300
Subject: [PATCH 43/49] PEP8 adjustments

---
 scrapy/crawler.py            |  1 -
 scrapy/exporters.py          |  1 -
 scrapy/http/cookies.py       |  2 +-
 scrapy/link.py               |  2 ++
 tests/test_pipeline_media.py |  2 --
 tests/test_proxy_connect.py  | 53 ++++++++++++++++++++----------------
 tests/test_utils_python.py   |  2 +-
 7 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4d7d9bac4..ab62c678c 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,6 +1,5 @@
 import logging
 import signal
-import sys
 import warnings
 
 from twisted.internet import reactor, defer
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f2999daea..e31ab1780 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
 
 import csv
 import io
-import sys
 import pprint
 import marshal
 import warnings
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index c39de0b52..0903fd4f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -165,7 +165,7 @@ class WrappedRequest(object):
 
     def get_header(self, name, default=None):
         return to_unicode(self.request.headers.get(name, default),
-                             errors='replace')
+                          errors='replace')
 
     def header_items(self):
         return [
diff --git a/scrapy/link.py b/scrapy/link.py
index be1888ef0..a809c5ca4 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+
+
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 70f11466b..0d23f51cc 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-import sys
-
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index bf56136b1..4147dc944 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdump
 sys.argv[0] = "mitmdump"
 sys.exit(mitmdump())
         """
-        cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-            'keys', 'mitmproxy-ca.pem')
-        self.proc = Popen([sys.executable,
-                           '-c', script,
-                           '--listen-host', '127.0.0.1',
-                           '--listen-port', '0',
-                           '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
-                           '--certs', cert_path,
-                           '--ssl-insecure',
-                           ],
-                           stdout=PIPE, env=get_testenv())
+        cert_path = os.path.join(
+            os.path.abspath(os.path.dirname(__file__)),
+            'keys',
+            'mitmproxy-ca.pem'
+        )
+        self.proc = Popen(
+            [
+                sys.executable,
+                '-c', script,
+                '--listen-host', '127.0.0.1',
+                '--listen-port', '0',
+                '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+                '--certs', cert_path,
+                '--ssl-insecure',
+            ],
+            stdout=PIPE,
+            env=get_testenv()
+        )
         line = self.proc.stdout.readline().decode('utf-8')
         host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
         address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
@@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase):
     @defer.inlineCallbacks
     def test_https_connect_tunnel(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, logs)
 
     @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
     @defer.inlineCallbacks
@@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase):
         proxy = os.environ['https_proxy']
         os.environ['https_proxy'] = proxy + '?noconnect'
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, logs)
 
     @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
     @defer.inlineCallbacks
     def test_https_connect_tunnel_error(self):
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(logs)
 
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         # The proxy returns a 407 error code but it does not reach the client;
         # he just sees a TunnelError.
-        self._assert_got_tunnel_error(l)
+        self._assert_got_tunnel_error(logs)
 
     @defer.inlineCallbacks
     def test_https_tunnel_without_leak_proxy_authorization_header(self):
         request = Request(self.mockserver.url("/echo", is_secure=True))
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, l)
+        self._assert_got_response_code(200, logs)
         echo = json.loads(crawler.spider.meta['responses'][0].text)
         self.assertTrue('Proxy-Authorization' not in echo['headers'])
 
@@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase):
     def test_https_noconnect_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as l:
+        with LogCapture() as logs:
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(407, l)
+        self._assert_got_response_code(407, logs)
 
     def _assert_got_response_code(self, code, log):
         print(log)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index faf0d4b73..b36c2a5e3 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -7,7 +7,7 @@ from itertools import count
 
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
-    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+    WeakKeyCache, get_func_args, to_bytes, to_unicode,
     without_none_values, MutableChain)
 
 

From 1718e450ef9549a4fc71b01dba1e6faf7a63238a Mon Sep 17 00:00:00 2001
From: Mabel Villalba <mabelvj@gmail.com>
Date: Mon, 18 Nov 2019 12:33:55 +0100
Subject: [PATCH 44/49] [start_url] Fixes #4133: Raise AttributeError error
 when empty 'start_urls' and 'start_url' found. Added test.

---
 scrapy/spiders/__init__.py | 5 +++++
 tests/test_spider.py       | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index e9c131e3b..5a35fcdb6 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -68,6 +68,11 @@ class Spider(object_ref):
 
     def start_requests(self):
         cls = self.__class__
+        if not self.start_urls and hasattr(self, 'start_url'):
+            raise AttributeError(
+                "Crawling could not start: 'start_urls' not found "
+                "or empty (but found 'start_url' attribute instead, "
+                "did you miss an 's'?)")
         if method_is_overridden(cls, Spider, 'make_requests_from_url'):
             warnings.warn(
                 "Spider.make_requests_from_url method is deprecated; it "
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 83fb68c2f..0a6640cec 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -391,6 +391,13 @@ class CrawlSpiderTest(SpiderTest):
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertFalse(spider._follow_links)
 
+    def test_start_url(self):
+        spider = self.spider_class("example.com")
+        spider.start_url = 'https://www.example.com'
+
+        with self.assertRaisesRegex(AttributeError,
+                                    r'^Crawling could not start.*$'):
+            list(spider.start_requests())
 
 class SitemapSpiderTest(SpiderTest):
 

From 55cc5c9068a7a39908f90e6403e6bd8c8ef11a4d Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Nov 2019 12:41:31 -0300
Subject: [PATCH 45/49] Skip pickle in bandit check

---
 .bandit.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.bandit.yml b/.bandit.yml
index 00554587a..cc7db3a66 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,6 +1,7 @@
 skips:
 - B101
 - B105
+- B301
 - B303
 - B306
 - B307
@@ -8,6 +9,7 @@ skips:
 - B320
 - B321
 - B402
+- B403
 - B404
 - B406
 - B410

From 9b4b43f8ac3a39d5dd6137223699d3d9e7dacec2 Mon Sep 17 00:00:00 2001
From: Grammy Jiang <gjiang@buildingsalive.com>
Date: Thu, 5 Dec 2019 11:25:19 +1100
Subject: [PATCH 46/49] Convert the relative imports to absolute imports

This commits converts the relative imports to absolute imports in the
entire package
---
 scrapy/__init__.py                      | 2 +-
 scrapy/contracts/default.py             | 2 +-
 scrapy/core/downloader/__init__.py      | 4 ++--
 scrapy/core/downloader/handlers/http.py | 6 ++++--
 scrapy/core/downloader/handlers/s3.py   | 2 +-
 scrapy/linkextractors/__init__.py       | 2 +-
 scrapy/linkextractors/regex.py          | 2 +-
 7 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 230e5cee3..fb8357f3c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
 # Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
 del _monkeypatches
 
 from twisted import version as _txv
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 24f6c2e77..e0d425874 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -4,7 +4,7 @@ from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
 
-from . import Contract
+from scrapy.contracts import Contract
 
 
 # contracts
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 213268741..c5474a57f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -10,8 +10,8 @@ from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.resolver import dnscache
 from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
 
 
 class Slot(object):
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 6111e132a..52535bd8b 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,2 +1,4 @@
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+    HTTP11DownloadHandler as HTTPDownloadHandler,
+)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 808d1bf21..f4a42ce12 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -3,7 +3,7 @@ from six.moves.urllib.parse import unquote
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 
 
 def _get_boto_connection():
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 8c3693f04..8411c4d59 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):
 
 
 # Top-level imports
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index e689b4727..49aa2be46 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -4,7 +4,7 @@ from six.moves.urllib.parse import urljoin
 from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
-from .sgml import SgmlLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor
 
 linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",

From af624ef414ab10d833925b4d6f9048468be01273 Mon Sep 17 00:00:00 2001
From: Wang Qin <37098874+dqwerter@users.noreply.github.com>
Date: Thu, 5 Dec 2019 09:29:12 +0800
Subject: [PATCH 47/49] Update overview.rst | Fix an inconsistency

There exists an inconsistency between the code (line 37 - 38) and the output 'quotes.json' (line 56 - 68).

Note that even though according to line 53 - 54  'quotes.json' is "reformatted here for better readability", it cannot explain why the "author" field precedes the "text" field.

Intended output for the code BEFORE change:
    [{
        "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d",
        "author": "Jane Austen"
    },
    {
        "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d",
        "author": "Groucho Marx"
    },
    {
        "text": "\u201cA day without sunshine is like, you know, night.\u201d",
        "author": "Steve Martin"
    },
    ...]

Intended output for the code After change (the inconsistency is fixed):
    [{
        "author": "Jane Austen",
        "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
    },
    {
        "author": "Groucho Marx",
        "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
    },
    {
        "author": "Steve Martin",
        "text": "\u201cA day without sunshine is like, you know, night.\u201d"
    },
    ...]
---
 docs/intro/overview.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 8b2fef065..01986b594 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').get(),
                     'author': quote.xpath('span/small/text()').get(),
+                    'text': quote.css('span.text::text').get(),
                 }
 
             next_page = response.css('li.next a::attr("href")').get()

From 07b8cd28aa84fb322a072467b49e2557d4aa3881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 5 Dec 2019 14:48:31 +0100
Subject: [PATCH 48/49] =?UTF-8?q?Mark=20bandit=E2=80=99s=20402=20check=20a?=
 =?UTF-8?q?s=20addressed=20by=20#4180=20(#4181)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .bandit.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.bandit.yml b/.bandit.yml
index cc7db3a66..243379b0b 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -8,7 +8,7 @@ skips:
 - B311
 - B320
 - B321
-- B402
+- B402  # https://github.com/scrapy/scrapy/issues/4180
 - B403
 - B404
 - B406

From 02cdc53fb82e3cc5e51771180f1f79186b52670a Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Fri, 13 Dec 2019 18:04:05 +0500
Subject: [PATCH 49/49] Add a test for a CrawlerProcess script. (#4218)

* Add a test for a CrawlerProcess script.

* Add tests/CrawlerProcess to collect_ignore.

* Remove an extra line.

* Fix/improve conftest.py.
---
 conftest.py                    |  9 ++++++++-
 tests/CrawlerProcess/simple.py | 15 +++++++++++++++
 tests/test_crawler.py          | 20 ++++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 tests/CrawlerProcess/simple.py

diff --git a/conftest.py b/conftest.py
index d54ce155c..d37c22436 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,12 +1,19 @@
+from pathlib import Path
+
 import pytest
 
 
+def _py_files(folder):
+    return (str(p) for p in Path(folder).rglob('*.py'))
+
+
 collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
+    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+    *_py_files("tests/CrawlerProcess")
 ]
 
-
 for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
new file mode 100644
index 000000000..5f6f1ae30
--- /dev/null
+++ b/tests/CrawlerProcess/simple.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8eb2389e2..e37a2ff0e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,7 @@
 import logging
+import os
+import subprocess
+import sys
 import warnings
 
 from twisted.internet import defer
@@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         yield runner.crawl(NoRequestsSpider)
 
         self.assertEqual(runner.bootstrap_failed, True)
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+    def run_script(self, script_name):
+        script_path = os.path.join(self.script_dir, script_name)
+        args = (sys.executable, script_path)
+        p = subprocess.Popen(args, env=get_testenv(),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        return stderr.decode('utf-8')
+
+    def test_simple(self):
+        log = self.run_script('simple.py')
+        self.assertIn('Spider closed (finished)', log)