Remove six.PY2 and six.PY3 conditionals.

docs/topics/downloader-middleware.rst

@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the anydbm_ module, but you can change it with the
+    By default, it uses the dbm_ module, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
 
 .. versionadded:: 0.13
 
-Default: ``'anydbm'``
+Default: ``'dbm'``
 
 The database module to use in the :ref:`DBM storage backend
 <httpcache-storage-dbm>`. This setting is specific to the DBM backend.
@@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html
+.. _dbm: https://docs.python.org/3/library/dbm.html

scrapy/_monkeypatches.py

@@ -1,16 +1,6 @@
-import six
 from six.moves import copyreg
 
 
-if six.PY2:
-    from urlparse import urlparse
-    
-    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
-    if urlparse('s3://bucket/key?key=value').query != 'key=value':
-        from urlparse import uses_query
-        uses_query.append('s3')
-
-
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
 import twisted.persisted.styles  # NOQA

scrapy/commands/fetch.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, six
+import sys
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
@@ -45,8 +45,7 @@ class Command(ScrapyCommand):
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
-        bytes_writer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):

scrapy/crawler.py

@@ -88,20 +88,9 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
-            # In Python 2 reraising an exception after yield discards
-            # the original traceback (see https://bugs.python.org/issue7563),
-            # so sys.exc_info() workaround is used.
-            # This workaround also works in Python 3, but it is not needed,
-            # and it is slower, so in Python 3 we use native `raise`.
-            if six.PY2:
-                exc_info = sys.exc_info()
-
             self.crawling = False
             if self.engine is not None:
                 yield self.engine.close()
-
-            if six.PY2:
-                six.reraise(*exc_info)
             raise
 
     def _create_spider(self, *args, **kwargs):

scrapy/exporters.py

@@ -216,7 +216,7 @@ class CsvItemExporter(BaseItemExporter):
             write_through=True,
             encoding=self.encoding,
             newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
-        ) if six.PY3 else file
+        )
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued

scrapy/extensions/feedexport.py

@@ -10,7 +10,6 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-import six
 from six.moves.urllib.parse import urlparse, unquote
 from ftplib import FTP
 
@@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
 
     def __init__(self, uri, _stdout=None):
         if not _stdout:
-            _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+            _stdout = sys.stdout.buffer
         self._stdout = _stdout
 
     def open(self, spider):

scrapy/http/request/form.py

@@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
-        raise ValueError('No <form> element found with %s' % encoded)
+        raise ValueError('No <form> element found with %s' % formxpath)
 
     # If we get here, it means that either formname was None
     # or invalid

scrapy/http/response/text.py

@@ -32,9 +32,6 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            if six.PY2 and self.encoding is None:
-                raise TypeError("Cannot convert unicode url - %s "
-                                "has no encoding" % type(self).__name__)
             self._url = to_native_str(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)

scrapy/item.py

@@ -4,8 +4,8 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-import collections
 from abc import ABCMeta
+from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
 from warnings import warn
@@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 class BaseItem(object_ref):
     """Base class for all scraped items.
 

scrapy/link.py

@@ -4,12 +4,6 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
-import warnings
-import six
-
-from scrapy.utils.python import to_bytes
-
-
 class Link(object):
     """Link objects represent an extracted link by the LinkExtractor."""
 
@@ -17,13 +11,8 @@ class Link(object):
 
     def __init__(self, url, text='', fragment='', nofollow=False):
         if not isinstance(url, str):
-            if six.PY2:
-                warnings.warn("Link urls must be str objects. "
-                              "Assuming utf-8 encoding (which could be wrong)")
-                url = to_bytes(url, encoding='utf8')
-            else:
-                got = url.__class__.__name__
-                raise TypeError("Link urls must be str objects, got %s" % got)
+            got = url.__class__.__name__
+            raise TypeError("Link urls must be str objects, got %s" % got)
         self.url = url
         self.text = text
         self.fragment = fragment

scrapy/mail.py

@@ -9,18 +9,13 @@ try:
     from cStringIO import StringIO as BytesIO
 except ImportError:
     from io import BytesIO
-import six
 
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
-if six.PY2:
-    from email.MIMENonMultipart import MIMENonMultipart
-    from email import Encoders
-else:
-    from email.mime.nonmultipart import MIMENonMultipart
-    from email import encoders as Encoders
+from email.mime.nonmultipart import MIMENonMultipart
+from email import encoders as Encoders
 
 from twisted.internet import defer, reactor, ssl
 

scrapy/settings/__init__.py

@@ -1,19 +1,13 @@
 import six
 import json
 import copy
-import collections
+from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
 from scrapy.settings import default_settings
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,

scrapy/settings/default_settings.py

@@ -17,8 +17,6 @@ import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
 
-import six
-
 AJAXCRAWL_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
@@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
+HTTPCACHE_DBM_MODULE = 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 

scrapy/utils/boto.py

@@ -1,7 +1,6 @@
 """Boto/botocore helpers"""
 
 from __future__ import absolute_import
-import six
 
 from scrapy.exceptions import NotConfigured
 
@@ -11,11 +10,4 @@ def is_botocore():
         import botocore
         return True
     except ImportError:
-        if six.PY2:
-            try:
-                import boto
-                return False
-            except ImportError:
-                raise NotConfigured('missing botocore or boto library')
-        else:
-            raise NotConfigured('missing botocore library')
+        raise NotConfigured('missing botocore library')

scrapy/utils/conf.py

@@ -1,13 +1,10 @@
+from configparser import ConfigParser
 import os
 import sys
 import numbers
 from operator import itemgetter
 
 import six
-if six.PY2:
-    from ConfigParser import SafeConfigParser as ConfigParser
-else:
-    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath

scrapy/utils/datatypes.py

@@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
+from collections.abc import Mapping
 import warnings
 
 import six
@@ -14,12 +15,6 @@ import six
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-if six.PY2:
-    Mapping = collections.Mapping
-else:
-    Mapping = collections.abc.Mapping
-
-
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -252,13 +247,12 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
-        if not six.PY2:
-            warnings.warn(
-                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-                "of collections.ChainMap (introduced in Python 3.3)",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+            "of collections.ChainMap (introduced in Python 3.3)",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.dicts = dicts
 
     def __getitem__(self, key):

scrapy/utils/gz.py

@@ -6,7 +6,6 @@ except ImportError:
     from io import BytesIO
 from gzip import GzipFile
 
-import six
 import re
 
 from scrapy.utils.decorators import deprecated
@@ -17,14 +16,8 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
-# - scrapy does not support Python 3.2
-# - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY2:
-    def read1(gzf, size=-1):
-        return gzf.read(size)
-else:
-    def read1(gzf, size=-1):
-        return gzf.read1(size)
+def read1(gzf, size=-1):
+    return gzf.read1(size)
 
 
 def gunzip(data):
@@ -37,7 +30,7 @@ def gunzip(data):
     chunk = b'.'
     while chunk:
         try:
-            chunk = read1(f, 8196)
+            chunk = f.read1(8196)
             output_list.append(chunk)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise

scrapy/utils/iterators.py

@@ -102,11 +102,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def row_to_unicode(row_):
         return [to_unicode(field, encoding) for field in row_]
 
-    # Python 3 csv reader input object needs to return strings
-    if six.PY3:
-        lines = StringIO(_body_or_str(obj, unicode=True))
-    else:
-        lines = BytesIO(_body_or_str(obj, unicode=False))
+    lines = StringIO(_body_or_str(obj, unicode=True))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter

scrapy/utils/python.py

@@ -113,10 +113,7 @@ def to_bytes(text, encoding=None, errors='strict'):
 def to_native_str(text, encoding=None, errors='strict'):
     """ Return str representation of ``text``
     (bytes in Python 2.x and unicode in Python 3.x). """
-    if six.PY2:
-        return to_bytes(text, encoding, errors)
-    else:
-        return to_unicode(text, encoding, errors)
+    return to_unicode(text, encoding, errors)
 
 
 def re_rsearch(pattern, text, chunk_size=1024):
@@ -189,7 +186,7 @@ def _getargspec_py23(func):
     """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
                                                         defaults)
 
-    Identical to inspect.getargspec() in python2, but uses
+    Was identical to inspect.getargspec() in python2, but uses
     inspect.getfullargspec() for python3 behind the scenes to avoid
     DeprecationWarning.
 
@@ -199,9 +196,6 @@ def _getargspec_py23(func):
     >>> _getargspec_py23(f)
     ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
     """
-    if six.PY2:
-        return inspect.getargspec(func)
-
     return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
 
 

tests/__init__.py

@@ -35,18 +35,3 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
-
-
-# FIXME: delete after dropping py2 support
-# Monkey patch the unittest module to prevent the
-# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
-import six
-if six.PY2:
-    import unittest
-    import twisted.trial.unittest
-    if not getattr(unittest.TestCase, 'assertRegex', None):
-        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
-    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
-        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
-    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
-        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp

tests/test_http_request.py

@@ -3,13 +3,12 @@ import cgi
 import unittest
 import re
 import json
+from urllib.parse import unquote_to_bytes
 import warnings
 
 import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
-if six.PY3:
-    from urllib.parse import unquote_to_bytes
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
 from scrapy.utils.python import to_bytes, to_native_str
@@ -1064,8 +1063,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs, {})
 
         xpath = u"//form[@name='\u03b1']"
-        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegex(ValueError, re.escape(encoded),
+        self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
 
@@ -1208,10 +1206,7 @@ def _qs(req, encoding='utf-8', to_unicode=False):
         qs = req.body
     else:
         qs = req.url.partition('?')[2]
-    if six.PY2:
-        uqs = unquote(to_native_str(qs, encoding))
-    elif six.PY3:
-        uqs = unquote_to_bytes(qs)
+    uqs = unquote_to_bytes(qs)
     if to_unicode:
         uqs = uqs.decode(encoding)
     return parse_qs(uqs, True)

tests/test_http_response.py

@@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the consturctor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
-        if not six.PY2:
-            self.assertRaises(TypeError, self.response_class, b"http://example.com")
+        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))

tests/test_item.py

@@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase):
         i['number'] = 123
         itemrepr = repr(i)
 
-        if six.PY2:
-            self.assertEqual(itemrepr,
-                             "{'name': u'John Doe', 'number': 123}")
-        else:
-            self.assertEqual(itemrepr,
-                             "{'name': 'John Doe', 'number': 123}")
+        self.assertEqual(itemrepr,
+                         "{'name': 'John Doe', 'number': 123}")
 
         i2 = eval(itemrepr)
         self.assertEqual(i2['name'], 'John Doe')

tests/test_link.py

@@ -1,6 +1,4 @@
 import unittest
-import warnings
-import six
 
 from scrapy.link import Link
 
@@ -46,12 +44,5 @@ class LinkTest(unittest.TestCase):
         self._assert_same_links(l1, l2)
 
     def test_non_str_url_py2(self):
-        if six.PY2:
-            with warnings.catch_warnings(record=True) as w:
-                link = Link(u"http://www.example.com/\xa3")
-                self.assertIsInstance(link.url, str)
-                self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            assert len(w) == 1, "warning not issued"
-        else:
-            with self.assertRaises(TypeError):
-                Link(b"http://www.example.com/\xc2\xa3")
+        with self.assertRaises(TypeError):
+            Link(b"http://www.example.com/\xc2\xa3")

tests/test_middleware.py

@@ -3,7 +3,6 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
-import six
 
 class M1(object):
 
@@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        if six.PY2:
-            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.im_class for x in mwman.methods['process']],
-                [M1, M3])
-        else:
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
-                [M1, M3])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+            [M1, M2])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+            [M2, M1])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+            [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()

tests/test_request_cb_kwargs.py

@@ -1,7 +1,6 @@
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
         self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")

tests/test_settings/__init__.py

@@ -60,9 +60,6 @@ class SettingsAttributeTest(unittest.TestCase):
 
 class BaseSettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = BaseSettings()
 
@@ -152,7 +149,7 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+        self.assertCountEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))
 
         for key in six.iterkeys(ctrl_attributes):
@@ -343,9 +340,6 @@ class BaseSettingsTest(unittest.TestCase):
 
 class SettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = Settings()
 

tests/test_utils_datatypes.py

@@ -1,11 +1,6 @@
 import copy
 import unittest
-
-import six
-if six.PY2:
-    from collections import Mapping, MutableMapping
-else:
-    from collections.abc import Mapping, MutableMapping
+from collections.abc import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
 

tests/test_utils_python.py

@@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
-            stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
-                get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+                get_func_args(six.text_type.split, True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, True), ['list'])
             self.assertEqual(
-                get_func_args(operator.itemgetter(2), stripself), ['obj'])
+                get_func_args(operator.itemgetter(2), True), ['obj'])
 
 
     def test_without_none_values(self):