Merge pull request #4115 from scrapy/remove-py2-code

Remove py2 code

docs/topics/downloader-middleware.rst

@@ -474,7 +474,7 @@ DBM storage backend
 
     A DBM_ storage backend is also available for the HTTP cache middleware.
 
-    By default, it uses the anydbm_ module, but you can change it with the
+    By default, it uses the :mod:`dbm`, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
 .. _httpcache-storage-custom:
@@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
 
 .. versionadded:: 0.13
 
-Default: ``'anydbm'``
+Default: ``'dbm'``
 
 The database module to use in the :ref:`DBM storage backend
 <httpcache-storage-dbm>`. This setting is specific to the DBM backend.
@@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html

scrapy/_monkeypatches.py

@@ -1,16 +1,6 @@
-import six
 from six.moves import copyreg
 
 
-if six.PY2:
-    from urlparse import urlparse
-    
-    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
-    if urlparse('s3://bucket/key?key=value').query != 'key=value':
-        from urlparse import uses_query
-        uses_query.append('s3')
-
-
 # Undo what Twisted's perspective broker adds to pickle register
 # to prevent bugs like Twisted#7989 while serializing requests
 import twisted.persisted.styles  # NOQA

scrapy/commands/fetch.py

@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, six
+import sys
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
@@ -45,8 +45,7 @@ class Command(ScrapyCommand):
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
-        bytes_writer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):

scrapy/core/downloader/handlers/http11.py

@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
     r"""
     Return binary content of a CONNECT request.
 
-    >>> from scrapy.utils.python import to_native_str as s
+    >>> from scrapy.utils.python import to_unicode as s
     >>> s(tunnel_request_data("example.com", 8080))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))

scrapy/crawler.py

@@ -3,7 +3,6 @@ import signal
 import logging
 import warnings
 
-import sys
 from twisted.internet import reactor, defer
 from zope.interface.verify import verifyClass, DoesNotImplement
 
@@ -88,20 +87,9 @@ class Crawler(object):
             yield self.engine.open_spider(self.spider, start_requests)
             yield defer.maybeDeferred(self.engine.start)
         except Exception:
-            # In Python 2 reraising an exception after yield discards
-            # the original traceback (see https://bugs.python.org/issue7563),
-            # so sys.exc_info() workaround is used.
-            # This workaround also works in Python 3, but it is not needed,
-            # and it is slower, so in Python 3 we use native `raise`.
-            if six.PY2:
-                exc_info = sys.exc_info()
-
             self.crawling = False
             if self.engine is not None:
                 yield self.engine.close()
-
-            if six.PY2:
-                six.reraise(*exc_info)
             raise
 
     def _create_spider(self, *args, **kwargs):

scrapy/downloadermiddlewares/cookies.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in request.headers.getlist('Cookie')]
             if cl:
                 cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in response.headers.getlist('Set-Cookie')]
             if cl:
                 cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)

scrapy/downloadermiddlewares/decompression.py

@@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
+from io import BytesIO
 import zipfile
 import tarfile
 import logging
@@ -11,11 +12,6 @@ from tempfile import mktemp
 
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from scrapy.responsetypes import responsetypes
 
 logger = logging.getLogger(__name__)

scrapy/downloadermiddlewares/httpproxy.py

@@ -1,10 +1,7 @@
 import base64
 from six.moves.urllib.parse import unquote, urlunparse
 from six.moves.urllib.request import getproxies, proxy_bypass
-try:
-    from urllib2 import _parse_proxy
-except ImportError:
-    from urllib.request import _parse_proxy
+from urllib.request import _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/downloadermiddlewares/robotstxt.py

@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-import sys
-import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
 from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)

scrapy/exporters.py

@@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats.
 
 import csv
 import io
-import sys
 import pprint
 import marshal
 import six
@@ -12,7 +11,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 import warnings
@@ -143,11 +142,11 @@ class XmlItemExporter(BaseItemExporter):
 
     def _beautify_newline(self, new_item=False):
         if self.indent is not None and (self.indent > 0 or new_item):
-            self._xg_characters('\n')
+            self.xg.characters('\n')
 
     def _beautify_indent(self, depth=1):
         if self.indent:
-            self._xg_characters(' ' * self.indent * depth)
+            self.xg.characters(' ' * self.indent * depth)
 
     def start_exporting(self):
         self.xg.startDocument()
@@ -182,26 +181,12 @@ class XmlItemExporter(BaseItemExporter):
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
         elif isinstance(serialized_value, six.text_type):
-            self._xg_characters(serialized_value)
+            self.xg.characters(serialized_value)
         else:
-            self._xg_characters(str(serialized_value))
+            self.xg.characters(str(serialized_value))
         self.xg.endElement(name)
         self._beautify_newline()
 
-    # Workaround for https://bugs.python.org/issue17606
-    # Before Python 2.7.4 xml.sax.saxutils required bytes;
-    # since 2.7.4 it requires unicode. The bug is likely to be
-    # fixed in 2.7.6, but 2.7.6 will still support unicode,
-    # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
-    if sys.version_info[:3] >= (2, 7, 4):
-        def _xg_characters(self, serialized_value):
-            if not isinstance(serialized_value, six.text_type):
-                serialized_value = serialized_value.decode(self.encoding)
-            return self.xg.characters(serialized_value)
-    else:  # pragma: no cover
-        def _xg_characters(self, serialized_value):
-            return self.xg.characters(serialized_value)
-
 
 class CsvItemExporter(BaseItemExporter):
 
@@ -216,7 +201,7 @@ class CsvItemExporter(BaseItemExporter):
             write_through=True,
             encoding=self.encoding,
             newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
-        ) if six.PY3 else file
+        )
         self.csv_writer = csv.writer(self.stream, **kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
@@ -246,7 +231,7 @@ class CsvItemExporter(BaseItemExporter):
     def _build_row(self, values):
         for s in values:
             try:
-                yield to_native_str(s, self.encoding)
+                yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 

scrapy/extensions/feedexport.py

@@ -10,7 +10,6 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-import six
 from six.moves.urllib.parse import urlparse, unquote
 from ftplib import FTP
 
@@ -65,7 +64,7 @@ class StdoutFeedStorage(object):
 
     def __init__(self, uri, _stdout=None):
         if not _stdout:
-            _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+            _stdout = sys.stdout.buffer
         self._stdout = _stdout
 
     def open(self, spider):

scrapy/http/cookies.py

@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
     CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
 )
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default),
-                             errors='replace')
+        return to_unicode(self.request.headers.get(name, default),
+                          errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k, errors='replace'),
-             [to_native_str(x, errors='replace') for x in v])
+            (to_unicode(k, errors='replace'),
+             [to_unicode(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -189,7 +189,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v, errors='replace')
+        return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all

scrapy/http/request/form.py

@@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
                 el = el.getparent()
                 if el is None:
                     break
-        encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
-        raise ValueError('No <form> element found with %s' % encoded)
+        raise ValueError('No <form> element found with %s' % formxpath)
 
     # If we get here, it means that either formname was None
     # or invalid

scrapy/http/response/__init__.py

@@ -88,7 +88,7 @@ class Response(object_ref):
     @property
     def text(self):
         """For subclasses of TextResponse, this will return the body
-        as text (unicode object in Python 2 and str in Python 3)
+        as str
         """
         raise AttributeError("Response content isn't text")
 

scrapy/http/response/text.py

@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace
 from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
 
 
 class TextResponse(Response):
@@ -32,10 +32,7 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            if six.PY2 and self.encoding is None:
-                raise TypeError("Cannot convert unicode url - %s "
-                                "has no encoding" % type(self).__name__)
-            self._url = to_native_str(url, self.encoding)
+            self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
@@ -84,11 +81,11 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b'Content-Type', b'')
-        return http_content_type_encoding(to_native_str(content_type))
+        return http_content_type_encoding(to_unicode(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+            content_type = to_unicode(self.headers.get(b'Content-Type', b''))
             benc, ubody = html_to_unicode(content_type, self.body,
                     auto_detect_fun=self._auto_detect_fun,
                     default_encoding=self._DEFAULT_ENCODING)

scrapy/item.py

@@ -4,8 +4,8 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-import collections
 from abc import ABCMeta
+from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
 from warnings import warn
@@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 class BaseItem(object_ref):
     """Base class for all scraped items.
 

scrapy/link.py

@@ -4,10 +4,6 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
-import warnings
-import six
-
-from scrapy.utils.python import to_bytes
 
 
 class Link(object):
@@ -17,13 +13,8 @@ class Link(object):
 
     def __init__(self, url, text='', fragment='', nofollow=False):
         if not isinstance(url, str):
-            if six.PY2:
-                warnings.warn("Link urls must be str objects. "
-                              "Assuming utf-8 encoding (which could be wrong)")
-                url = to_bytes(url, encoding='utf8')
-            else:
-                got = url.__class__.__name__
-                raise TypeError("Link urls must be str objects, got %s" % got)
+            got = url.__class__.__name__
+            raise TypeError("Link urls must be str objects, got %s" % got)
         self.url = url
         self.text = text
         self.fragment = fragment

scrapy/linkextractors/lxmlhtml.py

@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_native_str(url, encoding=response_encoding)
+            url = to_unicode(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',

scrapy/loader/processors.py

@@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
-try:
-    from collections import ChainMap
-except ImportError:
-    from scrapy.utils.datatypes import MergeDict as ChainMap
+from collections import ChainMap
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.loader.common import wrap_loader_context

scrapy/mail.py

@@ -3,24 +3,15 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+from io import BytesIO
 import logging
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-import six
-
 from email.utils import COMMASPACE, formatdate
 from six.moves.email_mime_multipart import MIMEMultipart
 from six.moves.email_mime_text import MIMEText
 from six.moves.email_mime_base import MIMEBase
-if six.PY2:
-    from email.MIMENonMultipart import MIMENonMultipart
-    from email import Encoders
-else:
-    from email.mime.nonmultipart import MIMENonMultipart
-    from email import encoders as Encoders
+from email.mime.nonmultipart import MIMENonMultipart
+from email import encoders as Encoders
 
 from twisted.internet import defer, reactor, ssl
 

scrapy/pipelines/files.py

@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import mimetypes
 import os
 import os.path
@@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline

scrapy/pipelines/images.py

@@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
+from io import BytesIO
 import six
 
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
-
 from PIL import Image
 
 from scrapy.utils.misc import md5sum

scrapy/responsetypes.py

@@ -10,7 +10,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
 class ResponseTypes(object):
@@ -55,12 +55,12 @@ class ResponseTypes(object):
         header """
         if content_encoding:
             return Response
-        mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+        mimetype = to_unicode(content_type).split(';')[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition,
+            filename = to_unicode(content_disposition,
                 encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)

scrapy/robotstxt.py

@@ -3,14 +3,14 @@ import logging
 from abc import ABCMeta, abstractmethod
 from six import with_metaclass
 
-from scrapy.utils.python import to_native_str, to_unicode
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
-            robotstxt_body = to_native_str(robotstxt_body)
+            robotstxt_body = to_unicode(robotstxt_body)
         else:
             robotstxt_body = robotstxt_body.decode('utf-8')
     except UnicodeDecodeError:
@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
         return o
 
     def allowed(self, url, user_agent):
-        user_agent = to_native_str(user_agent)
-        url = to_native_str(url)
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
 

scrapy/settings/__init__.py

@@ -1,19 +1,13 @@
 import six
 import json
 import copy
-import collections
+from collections.abc import MutableMapping
 from importlib import import_module
 from pprint import pformat
 
 from scrapy.settings import default_settings
 
 
-if six.PY2:
-    MutableMapping = collections.MutableMapping
-else:
-    MutableMapping = collections.abc.MutableMapping
-
-
 SETTINGS_PRIORITIES = {
     'default': 0,
     'command': 10,

scrapy/settings/default_settings.py

@@ -17,8 +17,6 @@ import sys
 from importlib import import_module
 from os.path import join, abspath, dirname
 
-import six
-
 AJAXCRAWL_ENABLED = False
 
 AUTOTHROTTLE_ENABLED = False
@@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
+HTTPCACHE_DBM_MODULE = 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 

scrapy/spidermiddlewares/referer.py

@@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
 
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
             if isinstance(resp_or_url, Response):
                 policy_header = resp_or_url.headers.get('Referrer-Policy')
                 if policy_header is not None:
-                    policy_name = to_native_str(policy_header.decode('latin1'))
+                    policy_name = to_unicode(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 

scrapy/utils/boto.py

@@ -1,7 +1,6 @@
 """Boto/botocore helpers"""
 
 from __future__ import absolute_import
-import six
 
 from scrapy.exceptions import NotConfigured
 
@@ -11,11 +10,4 @@ def is_botocore():
         import botocore
         return True
     except ImportError:
-        if six.PY2:
-            try:
-                import boto
-                return False
-            except ImportError:
-                raise NotConfigured('missing botocore or boto library')
-        else:
-            raise NotConfigured('missing botocore library')
+        raise NotConfigured('missing botocore library')

scrapy/utils/conf.py

@@ -1,13 +1,10 @@
+from configparser import ConfigParser
 import os
 import sys
 import numbers
 from operator import itemgetter
 
 import six
-if six.PY2:
-    from ConfigParser import SafeConfigParser as ConfigParser
-else:
-    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath

scrapy/utils/datatypes.py

@@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
+from collections.abc import Mapping
 import warnings
 
 import six
@@ -14,12 +15,6 @@ import six
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
-if six.PY2:
-    Mapping = collections.Mapping
-else:
-    Mapping = collections.abc.Mapping
-
-
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -252,13 +247,12 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
-        if not six.PY2:
-            warnings.warn(
-                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-                "of collections.ChainMap (introduced in Python 3.3)",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+            "of collections.ChainMap (introduced in Python 3.3)",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.dicts = dicts
 
     def __getitem__(self, key):

scrapy/utils/gz.py

@@ -1,13 +1,7 @@
-import struct
-
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from gzip import GzipFile
-
-import six
+from io import BytesIO
 import re
+import struct
 
 from scrapy.utils.decorators import deprecated
 
@@ -17,14 +11,9 @@ from scrapy.utils.decorators import deprecated
 #   (regression or bug-fix compared to Python 3.4)
 # - read1(), which fetches data before raising EOFError on next call
 #   works here but is only available from Python>=3.3
-# - scrapy does not support Python 3.2
-# - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY2:
-    def read1(gzf, size=-1):
-        return gzf.read(size)
-else:
-    def read1(gzf, size=-1):
-        return gzf.read1(size)
+@deprecated('GzipFile.read1')
+def read1(gzf, size=-1):
+    return gzf.read1(size)
 
 
 def gunzip(data):
@@ -37,7 +26,7 @@ def gunzip(data):
     chunk = b'.'
     while chunk:
         try:
-            chunk = read1(f, 8196)
+            chunk = f.read1(8196)
             output_list.append(chunk)
         except (IOError, EOFError, struct.error):
             # complete only if there is some data, otherwise re-raise

scrapy/utils/iterators.py

@@ -1,11 +1,7 @@
 import re
 import csv
-import logging
-try:
-    from cStringIO import StringIO as BytesIO
-except ImportError:
-    from io import BytesIO
 from io import StringIO
+import logging
 import six
 
 from scrapy.http import TextResponse, Response
@@ -102,11 +98,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
     def row_to_unicode(row_):
         return [to_unicode(field, encoding) for field in row_]
 
-    # Python 3 csv reader input object needs to return strings
-    if six.PY3:
-        lines = StringIO(_body_or_str(obj, unicode=True))
-    else:
-        lines = BytesIO(_body_or_str(obj, unicode=False))
+    lines = StringIO(_body_or_str(obj, unicode=True))
 
     kwargs = {}
     if delimiter: kwargs["delimiter"] = delimiter

scrapy/utils/python.py

@@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'):
     if isinstance(text, six.text_type):
         return text
     if not isinstance(text, (bytes, six.text_type)):
-        raise TypeError('to_unicode must receive a bytes, str or unicode '
+        raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
@@ -103,20 +103,17 @@ def to_bytes(text, encoding=None, errors='strict'):
     if isinstance(text, bytes):
         return text
     if not isinstance(text, six.string_types):
-        raise TypeError('to_bytes must receive a unicode, str or bytes '
+        raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
     return text.encode(encoding, errors)
 
 
+@deprecated('to_unicode')
 def to_native_str(text, encoding=None, errors='strict'):
-    """ Return str representation of ``text``
-    (bytes in Python 2.x and unicode in Python 3.x). """
-    if six.PY2:
-        return to_bytes(text, encoding, errors)
-    else:
-        return to_unicode(text, encoding, errors)
+    """ Return str representation of ``text``. """
+    return to_unicode(text, encoding, errors)
 
 
 def re_rsearch(pattern, text, chunk_size=1024):
@@ -189,7 +186,7 @@ def _getargspec_py23(func):
     """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
                                                         defaults)
 
-    Identical to inspect.getargspec() in python2, but uses
+    Was identical to inspect.getargspec() in python2, but uses
     inspect.getfullargspec() for python3 behind the scenes to avoid
     DeprecationWarning.
 
@@ -199,9 +196,6 @@ def _getargspec_py23(func):
     >>> _getargspec_py23(f)
     ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
     """
-    if six.PY2:
-        return inspect.getargspec(func)
-
     return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
 
 

scrapy/utils/reqser.py

@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
 import six
 
 from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 
 
@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
         eb = _get_method(spider, eb)
     request_cls = load_object(d['_class']) if '_class' in d else Request
     return request_cls(
-        url=to_native_str(d['url']),
+        url=to_unicode(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],

scrapy/utils/request.py

@@ -9,7 +9,7 @@ import weakref
 from six.moves.urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 from w3lib.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
@@ -97,4 +97,4 @@ def referer_str(request):
     referrer = request.headers.get('Referer')
     if referrer is None:
         return referrer
-    return to_native_str(referrer, errors='replace')
+    return to_unicode(referrer, errors='replace')

scrapy/utils/response.py

@@ -8,7 +8,7 @@ import webbrowser
 import tempfile
 
 from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
@@ -36,7 +36,7 @@ def response_status_message(status):
     """Return status code plus status text descriptive message
     """
     message = http.RESPONSES.get(int(status), "Unknown Status")
-    return '%s %s' % (status, to_native_str(message))
+    return '%s %s' % (status, to_unicode(message))
 
 
 def response_httprepr(response):

scrapy/utils/ssl.py

@@ -3,7 +3,7 @@
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
 
 
 def ffi_buf_to_string(buf):
-    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+    return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
 def x509name_to_string(x509name):

tests/__init__.py

@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
     os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
                                                  os.environ['COV_CORE_CONFIG'])
 
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                              'sample_data')
 
@@ -35,18 +30,3 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
-
-
-# FIXME: delete after dropping py2 support
-# Monkey patch the unittest module to prevent the
-# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
-import six
-if six.PY2:
-    import unittest
-    import twisted.trial.unittest
-    if not getattr(unittest.TestCase, 'assertRegex', None):
-        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
-    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
-        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
-    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
-        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp

tests/test_cmdline/__init__.py

@@ -1,3 +1,4 @@
+from io import StringIO
 import json
 import os
 import pstats
@@ -7,10 +8,6 @@ from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from io import StringIO
 
 from scrapy.utils.test import get_testenv
 

tests/test_command_parse.py

@@ -1,17 +1,16 @@
 import os
 from os.path import join, abspath
-from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from tests.test_commands import CommandTest
 
 
 def _textmode(bstr):
     """Normalize input the same as writing to a file
     and reading from it in text mode"""
-    return to_native_str(bstr).replace(os.linesep, '\n')
+    return to_unicode(bstr).replace(os.linesep, '\n')
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'

tests/test_commands.py

@@ -10,13 +10,10 @@ from contextlib import contextmanager
 from threading import Timer
 
 from twisted.trial import unittest
-from twisted.internet import defer
 
 import scrapy
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
-from scrapy.utils.testsite import SiteTest
-from scrapy.utils.testproc import ProcessTest
 from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        return p, to_native_str(stdout), to_native_str(stderr)
+        return p, to_unicode(stdout), to_unicode(stderr)
 
 
 class StartprojectTest(ProjectTest):

tests/test_downloader_handlers.py

@@ -2,11 +2,8 @@ import os
 import six
 import shutil
 import tempfile
+from unittest import mock
 import contextlib
-try:
-    from unittest import mock
-except ImportError:
-    import mock
 
 from testfixtures import LogCapture
 from twisted.trial import unittest

tests/test_downloadermiddleware.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.utils.test import get_crawler
 from scrapy.utils.python import to_bytes
-from tests import mock
 
 
 class ManagerTestCase(TestCase):

tests/test_downloadermiddleware_robotstxt.py

@@ -1,5 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+
+from unittest import mock
+
 from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
@@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
-from tests import mock
 from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 

tests/test_extension_telnet.py

@@ -1,8 +1,3 @@
-try:
-    import unittest.mock as mock
-except ImportError:
-    import mock
-
 from twisted.trial import unittest
 from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials

tests/test_feedexport.py

@@ -7,6 +7,7 @@ from io import BytesIO
 import tempfile
 import shutil
 import string
+from unittest import mock
 from six.moves.urllib.parse import urljoin, urlparse, quote
 from six.moves.urllib.request import pathname2url
 
@@ -15,7 +16,6 @@ from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.crawler import CrawlerRunner
 from scrapy.settings import Settings
-from tests import mock
 from tests.mockserver import MockServer
 from w3lib.url import path_to_file_uri
 
@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
-from scrapy.utils.python import to_native_str
-from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_unicode
 
 from pathlib import Path
 
@@ -459,7 +458,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'csv'})
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(to_native_str(data).splitlines())
+        reader = csv.DictReader(to_unicode(data).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -473,7 +472,7 @@ class FeedExportTest(unittest.TestCase):
         settings = settings or {}
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 

tests/test_http_request.py

@@ -3,18 +3,15 @@ import cgi
 import unittest
 import re
 import json
+from unittest import mock
+from urllib.parse import unquote_to_bytes
 import warnings
 
-import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
-if six.PY3:
-    from urllib.parse import unquote_to_bytes
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
-from scrapy.utils.python import to_bytes, to_native_str
-
-from tests import mock
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
@@ -351,8 +348,8 @@ class FormRequestTest(RequestTest):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
-        first = to_native_str(first).split("&")
-        second = to_native_str(second).split("&")
+        first = to_unicode(first).split("&")
+        second = to_unicode(second).split("&")
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_empty_formdata(self):
@@ -1064,8 +1061,7 @@ class FormRequestTest(RequestTest):
         self.assertEqual(fs, {})
 
         xpath = u"//form[@name='\u03b1']"
-        encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegex(ValueError, re.escape(encoded),
+        self.assertRaisesRegex(ValueError, re.escape(xpath),
                                self.request_class.from_response,
                                response, formxpath=xpath)
 
@@ -1208,10 +1204,7 @@ def _qs(req, encoding='utf-8', to_unicode=False):
         qs = req.body
     else:
         qs = req.url.partition('?')[2]
-    if six.PY2:
-        uqs = unquote(to_native_str(qs, encoding))
-    elif six.PY3:
-        uqs = unquote_to_bytes(qs)
+    uqs = unquote_to_bytes(qs)
     if to_unicode:
         uqs = uqs.decode(encoding)
     return parse_qs(uqs, True)

tests/test_http_response.py

@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.exceptions import NotSupported
 from scrapy.link import Link
 from tests import get_testdata
@@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase):
         # Response requires url in the consturctor
         self.assertRaises(Exception, self.response_class)
         self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
-        if not six.PY2:
-            self.assertRaises(TypeError, self.response_class, b"http://example.com")
+        self.assertRaises(TypeError, self.response_class, b"http://example.com")
         # body can be str or None
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class))
         self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class))
@@ -205,11 +204,11 @@ class TextResponseTest(BaseResponseTest):
         assert isinstance(resp.url, str)
 
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 

tests/test_item.py

@@ -1,12 +1,12 @@
 import sys
 import unittest
+from unittest import mock
 from warnings import catch_warnings
 
 import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
-from tests import mock
 
 
 PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
@@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase):
         i['number'] = 123
         itemrepr = repr(i)
 
-        if six.PY2:
-            self.assertEqual(itemrepr,
-                             "{'name': u'John Doe', 'number': 123}")
-        else:
-            self.assertEqual(itemrepr,
-                             "{'name': 'John Doe', 'number': 123}")
+        self.assertEqual(itemrepr,
+                         "{'name': 'John Doe', 'number': 123}")
 
         i2 = eval(itemrepr)
         self.assertEqual(i2['name'], 'John Doe')

tests/test_link.py

@@ -1,6 +1,4 @@
 import unittest
-import warnings
-import six
 
 from scrapy.link import Link
 
@@ -45,13 +43,6 @@ class LinkTest(unittest.TestCase):
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
-    def test_non_str_url_py2(self):
-        if six.PY2:
-            with warnings.catch_warnings(record=True) as w:
-                link = Link(u"http://www.example.com/\xa3")
-                self.assertIsInstance(link.url, str)
-                self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3')
-            assert len(w) == 1, "warning not issued"
-        else:
-            with self.assertRaises(TypeError):
-                Link(b"http://www.example.com/\xc2\xa3")
+    def test_bytes_url(self):
+        with self.assertRaises(TypeError):
+            Link(b"http://www.example.com/\xc2\xa3")

tests/test_middleware.py

@@ -3,7 +3,6 @@ from twisted.trial import unittest
 from scrapy.settings import Settings
 from scrapy.exceptions import NotConfigured
 from scrapy.middleware import MiddlewareManager
-import six
 
 class M1(object):
 
@@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase):
 
     def test_methods(self):
         mwman = TestMiddlewareManager(M1(), M2(), M3())
-        if six.PY2:
-            self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.im_class for x in mwman.methods['process']],
-                [M1, M3])
-        else:
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
-                [M1, M2])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
-                [M2, M1])
-            self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
-                [M1, M3])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+            [M1, M2])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+            [M2, M1])
+        self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+            [M1, M3])
 
     def test_enabled(self):
         m1, m2, m3 = M1(), M2(), M3()

tests/test_pipeline_files.py

@@ -1,10 +1,9 @@
 import os
 import random
 import time
-import hashlib
-import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from unittest import mock
 from six.moves.urllib.parse import urlparse
 from six import BytesIO
 
@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
 from scrapy.utils.boto import is_botocore
 
-from tests import mock
-
 
 def _mocked_download_func(request, info):
     response = request.meta.get('response')

tests/test_pipeline_media.py

@@ -1,7 +1,5 @@
 from __future__ import print_function
 
-import sys
-
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -144,10 +142,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
 
         # The Failure should encapsulate a FileException ...
         self.assertEqual(failure.value, file_exc)
-        # ... and if we're running on Python 3 ...
-        if sys.version_info.major >= 3:
-            # ... it should have the returnValue exception set as its context
-            self.assertEqual(failure.value.__context__, def_gen_return_exc)
+        # ... and it should have the returnValue exception set as its context
+        self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
         fp = request_fingerprint(request)

tests/test_request_cb_kwargs.py

@@ -1,7 +1,6 @@
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-import six
 
 from scrapy.http import Request
 from scrapy.crawler import CrawlerRunner
@@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
         self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
         self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
         self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
-        # py2 and py3 messages are different
-        exc_message = str(exceptions['takes_more'].exc_info[1])
-        if six.PY2:
-            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
-        elif six.PY3:
-            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
+        self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")

tests/test_robotstxt_interface.py

@@ -1,6 +1,5 @@
 # coding=utf-8
 from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
 
 
 def reppy_available():

tests/test_settings/__init__.py

@@ -1,10 +1,9 @@
 import six
 import unittest
-import warnings
+from unittest import mock
 
 from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
                              SETTINGS_PRIORITIES, get_settings_priority)
-from tests import mock
 from . import default_settings
 
 
@@ -60,9 +59,6 @@ class SettingsAttributeTest(unittest.TestCase):
 
 class BaseSettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = BaseSettings()
 
@@ -152,7 +148,7 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertItemsEqual(six.iterkeys(self.settings.attributes),
+        self.assertCountEqual(six.iterkeys(self.settings.attributes),
                               six.iterkeys(ctrl_attributes))
 
         for key in six.iterkeys(ctrl_attributes):
@@ -343,9 +339,6 @@ class BaseSettingsTest(unittest.TestCase):
 
 class SettingsTest(unittest.TestCase):
 
-    if six.PY3:
-        assertItemsEqual = unittest.TestCase.assertCountEqual
-
     def setUp(self):
         self.settings = Settings()
 

tests/test_spider.py

@@ -1,5 +1,6 @@
 import gzip
 import inspect
+from unittest import mock
 import warnings
 from io import BytesIO
 
@@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.test import get_crawler
 
-from tests import mock
-
 
 class SpiderTest(unittest.TestCase):
 

tests/test_spidermiddleware.py

@@ -1,3 +1,5 @@
+from unittest import mock
+
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
@@ -6,7 +8,6 @@ from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
-from tests import mock
 
 
 class SpiderMiddlewareTestCase(TestCase):

tests/test_stats.py

@@ -1,10 +1,6 @@
 from datetime import datetime
 import unittest
-
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+from unittest import mock
 
 from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider

tests/test_utils_datatypes.py

@@ -1,11 +1,6 @@
 import copy
 import unittest
-
-import six
-if six.PY2:
-    from collections import Mapping, MutableMapping
-else:
-    from collections.abc import Mapping, MutableMapping
+from collections.abc import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
 

tests/test_utils_deprecate.py

@@ -2,12 +2,11 @@
 from __future__ import absolute_import
 import inspect
 import unittest
+from unittest import mock
 import warnings
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import create_deprecated_class, update_classpath
 
-from tests import mock
-
 
 class MyWarning(UserWarning):
     pass

tests/test_utils_misc/__init__.py

@@ -1,11 +1,11 @@
 import sys
 import os
 import unittest
+from unittest import mock
 
 from scrapy.item import Item, Field
 from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
-from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 

tests/test_utils_python.py

@@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase):
             self.assertEqual(get_func_args(" ".join), [])
             self.assertEqual(get_func_args(operator.itemgetter(2)), [])
         else:
-            stripself = not six.PY2  # PyPy3 exposes them as methods
             self.assertEqual(
-                get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
-            self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+                get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit'])
+            self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
             self.assertEqual(
-                get_func_args(operator.itemgetter(2), stripself), ['obj'])
+                get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
 
 
     def test_without_none_values(self):

tests/test_utils_reqser.py

@@ -80,8 +80,6 @@ class RequestSerializationTest(unittest.TestCase):
         self._assert_serializes_ok(r, spider=self.spider)
 
     def test_mixin_private_callback_serialization(self):
-        if sys.version_info[0] < 3:
-            return
         r = Request("http://www.example.com",
                     callback=self.spider._TestSpiderMixin__mixin_callback,
                     errback=self.spider.handle_error)
@@ -119,9 +117,8 @@ class RequestSerializationTest(unittest.TestCase):
     def test_private_name_mangling(self):
         self._assert_mangles_to(
             self.spider, '_TestSpider__parse_item_private')
-        if sys.version_info[0] >= 3:
-            self._assert_mangles_to(
-                self.spider, '_TestSpiderMixin__mixin_callback')
+        self._assert_mangles_to(
+            self.spider, '_TestSpiderMixin__mixin_callback')
 
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)

tests/test_utils_trackref.py

@@ -1,7 +1,7 @@
 import six
 import unittest
+from unittest import mock
 from scrapy.utils import trackref
-from tests import mock
 
 
 class Foo(trackref.object_ref):