Replace to_native_str calls with to_unicode.

scrapy/core/downloader/handlers/http11.py

@@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
     r"""
     Return binary content of a CONNECT request.
 
-    >>> from scrapy.utils.python import to_native_str as s
+    >>> from scrapy.utils.python import to_unicode as s
     >>> s(tunnel_request_data("example.com", 8080))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))

scrapy/downloadermiddlewares/cookies.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +53,7 @@ class CookiesMiddleware(object):
 
     def _debug_cookie(self, request, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in request.headers.getlist('Cookie')]
             if cl:
                 cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +62,7 @@ class CookiesMiddleware(object):
 
     def _debug_set_cookie(self, response, spider):
         if self.debug:
-            cl = [to_native_str(c, errors='replace')
+            cl = [to_unicode(c, errors='replace')
                   for c in response.headers.getlist('Set-Cookie')]
             if cl:
                 cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)

scrapy/downloadermiddlewares/robotstxt.py

@@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-import sys
-import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
 from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)

scrapy/exporters.py

@@ -12,7 +12,7 @@ from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
 import warnings
@@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter):
     def _build_row(self, values):
         for s in values:
             try:
-                yield to_native_str(s, self.encoding)
+                yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 

scrapy/http/cookies.py

@@ -3,7 +3,7 @@ from six.moves.http_cookiejar import (
     CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
 )
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 class CookieJar(object):
@@ -165,13 +165,13 @@ class WrappedRequest(object):
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_native_str(self.request.headers.get(name, default),
+        return to_unicode(self.request.headers.get(name, default),
                              errors='replace')
 
     def header_items(self):
         return [
-            (to_native_str(k, errors='replace'),
-             [to_native_str(x, errors='replace') for x in v])
+            (to_unicode(k, errors='replace'),
+             [to_unicode(x, errors='replace') for x in v])
             for k, v in self.request.headers.items()
         ]
 
@@ -189,7 +189,7 @@ class WrappedResponse(object):
 
     # python3 cookiejars calls get_all
     def get_all(self, name, default=None):
-        return [to_native_str(v, errors='replace')
+        return [to_unicode(v, errors='replace')
                 for v in self.response.headers.getlist(name)]
     # python2 cookiejars calls getheaders
     getheaders = get_all

scrapy/http/response/text.py

@@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace
 from scrapy.http.request import Request
 from scrapy.http.response import Response
 from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
 
 
 class TextResponse(Response):
@@ -32,7 +32,7 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, six.text_type):
-            self._url = to_native_str(url, self.encoding)
+            self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
@@ -81,11 +81,11 @@ class TextResponse(Response):
     @memoizemethod_noargs
     def _headers_encoding(self):
         content_type = self.headers.get(b'Content-Type', b'')
-        return http_content_type_encoding(to_native_str(content_type))
+        return http_content_type_encoding(to_unicode(content_type))
 
     def _body_inferred_encoding(self):
         if self._cached_benc is None:
-            content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+            content_type = to_unicode(self.headers.get(b'Content-Type', b''))
             benc, ubody = html_to_unicode(content_type, self.body,
                     auto_detect_fun=self._auto_detect_fun,
                     default_encoding=self._DEFAULT_ENCODING)

scrapy/linkextractors/lxmlhtml.py

@@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_native_str(url, encoding=response_encoding)
+            url = to_unicode(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',

scrapy/responsetypes.py

@@ -10,7 +10,7 @@ import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
 
 
 class ResponseTypes(object):
@@ -55,12 +55,12 @@ class ResponseTypes(object):
         header """
         if content_encoding:
             return Response
-        mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+        mimetype = to_unicode(content_type).split(';')[0].strip().lower()
         return self.from_mimetype(mimetype)
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition,
+            filename = to_unicode(content_disposition,
                 encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)

scrapy/robotstxt.py

@@ -3,14 +3,14 @@ import logging
 from abc import ABCMeta, abstractmethod
 from six import with_metaclass
 
-from scrapy.utils.python import to_native_str, to_unicode
+from scrapy.utils.python import to_unicode
 
 logger = logging.getLogger(__name__)
 
 def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     try:
         if to_native_str_type:
-            robotstxt_body = to_native_str(robotstxt_body)
+            robotstxt_body = to_unicode(robotstxt_body)
         else:
             robotstxt_body = robotstxt_body.decode('utf-8')
     except UnicodeDecodeError:
@@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser):
         return o
 
     def allowed(self, url, user_agent):
-        user_agent = to_native_str(user_agent)
-        url = to_native_str(url)
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
 

scrapy/spidermiddlewares/referer.py

@@ -10,8 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
 
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
             if isinstance(resp_or_url, Response):
                 policy_header = resp_or_url.headers.get('Referrer-Policy')
                 if policy_header is not None:
-                    policy_name = to_native_str(policy_header.decode('latin1'))
+                    policy_name = to_unicode(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 

scrapy/utils/reqser.py

@@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests.
 import six
 
 from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 
 
@@ -54,7 +54,7 @@ def request_from_dict(d, spider=None):
         eb = _get_method(spider, eb)
     request_cls = load_object(d['_class']) if '_class' in d else Request
     return request_cls(
-        url=to_native_str(d['url']),
+        url=to_unicode(d['url']),
         callback=cb,
         errback=eb,
         method=d['method'],

scrapy/utils/request.py

@@ -9,7 +9,7 @@ import weakref
 from six.moves.urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 from w3lib.url import canonicalize_url
 from scrapy.utils.httpobj import urlparse_cached
@@ -97,4 +97,4 @@ def referer_str(request):
     referrer = request.headers.get('Referer')
     if referrer is None:
         return referrer
-    return to_native_str(referrer, errors='replace')
+    return to_unicode(referrer, errors='replace')

scrapy/utils/response.py

@@ -8,7 +8,7 @@ import webbrowser
 import tempfile
 
 from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 from w3lib import html
 
 
@@ -36,7 +36,7 @@ def response_status_message(status):
     """Return status code plus status text descriptive message
     """
     message = http.RESPONSES.get(int(status), "Unknown Status")
-    return '%s %s' % (status, to_native_str(message))
+    return '%s %s' % (status, to_unicode(message))
 
 
 def response_httprepr(response):

scrapy/utils/ssl.py

@@ -3,7 +3,7 @@
 import OpenSSL
 import OpenSSL._util as pyOpenSSLutil
 
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 
 
 # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
@@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
 
 
 def ffi_buf_to_string(buf):
-    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+    return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
 def x509name_to_string(x509name):

tests/test_command_parse.py

@@ -1,17 +1,16 @@
 import os
 from os.path import join, abspath
-from twisted.trial import unittest
 from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from tests.test_commands import CommandTest
 
 
 def _textmode(bstr):
     """Normalize input the same as writing to a file
     and reading from it in text mode"""
-    return to_native_str(bstr).replace(os.linesep, '\n')
+    return to_unicode(bstr).replace(os.linesep, '\n')
 
 class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
     command = 'parse'

tests/test_commands.py

@@ -10,13 +10,10 @@ from contextlib import contextmanager
 from threading import Timer
 
 from twisted.trial import unittest
-from twisted.internet import defer
 
 import scrapy
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
-from scrapy.utils.testsite import SiteTest
-from scrapy.utils.testproc import ProcessTest
 from tests.test_crawler import ExceptionSpider, NoRequestsSpider
 
 
@@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase):
         finally:
             timer.cancel()
 
-        return p, to_native_str(stdout), to_native_str(stderr)
+        return p, to_unicode(stdout), to_unicode(stderr)
 
 
 class StartprojectTest(ProjectTest):

tests/test_feedexport.py

@@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import (
     S3FeedStorage, StdoutFeedStorage,
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
-from scrapy.utils.python import to_native_str
-from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_unicode
 
 from pathlib import Path
 
@@ -459,7 +458,7 @@ class FeedExportTest(unittest.TestCase):
         settings.update({'FEED_FORMAT': 'csv'})
         data = yield self.exported_data(items, settings)
 
-        reader = csv.DictReader(to_native_str(data).splitlines())
+        reader = csv.DictReader(to_unicode(data).splitlines())
         got_rows = list(reader)
         if ordered:
             self.assertEqual(reader.fieldnames, header)
@@ -473,7 +472,7 @@ class FeedExportTest(unittest.TestCase):
         settings = settings or {}
         settings.update({'FEED_FORMAT': 'jl'})
         data = yield self.exported_data(items, settings)
-        parsed = [json.loads(to_native_str(line)) for line in data.splitlines()]
+        parsed = [json.loads(to_unicode(line)) for line in data.splitlines()]
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         self.assertEqual(rows, parsed)
 

tests/test_http_request.py

@@ -7,12 +7,11 @@ from unittest import mock
 from urllib.parse import unquote_to_bytes
 import warnings
 
-import six
 from six.moves import xmlrpc_client as xmlrpclib
 from six.moves.urllib.parse import urlparse, parse_qs, unquote
 
 from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 class RequestTest(unittest.TestCase):
@@ -349,8 +348,8 @@ class FormRequestTest(RequestTest):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
-        first = to_native_str(first).split("&")
-        second = to_native_str(second).split("&")
+        first = to_unicode(first).split("&")
+        second = to_unicode(second).split("&")
         return self.assertEqual(sorted(first), sorted(second), msg)
 
     def test_empty_formdata(self):

tests/test_http_response.py

@@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding
 from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
                          XmlResponse, Headers)
 from scrapy.selector import Selector
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
 from scrapy.exceptions import NotSupported
 from scrapy.link import Link
 from tests import get_testdata
@@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest):
         assert isinstance(resp.url, str)
 
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
-        self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3'))
+        self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3'))
         resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
         self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
 

tests/test_robotstxt_interface.py

@@ -1,6 +1,5 @@
 # coding=utf-8
 from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
 
 
 def reppy_available():