From a138fb05d4f0d90e2002e85a348a5be34904d3d8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 21:35:13 +0500 Subject: [PATCH] Replace to_native_str calls with to_unicode. --- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/downloadermiddlewares/cookies.py | 6 +++--- scrapy/downloadermiddlewares/robotstxt.py | 3 --- scrapy/exporters.py | 4 ++-- scrapy/http/cookies.py | 10 +++++----- scrapy/http/response/text.py | 8 ++++---- scrapy/linkextractors/lxmlhtml.py | 4 ++-- scrapy/responsetypes.py | 6 +++--- scrapy/robotstxt.py | 8 ++++---- scrapy/spidermiddlewares/referer.py | 5 ++--- scrapy/utils/reqser.py | 4 ++-- scrapy/utils/request.py | 4 ++-- scrapy/utils/response.py | 4 ++-- scrapy/utils/ssl.py | 4 ++-- tests/test_command_parse.py | 5 ++--- tests/test_commands.py | 7 ++----- tests/test_feedexport.py | 7 +++---- tests/test_http_request.py | 7 +++---- tests/test_http_response.py | 6 +++--- tests/test_robotstxt_interface.py | 1 - 20 files changed, 47 insertions(+), 58 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 91b45a8fc..7d917cb74 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None): r""" Return binary content of a CONNECT request. - >>> from scrapy.utils.python import to_native_str as s + >>> from scrapy.utils.python import to_unicode as s >>> s(tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' >>> s(tunnel_request_data("example.com", 8080, b"123")) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 321c0171b..0d2b9900c 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -6,7 +6,7 @@ from collections import defaultdict from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ class CookiesMiddleware(object): def _debug_cookie(self, request, spider): if self.debug: - cl = [to_native_str(c, errors='replace') + cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) @@ -62,7 +62,7 @@ class CookiesMiddleware(object): def _debug_set_cookie(self, response, spider): if self.debug: - cl = [to_native_str(c, errors='replace') + cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a5dfb79c..251706c50 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ import logging -import sys -import re from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import to_native_str from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1aa195b0b..8eb52995e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -12,7 +12,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike +from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning import warnings @@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter): def _build_row(self, values): for s in values: try: - yield to_native_str(s, self.encoding) + yield to_unicode(s, self.encoding) except TypeError: yield s diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 4e8056750..4532c3ab7 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -3,7 +3,7 @@ from six.moves.http_cookiejar import ( CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE ) from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode class CookieJar(object): @@ -165,13 +165,13 @@ class WrappedRequest(object): return name in self.request.headers def get_header(self, name, default=None): - return to_native_str(self.request.headers.get(name, default), + return to_unicode(self.request.headers.get(name, default), errors='replace') def header_items(self): return [ - (to_native_str(k, errors='replace'), - [to_native_str(x, errors='replace') for x in v]) + (to_unicode(k, errors='replace'), + [to_unicode(x, errors='replace') for x in v]) for k, v in self.request.headers.items() ] @@ -189,7 +189,7 @@ class WrappedResponse(object): # python3 cookiejars calls get_all def get_all(self, name, default=None): - return [to_native_str(v, errors='replace') + return [to_unicode(v, errors='replace') for v in self.response.headers.getlist(name)] # python2 cookiejars calls getheaders getheaders = get_all diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a8010877c..37f450e54 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url -from scrapy.utils.python import memoizemethod_noargs, to_native_str +from scrapy.utils.python import memoizemethod_noargs, to_unicode class TextResponse(Response): @@ -32,7 +32,7 @@ class TextResponse(Response): def _set_url(self, url): if isinstance(url, six.text_type): - self._url = to_native_str(url, self.encoding) + self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) @@ -81,11 +81,11 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): content_type = self.headers.get(b'Content-Type', b'') - return http_content_type_encoding(to_native_str(content_type)) + return http_content_type_encoding(to_unicode(content_type)) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_native_str(self.headers.get(b'Content-Type', b'')) + content_type = to_unicode(self.headers.get(b'Content-Type', b'')) benc, ubody = html_to_unicode(content_type, self.body, auto_detect_fun=self._auto_detect_fun, default_encoding=self._DEFAULT_ENCODING) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8f6f93a44..890c019c8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_native_str +from scrapy.utils.python import unique as unique_list, to_unicode from scrapy.utils.response import get_base_url from scrapy.linkextractors import FilteringLinkExtractor @@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - url = to_native_str(url, encoding=response_encoding) + url = to_unicode(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 4a2d5bf52..de62276c8 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -10,7 +10,7 @@ import six from scrapy.http import Response from scrapy.utils.misc import load_object -from scrapy.utils.python import binary_is_text, to_bytes, to_native_str +from scrapy.utils.python import binary_is_text, to_bytes, to_unicode class ResponseTypes(object): @@ -55,12 +55,12 @@ class ResponseTypes(object): header """ if content_encoding: return Response - mimetype = to_native_str(content_type).split(';')[0].strip().lower() + mimetype = to_unicode(content_type).split(';')[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): try: - filename = to_native_str(content_disposition, + filename = to_unicode(content_disposition, encoding='latin-1', errors='replace').split(';')[1].split('=')[1] filename = filename.strip('"\'') return self.from_filename(filename) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..95a8c09b8 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,14 +3,14 @@ import logging from abc import ABCMeta, abstractmethod from six import with_metaclass -from scrapy.utils.python import to_native_str, to_unicode +from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: - robotstxt_body = to_native_str(robotstxt_body) + robotstxt_body = to_unicode(robotstxt_body) else: robotstxt_body = robotstxt_body.decode('utf-8') except UnicodeDecodeError: @@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser): return o def allowed(self, url, user_agent): - user_agent = to_native_str(user_agent) - url = to_native_str(url) + user_agent = to_unicode(user_agent) + url = to_unicode(url) return self.rp.can_fetch(user_agent, url) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1ddfb37f4..c76e4d5a2 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,8 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_native_str -from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -322,7 +321,7 @@ class RefererMiddleware(object): if isinstance(resp_or_url, Response): policy_header = resp_or_url.headers.get('Referrer-Policy') if policy_header is not None: - policy_name = to_native_str(policy_header.decode('latin1')) + policy_name = to_unicode(policy_header.decode('latin1')) if policy_name is None: return self.default_policy() diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index c7ea7b425..495564ac0 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests. import six from scrapy.http import Request -from scrapy.utils.python import to_unicode, to_native_str +from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -54,7 +54,7 @@ def request_from_dict(d, spider=None): eb = _get_method(spider, eb) request_cls = load_object(d['_class']) if '_class' in d else Request return request_cls( - url=to_native_str(d['url']), + url=to_unicode(d['url']), callback=cb, errback=eb, method=d['method'], diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index fb5af66a2..63d0ae772 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -9,7 +9,7 @@ import weakref from six.moves.urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode from w3lib.url import canonicalize_url from scrapy.utils.httpobj import urlparse_cached @@ -97,4 +97,4 @@ def referer_str(request): referrer = request.headers.get('Referer') if referrer is None: return referrer - return to_native_str(referrer, errors='replace') + return to_unicode(referrer, errors='replace') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c3236afd4..feab07431 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -8,7 +8,7 @@ import webbrowser import tempfile from twisted.web import http -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode from w3lib import html @@ -36,7 +36,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_native_str(message)) + return '%s %s' % (status, to_unicode(message)) def response_httprepr(response): diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 02aed60ee..6e81b33ff 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -3,7 +3,7 @@ import OpenSSL import OpenSSL._util as pyOpenSSLutil -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. @@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) def ffi_buf_to_string(buf): - return to_native_str(pyOpenSSLutil.ffi.string(buf)) + return to_unicode(pyOpenSSLutil.ffi.string(buf)) def x509name_to_string(x509name): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 62d5d76b4..b134beb88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,17 +1,16 @@ import os from os.path import join, abspath -from twisted.trial import unittest from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from tests.test_commands import CommandTest def _textmode(bstr): """Normalize input the same as writing to a file and reading from it in text mode""" - return to_native_str(bstr).replace(os.linesep, '\n') + return to_unicode(bstr).replace(os.linesep, '\n') class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' diff --git a/tests/test_commands.py b/tests/test_commands.py index b8445ae6c..536379170 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,13 +10,10 @@ from contextlib import contextmanager from threading import Timer from twisted.trial import unittest -from twisted.internet import defer import scrapy -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from scrapy.utils.testsite import SiteTest -from scrapy.utils.testproc import ProcessTest from tests.test_crawler import ExceptionSpider, NoRequestsSpider @@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase): finally: timer.cancel() - return p, to_native_str(stdout), to_native_str(stderr) + return p, to_unicode(stdout), to_unicode(stderr) class StartprojectTest(ProjectTest): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 0c70bf80e..87139e81f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler -from scrapy.utils.python import to_native_str -from scrapy.utils.project import get_project_settings +from scrapy.utils.python import to_unicode from pathlib import Path @@ -459,7 +458,7 @@ class FeedExportTest(unittest.TestCase): settings.update({'FEED_FORMAT': 'csv'}) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_native_str(data).splitlines()) + reader = csv.DictReader(to_unicode(data).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -473,7 +472,7 @@ class FeedExportTest(unittest.TestCase): settings = settings or {} settings.update({'FEED_FORMAT': 'jl'}) data = yield self.exported_data(items, settings) - parsed = [json.loads(to_native_str(line)) for line in data.splitlines()] + parsed = [json.loads(to_unicode(line)) for line in data.splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9fe201579..3518da21c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,12 +7,11 @@ from unittest import mock from urllib.parse import unquote_to_bytes import warnings -import six from six.moves import xmlrpc_client as xmlrpclib from six.moves.urllib.parse import urlparse, parse_qs, unquote from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): @@ -349,8 +348,8 @@ class FormRequestTest(RequestTest): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): - first = to_native_str(first).split("&") - second = to_native_str(second).split("&") + first = to_unicode(first).split("&") + second = to_unicode(second).split("&") return self.assertEqual(sorted(first), sorted(second), msg) def test_empty_formdata(self): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index ec3b51086..883c943da 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from scrapy.exceptions import NotSupported from scrapy.link import Link from tests import get_testdata @@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest): assert isinstance(resp.url, str) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') - self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3')) + self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3')) + self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9aaab560a..cd7480e33 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,6 +1,5 @@ # coding=utf-8 from twisted.trial import unittest -from scrapy.utils.python import to_native_str def reppy_available():