diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a4695f1a2..e083ce6e9 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -9,7 +9,7 @@ from six.moves.urllib.parse import urljoin, urlencode import lxml.html import six from scrapy.http.request import Request -from scrapy.utils.python import unicode_to_str +from scrapy.utils.python import to_bytes class FormRequest(Request): @@ -48,7 +48,7 @@ def _get_form_url(form, url): def _urlencode(seq, enc): - values = [(unicode_to_str(k, enc), unicode_to_str(v, enc)) + values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if hasattr(vs, '__iter__') else [vs])] return urlencode(values, doseq=1) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 1c31a15b5..c952a5f83 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -10,7 +10,7 @@ import lxml.etree as etree from scrapy.selector import Selector from scrapy.link import Link from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import unique as unique_list, str_to_unicode +from scrapy.utils.python import unique as unique_list from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.response import get_base_url diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index a9b8fe9de..e4c2c274f 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -10,7 +10,7 @@ from scrapy.selector import Selector from scrapy.link import Link from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import unique as unique_list, str_to_unicode +from scrapy.utils.python import unique as unique_list, to_unicode from scrapy.utils.response import get_base_url from scrapy.exceptions import ScrapyDeprecationWarning @@ -44,7 +44,7 @@ class BaseSgmlLinkExtractor(SGMLParser): link.url = link.url.encode(response_encoding) link.url = urljoin(base_url, link.url) link.url = safe_url_string(link.url, response_encoding) - link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip() + link.text = to_unicode(link.text, response_encoding, errors='replace').strip() ret.append(link) return ret diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 0a2f40c6a..69ea9a03d 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -7,7 +7,7 @@ import six from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref -from scrapy.utils.python import unicode_to_str, flatten, iflatten +from scrapy.utils.python import to_bytes, flatten, iflatten from scrapy.utils.decorators import deprecated from scrapy.http import HtmlResponse, XmlResponse from .lxmldocument import LxmlDocument @@ -44,7 +44,7 @@ def _st(response, st): def _response_from_text(text, st): rt = XmlResponse if st == 'xml' else HtmlResponse return rt(url='about:blank', encoding='utf-8', - body=unicode_to_str(text, 'utf-8')) + body=to_bytes(text, 'utf-8')) class Selector(object_ref): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 58ddf6215..b9f682431 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -11,7 +11,7 @@ import six from scrapy.http import TextResponse, Response from scrapy.selector import Selector -from scrapy.utils.python import re_rsearch, str_to_unicode +from scrapy.utils.python import re_rsearch, to_unicode logger = logging.getLogger(__name__) @@ -98,7 +98,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8' def _getrow(csv_r): - return [str_to_unicode(field, encoding) for field in next(csv_r)] + return [to_unicode(field, encoding) for field in next(csv_r)] lines = BytesIO(_body_or_str(obj, unicode=False)) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index b6100f899..5d5d5b004 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,9 +1,5 @@ """ This module contains essential stuff that should've come with Python itself ;) - -It also contains functions (or functionality) which is in Python versions -higher than 2.5 which used to be the lowest version supported by Scrapy. - """ import os import re @@ -13,6 +9,8 @@ import errno import six from functools import partial, wraps +from scrapy.utils.decorators import deprecated + def flatten(x): """flatten(sequence) -> list @@ -56,37 +54,44 @@ def unique(list_, key=lambda x: x): return result +@deprecated("scrapy.utils.python.to_unicode") def str_to_unicode(text, encoding=None, errors='strict'): - """Return the unicode representation of text in the given encoding. Unlike - .encode(encoding) this function can be applied directly to a unicode - object without the risk of double-decoding problems (which can happen if - you don't use the default 'ascii' encoding) - """ + """ This function is deprecated. + Please use scrapy.utils.python.to_unicode. """ + return to_unicode(text, encoding, errors) - if encoding is None: - encoding = 'utf-8' - if isinstance(text, str): - return text.decode(encoding, errors) - elif isinstance(text, unicode): - return text - else: - raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__) +@deprecated("scrapy.utils.python.to_bytes") def unicode_to_str(text, encoding=None, errors='strict'): - """Return the str representation of text in the given encoding. Unlike - .encode(encoding) this function can be applied directly to a str - object without the risk of double-decoding problems (which can happen if - you don't use the default 'ascii' encoding) - """ + """ This function is deprecated. Please use scrapy.utils.python.to_bytes """ + return to_bytes(text, encoding, errors) + +def to_unicode(text, encoding=None, errors='strict'): + """Return the unicode representation of a bytes object `text`. If `text` + is already an unicode object, return it as-is.""" + if isinstance(text, six.text_type): + return text + if not isinstance(text, (bytes, six.text_type)): + raise TypeError('to_unicode must receive a bytes, str or unicode ' + 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' - if isinstance(text, unicode): - return text.encode(encoding, errors) - elif isinstance(text, str): + return text.decode(encoding, errors) + + +def to_bytes(text, encoding=None, errors='strict'): + """Return the binary representation of `text`. If `text` + is already a bytes object, return it as-is.""" + if isinstance(text, bytes): return text - else: - raise TypeError('unicode_to_str must receive a unicode or str object, got %s' % type(text).__name__) + if not isinstance(text, six.string_types): + raise TypeError('to_bytes must receive a unicode, str or bytes ' + 'object, got %s' % type(text).__name__) + if encoding is None: + encoding = 'utf-8' + return text.encode(encoding, errors) + def re_rsearch(pattern, text, chunk_size=1024): """ @@ -117,6 +122,7 @@ def re_rsearch(pattern, text, chunk_size=1024): return (offset + matches[-1].span()[0], offset + matches[-1].span()[1]) return None + def memoizemethod_noargs(method): """Decorator to cache the result of a method (without arguments) using a weak reference to its object @@ -131,6 +137,7 @@ def memoizemethod_noargs(method): _BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"]) + def isbinarytext(text): """Return True if the given text is considered binary, or false otherwise, by looking for binary bytes at their chars @@ -138,6 +145,7 @@ def isbinarytext(text): assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ return any(c in _BINARYCHARS for c in text) + def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): @@ -164,6 +172,7 @@ def get_func_args(func, stripself=False): func_args.pop(0) return func_args + def get_spec(func): """Returns (args, kwargs) tuple for a function >>> import re @@ -200,6 +209,7 @@ def get_spec(func): kwargs = dict(zip(spec.args[firstdefault:], defaults)) return args, kwargs + def equal_attributes(obj1, obj2, attributes): """Compare two objects attributes""" # not attributes given return False by default @@ -249,6 +259,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): d[k] = v return d + def is_writable(path): """Return True if the given path can be written (if it exists) or created (if it doesn't exist) @@ -258,6 +269,7 @@ def is_writable(path): else: return os.access(os.path.dirname(path), os.W_OK) + def setattr_default(obj, name, value): """Set attribute value, but only if it's not already set. Similar to setdefault() for dicts. diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 5c645aaf0..8a8c56814 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -12,7 +12,7 @@ from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag, # scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code from w3lib.url import * -from scrapy.utils.python import unicode_to_str +from scrapy.utils.python import to_bytes def url_is_from_any_domain(url, domains): @@ -72,8 +72,9 @@ def parse_url(url, encoding=None): """Return urlparsed url from the given argument (which could be an already parsed url) """ - return url if isinstance(url, ParseResult) else \ - urlparse(unicode_to_str(url, encoding)) + if isinstance(url, ParseResult): + return url + return urlparse(to_bytes(url, encoding)) def escape_ajax(url): diff --git a/tests/test_exporters.py b/tests/test_exporters.py index df1f8f4d5..b24633959 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -8,7 +8,7 @@ from six.moves import cPickle as pickle import lxml.etree from scrapy.item import Item, Field -from scrapy.utils.python import str_to_unicode +from scrapy.utils.python import to_unicode from scrapy.exporters import ( BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter, XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter @@ -35,7 +35,7 @@ class BaseItemExporterTest(unittest.TestCase): def _assert_expected_item(self, exported_dict): for k, v in exported_dict.items(): - exported_dict[k] = str_to_unicode(v) + exported_dict[k] = to_unicode(v) self.assertEqual(self.i, exported_dict) def assertItemExportWorks(self, item): diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index badfefe04..fa77356de 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,45 +3,48 @@ import operator import unittest from itertools import count -from scrapy.utils.python import str_to_unicode, unicode_to_str, \ - memoizemethod_noargs, isbinarytext, equal_attributes, \ - WeakKeyCache, stringify_dict, get_func_args +from scrapy.utils.python import ( + memoizemethod_noargs, isbinarytext, equal_attributes, + WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode) __doctests__ = ['scrapy.utils.python'] + +class ToUnicodeTest(unittest.TestCase): + def test_converting_an_utf8_encoded_string_to_unicode(self): + self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e') + + def test_converting_a_latin_1_encoded_string_to_unicode(self): + self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e') + + def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): + self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') + + def test_converting_a_strange_object_should_raise_TypeError(self): + self.assertRaises(TypeError, to_unicode, 423) + + def test_check_errors_argument_works(self): + self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace')) + + +class ToBytesTest(unittest.TestCase): + def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): + self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49') + + def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): + self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49') + + def test_converting_a_regular_string_to_string_should_return_the_same_object(self): + self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e') + + def test_converting_a_strange_object_should_raise_TypeError(self): + self.assertRaises(TypeError, to_bytes, unittest) + + def test_check_errors_argument_works(self): + self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace')) + + class UtilsPythonTestCase(unittest.TestCase): - def test_str_to_unicode(self): - # converting an utf-8 encoded string to unicode - self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e') - - # converting a latin-1 encoded string to unicode - self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e') - - # converting a unicode to unicode should return the same object - self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') - - # converting a strange object should raise TypeError - self.assertRaises(TypeError, str_to_unicode, 423) - - # check errors argument works - assert u'\ufffd' in str_to_unicode('a\xedb', 'utf-8', errors='replace') - - def test_unicode_to_str(self): - # converting a unicode object to an utf-8 encoded string - self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49') - - # converting a unicode object to a latin-1 encoded string - self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49') - - # converting a regular string to string should return the same object - self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e') - - # converting a strange object should raise TypeError - self.assertRaises(TypeError, unicode_to_str, unittest) - - # check errors argument works - assert '?' in unicode_to_str(u'a\ufffdb', 'latin-1', errors='replace') - def test_memoizemethod_noargs(self): class A(object):