Rename unicode_to_str and str_to_unicode method. Fixes GH-778.

scrapy/http/request/form.py

@@ -9,7 +9,7 @@ from six.moves.urllib.parse import urljoin, urlencode
 import lxml.html
 import six
 from scrapy.http.request import Request
-from scrapy.utils.python import unicode_to_str
+from scrapy.utils.python import to_bytes
 
 
 class FormRequest(Request):
@@ -48,7 +48,7 @@ def _get_form_url(form, url):
 
 
 def _urlencode(seq, enc):
-    values = [(unicode_to_str(k, enc), unicode_to_str(v, enc))
+    values = [(to_bytes(k, enc), to_bytes(v, enc))
               for k, vs in seq
               for v in (vs if hasattr(vs, '__iter__') else [vs])]
     return urlencode(values, doseq=1)

scrapy/linkextractors/lxmlhtml.py

@@ -10,7 +10,7 @@ import lxml.etree as etree
 from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import unique as unique_list, str_to_unicode
+from scrapy.utils.python import unique as unique_list
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.response import get_base_url
 

scrapy/linkextractors/sgml.py

@@ -10,7 +10,7 @@ from scrapy.selector import Selector
 from scrapy.link import Link
 from scrapy.linkextractors import FilteringLinkExtractor
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.python import unique as unique_list, str_to_unicode
+from scrapy.utils.python import unique as unique_list, to_unicode
 from scrapy.utils.response import get_base_url
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -44,7 +44,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
                 link.url = link.url.encode(response_encoding)
             link.url = urljoin(base_url, link.url)
             link.url = safe_url_string(link.url, response_encoding)
-            link.text = str_to_unicode(link.text, response_encoding, errors='replace').strip()
+            link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
             ret.append(link)
 
         return ret

scrapy/selector/unified.py

@@ -7,7 +7,7 @@ import six
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
-from scrapy.utils.python import unicode_to_str, flatten, iflatten
+from scrapy.utils.python import to_bytes, flatten, iflatten
 from scrapy.utils.decorators import deprecated
 from scrapy.http import HtmlResponse, XmlResponse
 from .lxmldocument import LxmlDocument
@@ -44,7 +44,7 @@ def _st(response, st):
 def _response_from_text(text, st):
     rt = XmlResponse if st == 'xml' else HtmlResponse
     return rt(url='about:blank', encoding='utf-8',
-              body=unicode_to_str(text, 'utf-8'))
+              body=to_bytes(text, 'utf-8'))
 
 
 class Selector(object_ref):

scrapy/utils/iterators.py

@@ -11,7 +11,7 @@ import six
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
-from scrapy.utils.python import re_rsearch, str_to_unicode
+from scrapy.utils.python import re_rsearch, to_unicode
 
 logger = logging.getLogger(__name__)
 
@@ -98,7 +98,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
     encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
     def _getrow(csv_r):
-        return [str_to_unicode(field, encoding) for field in next(csv_r)]
+        return [to_unicode(field, encoding) for field in next(csv_r)]
 
     lines = BytesIO(_body_or_str(obj, unicode=False))
 

scrapy/utils/python.py

@@ -1,9 +1,5 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
-
-It also contains functions (or functionality) which is in Python versions
-higher than 2.5 which used to be the lowest version supported by Scrapy.
-
 """
 import os
 import re
@@ -13,6 +9,8 @@ import errno
 import six
 from functools import partial, wraps
 
+from scrapy.utils.decorators import deprecated
+
 
 def flatten(x):
     """flatten(sequence) -> list
@@ -56,37 +54,44 @@ def unique(list_, key=lambda x: x):
     return result
 
 
+@deprecated("scrapy.utils.python.to_unicode")
 def str_to_unicode(text, encoding=None, errors='strict'):
-    """Return the unicode representation of text in the given encoding. Unlike
-    .encode(encoding) this function can be applied directly to a unicode
-    object without the risk of double-decoding problems (which can happen if
-    you don't use the default 'ascii' encoding)
-    """
+    """ This function is deprecated.
+    Please use scrapy.utils.python.to_unicode. """
+    return to_unicode(text, encoding, errors)
 
-    if encoding is None:
-        encoding = 'utf-8'
-    if isinstance(text, str):
-        return text.decode(encoding, errors)
-    elif isinstance(text, unicode):
-        return text
-    else:
-        raise TypeError('str_to_unicode must receive a str or unicode object, got %s' % type(text).__name__)
 
+@deprecated("scrapy.utils.python.to_bytes")
 def unicode_to_str(text, encoding=None, errors='strict'):
-    """Return the str representation of text in the given encoding. Unlike
-    .encode(encoding) this function can be applied directly to a str
-    object without the risk of double-decoding problems (which can happen if
-    you don't use the default 'ascii' encoding)
-    """
+    """ This function is deprecated. Please use scrapy.utils.python.to_bytes """
+    return to_bytes(text, encoding, errors)
 
+
+def to_unicode(text, encoding=None, errors='strict'):
+    """Return the unicode representation of a bytes object `text`. If `text`
+    is already an unicode object, return it as-is."""
+    if isinstance(text, six.text_type):
+        return text
+    if not isinstance(text, (bytes, six.text_type)):
+        raise TypeError('to_unicode must receive a bytes, str or unicode '
+                        'object, got %s' % type(text).__name__)
     if encoding is None:
         encoding = 'utf-8'
-    if isinstance(text, unicode):
-        return text.encode(encoding, errors)
-    elif isinstance(text, str):
+    return text.decode(encoding, errors)
+
+
+def to_bytes(text, encoding=None, errors='strict'):
+    """Return the binary representation of `text`. If `text`
+    is already a bytes object, return it as-is."""
+    if isinstance(text, bytes):
         return text
-    else:
-        raise TypeError('unicode_to_str must receive a unicode or str object, got %s' % type(text).__name__)
+    if not isinstance(text, six.string_types):
+        raise TypeError('to_bytes must receive a unicode, str or bytes '
+                        'object, got %s' % type(text).__name__)
+    if encoding is None:
+        encoding = 'utf-8'
+    return text.encode(encoding, errors)
+
 
 def re_rsearch(pattern, text, chunk_size=1024):
     """
@@ -117,6 +122,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
             return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
     return None
 
+
 def memoizemethod_noargs(method):
     """Decorator to cache the result of a method (without arguments) using a
     weak reference to its object
@@ -131,6 +137,7 @@ def memoizemethod_noargs(method):
 
 _BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])
 
+
 def isbinarytext(text):
     """Return True if the given text is considered binary, or false
     otherwise, by looking for binary bytes at their chars
@@ -138,6 +145,7 @@ def isbinarytext(text):
     assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__
     return any(c in _BINARYCHARS for c in text)
 
+
 def get_func_args(func, stripself=False):
     """Return the argument name list of a callable"""
     if inspect.isfunction(func):
@@ -164,6 +172,7 @@ def get_func_args(func, stripself=False):
         func_args.pop(0)
     return func_args
 
+
 def get_spec(func):
     """Returns (args, kwargs) tuple for a function
     >>> import re
@@ -200,6 +209,7 @@ def get_spec(func):
     kwargs = dict(zip(spec.args[firstdefault:], defaults))
     return args, kwargs
 
+
 def equal_attributes(obj1, obj2, attributes):
     """Compare two objects attributes"""
     # not attributes given return False by default
@@ -249,6 +259,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
         d[k] = v
     return d
 
+
 def is_writable(path):
     """Return True if the given path can be written (if it exists) or created
     (if it doesn't exist)
@@ -258,6 +269,7 @@ def is_writable(path):
     else:
         return os.access(os.path.dirname(path), os.W_OK)
 
+
 def setattr_default(obj, name, value):
     """Set attribute value, but only if it's not already set. Similar to
     setdefault() for dicts.

scrapy/utils/url.py

@@ -12,7 +12,7 @@ from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag,
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code
 from w3lib.url import *
-from scrapy.utils.python import unicode_to_str
+from scrapy.utils.python import to_bytes
 
 
 def url_is_from_any_domain(url, domains):
@@ -72,8 +72,9 @@ def parse_url(url, encoding=None):
     """Return urlparsed url from the given argument (which could be an already
     parsed url)
     """
-    return url if isinstance(url, ParseResult) else \
-        urlparse(unicode_to_str(url, encoding))
+    if isinstance(url, ParseResult):
+        return url
+    return urlparse(to_bytes(url, encoding))
 
 
 def escape_ajax(url):

tests/test_exporters.py

@@ -8,7 +8,7 @@ from six.moves import cPickle as pickle
 import lxml.etree
 
 from scrapy.item import Item, Field
-from scrapy.utils.python import str_to_unicode
+from scrapy.utils.python import to_unicode
 from scrapy.exporters import (
     BaseItemExporter, PprintItemExporter, PickleItemExporter, CsvItemExporter,
     XmlItemExporter, JsonLinesItemExporter, JsonItemExporter, PythonItemExporter
@@ -35,7 +35,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
     def _assert_expected_item(self, exported_dict):
         for k, v in exported_dict.items():
-            exported_dict[k] = str_to_unicode(v)
+            exported_dict[k] = to_unicode(v)
         self.assertEqual(self.i, exported_dict)
 
     def assertItemExportWorks(self, item):

tests/test_utils_python.py

@@ -3,45 +3,48 @@ import operator
 import unittest
 from itertools import count
 
-from scrapy.utils.python import str_to_unicode, unicode_to_str, \
-    memoizemethod_noargs, isbinarytext, equal_attributes, \
-    WeakKeyCache, stringify_dict, get_func_args
+from scrapy.utils.python import (
+    memoizemethod_noargs, isbinarytext, equal_attributes,
+    WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode)
 
 __doctests__ = ['scrapy.utils.python']
 
+
+class ToUnicodeTest(unittest.TestCase):
+    def test_converting_an_utf8_encoded_string_to_unicode(self):
+        self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
+
+    def test_converting_a_latin_1_encoded_string_to_unicode(self):
+        self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
+
+    def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
+        self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
+
+    def test_converting_a_strange_object_should_raise_TypeError(self):
+        self.assertRaises(TypeError, to_unicode, 423)
+
+    def test_check_errors_argument_works(self):
+        self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace'))
+
+
+class ToBytesTest(unittest.TestCase):
+    def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
+        self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49')
+
+    def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
+        self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49')
+
+    def test_converting_a_regular_string_to_string_should_return_the_same_object(self):
+        self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e')
+
+    def test_converting_a_strange_object_should_raise_TypeError(self):
+        self.assertRaises(TypeError, to_bytes, unittest)
+
+    def test_check_errors_argument_works(self):
+        self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
+
+
 class UtilsPythonTestCase(unittest.TestCase):
-    def test_str_to_unicode(self):
-        # converting an utf-8 encoded string to unicode
-        self.assertEqual(str_to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
-
-        # converting a latin-1 encoded string to unicode
-        self.assertEqual(str_to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
-
-        # converting a unicode to unicode should return the same object
-        self.assertEqual(str_to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
-
-        # converting a strange object should raise TypeError
-        self.assertRaises(TypeError, str_to_unicode, 423)
-
-        # check errors argument works
-        assert u'\ufffd' in str_to_unicode('a\xedb', 'utf-8', errors='replace')
-
-    def test_unicode_to_str(self):
-        # converting a unicode object to an utf-8 encoded string
-        self.assertEqual(unicode_to_str(u'\xa3 49'), '\xc2\xa3 49')
-
-        # converting a unicode object to a latin-1 encoded string
-        self.assertEqual(unicode_to_str(u'\xa3 49', 'latin-1'), '\xa3 49')
-
-        # converting a regular string to string should return the same object
-        self.assertEqual(unicode_to_str('lel\xf1e'), 'lel\xf1e')
-
-        # converting a strange object should raise TypeError
-        self.assertRaises(TypeError, unicode_to_str, unittest)
-
-        # check errors argument works
-        assert '?' in unicode_to_str(u'a\ufffdb', 'latin-1', errors='replace')
-
     def test_memoizemethod_noargs(self):
         class A(object):