port utils.python

* stringify_dict is deprecated
* is_writable is deprecated
* setattr_default is deprecated
* get_spec is untested
* re_rsearch is untested
* retry_on_eintr is untested

scrapy/utils/python.py

@@ -142,11 +142,14 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
-    pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern
+    if isinstance(pattern, six.string_types):
+        pattern = re.compile(pattern)
+
     for chunk, offset in _chunk_iter():
         matches = [match for match in pattern.finditer(chunk)]
         if matches:
-            return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
+            start, end = matches[-1].span()
+            return offset + start, offset + end
     return None
 
 
@@ -162,14 +165,16 @@ def memoizemethod_noargs(method):
         return cache[self]
     return new_method
 
-_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])
+_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 
 def isbinarytext(text):
-    """Return True if the given text is considered binary, or false
+    """Return True if the given text is considered binary, or False
     otherwise, by looking for binary bytes at their chars
     """
-    assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__
+    if not isinstance(text, bytes):
+        raise TypeError("text must be bytes, got '%s'" % type(text).__name__)
     return any(c in _BINARYCHARS for c in text)
 
 
@@ -273,20 +278,22 @@ class WeakKeyCache(object):
         return self._weakdict[key]
 
 
+@deprecated
 def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
-    """Return a (new) dict with the unicode keys (and values if, keys_only is
+    """Return a (new) dict with unicode keys (and values when "keys_only" is
     False) of the given dict converted to strings. `dct_or_tuples` can be a
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
     for k, v in six.iteritems(dict(dct_or_tuples)):
-        k = k.encode(encoding) if isinstance(k, unicode) else k
+        k = k.encode(encoding) if isinstance(k, six.text_type) else k
         if not keys_only:
-            v = v.encode(encoding) if isinstance(v, unicode) else v
+            v = v.encode(encoding) if isinstance(v, six.text_type) else v
         d[k] = v
     return d
 
 
+@deprecated
 def is_writable(path):
     """Return True if the given path can be written (if it exists) or created
     (if it doesn't exist)
@@ -297,6 +304,7 @@ def is_writable(path):
         return os.access(os.path.dirname(path), os.W_OK)
 
 
+@deprecated
 def setattr_default(obj, name, value):
     """Set attribute value, but only if it's not already set. Similar to
     setdefault() for dicts.

tests/test_utils_python.py

@@ -2,6 +2,7 @@ import functools
 import operator
 import unittest
 from itertools import count
+import six
 
 from scrapy.utils.python import (
     memoizemethod_noargs, isbinarytext, equal_attributes,
@@ -44,7 +45,7 @@ class ToBytesTest(unittest.TestCase):
         self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
 
 
-class UtilsPythonTestCase(unittest.TestCase):
+class MemoizedMethodTest(unittest.TestCase):
     def test_memoizemethod_noargs(self):
         class A(object):
 
@@ -62,19 +63,23 @@ class UtilsPythonTestCase(unittest.TestCase):
         assert one is two
         assert one is not three
 
+
+class IsBinaryTextTest(unittest.TestCase):
     def test_isbinarytext(self):
+        assert not isbinarytext(b"hello")
 
-        # basic tests
-        assert not isbinarytext("hello")
-
-        # utf-16 strings contain null bytes
+    def utf_16_strings_contain_null_bytes(self):
         assert not isbinarytext(u"hello".encode('utf-16'))
 
-        # one with encoding
-        assert not isbinarytext("<div>Price \xa3</div>")
+    def test_one_with_encoding(self):
+        assert not isbinarytext(b"<div>Price \xa3</div>")
 
-        # finally some real binary bytes
-        assert isbinarytext("\x02\xa3")
+    def test_finally_some_real_binary_bytes(self):
+        assert isbinarytext(b"\x02\xa3")
+
+
+
+class UtilsPythonTestCase(unittest.TestCase):
 
     def test_equal_attributes(self):
         class Obj:
@@ -134,29 +139,32 @@ class UtilsPythonTestCase(unittest.TestCase):
         del k
         self.assertFalse(len(wk._weakdict))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict(self):
-        d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
+        d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
-        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_tuples(self):
         tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
         d = dict(tuples)
         d2 = stringify_dict(tuples, keys_only=False)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
-        self.failIf(any(isinstance(x, unicode) for x in d2.values()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
 
+    @unittest.skipUnless(six.PY2, "deprecated function")
     def test_stringify_dict_keys_only(self):
         d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
         d2 = stringify_dict(d)
         self.assertEqual(d, d2)
-        self.failIf(d is d2) # shouldn't modify in place
-        self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
+        self.failIf(d is d2)  # shouldn't modify in place
+        self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
 
     def test_get_func_args(self):
         def f1(a, b, c):
@@ -194,7 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(object), [])
 
         # TODO: how do we fix this to return the actual argument names?
-        self.assertEqual(get_func_args(unicode.split), [])
+        self.assertEqual(get_func_args(six.text_type.split), [])
         self.assertEqual(get_func_args(" ".join), [])
         self.assertEqual(get_func_args(operator.itemgetter(2)), [])