From 407349865279795387b9ff84afb9011e2303bb9d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 23 Jul 2015 18:33:56 +0200 Subject: [PATCH] port utils.python * stringify_dict is deprecated * is_writable is deprecated * setattr_default is deprecated * get_spec is untested * re_rsearch is untested * retry_on_eintr is untested --- scrapy/utils/python.py | 24 +++++++++++++------- tests/test_utils_python.py | 46 ++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index f691a302f..57016811f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -142,11 +142,14 @@ def re_rsearch(pattern, text, chunk_size=1024): yield (text[offset:], offset) yield (text, 0) - pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern + if isinstance(pattern, six.string_types): + pattern = re.compile(pattern) + for chunk, offset in _chunk_iter(): matches = [match for match in pattern.finditer(chunk)] if matches: - return (offset + matches[-1].span()[0], offset + matches[-1].span()[1]) + start, end = matches[-1].span() + return offset + start, offset + end return None @@ -162,14 +165,16 @@ def memoizemethod_noargs(method): return cache[self] return new_method -_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"]) +_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} +_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} def isbinarytext(text): - """Return True if the given text is considered binary, or false + """Return True if the given text is considered binary, or False otherwise, by looking for binary bytes at their chars """ - assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ + if not isinstance(text, bytes): + raise TypeError("text must be bytes, got '%s'" % type(text).__name__) return any(c in _BINARYCHARS for c in text) @@ -273,20 +278,22 @@ class WeakKeyCache(object): return self._weakdict[key] +@deprecated def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): - """Return a (new) dict with the unicode keys (and values if, keys_only is + """Return a (new) dict with unicode keys (and values when "keys_only" is False) of the given dict converted to strings. `dct_or_tuples` can be a dict or a list of tuples, like any dict constructor supports. """ d = {} for k, v in six.iteritems(dict(dct_or_tuples)): - k = k.encode(encoding) if isinstance(k, unicode) else k + k = k.encode(encoding) if isinstance(k, six.text_type) else k if not keys_only: - v = v.encode(encoding) if isinstance(v, unicode) else v + v = v.encode(encoding) if isinstance(v, six.text_type) else v d[k] = v return d +@deprecated def is_writable(path): """Return True if the given path can be written (if it exists) or created (if it doesn't exist) @@ -297,6 +304,7 @@ def is_writable(path): return os.access(os.path.dirname(path), os.W_OK) +@deprecated def setattr_default(obj, name, value): """Set attribute value, but only if it's not already set. Similar to setdefault() for dicts. diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index a5f183e6e..3b99fec5b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -2,6 +2,7 @@ import functools import operator import unittest from itertools import count +import six from scrapy.utils.python import ( memoizemethod_noargs, isbinarytext, equal_attributes, @@ -44,7 +45,7 @@ class ToBytesTest(unittest.TestCase): self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace')) -class UtilsPythonTestCase(unittest.TestCase): +class MemoizedMethodTest(unittest.TestCase): def test_memoizemethod_noargs(self): class A(object): @@ -62,19 +63,23 @@ class UtilsPythonTestCase(unittest.TestCase): assert one is two assert one is not three + +class IsBinaryTextTest(unittest.TestCase): def test_isbinarytext(self): + assert not isbinarytext(b"hello") - # basic tests - assert not isbinarytext("hello") - - # utf-16 strings contain null bytes + def utf_16_strings_contain_null_bytes(self): assert not isbinarytext(u"hello".encode('utf-16')) - # one with encoding - assert not isbinarytext("
Price \xa3
") + def test_one_with_encoding(self): + assert not isbinarytext(b"
Price \xa3
") - # finally some real binary bytes - assert isbinarytext("\x02\xa3") + def test_finally_some_real_binary_bytes(self): + assert isbinarytext(b"\x02\xa3") + + + +class UtilsPythonTestCase(unittest.TestCase): def test_equal_attributes(self): class Obj: @@ -134,29 +139,32 @@ class UtilsPythonTestCase(unittest.TestCase): del k self.assertFalse(len(wk._weakdict)) + @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict(self): - d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} + d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, unicode) for x in d2.keys())) - self.failIf(any(isinstance(x, unicode) for x in d2.values())) + self.failIf(d is d2) # shouldn't modify in place + self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) + self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_tuples(self): tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')] d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys()) - self.failIf(any(isinstance(x, unicode) for x in d2.values())) + self.failIf(d is d2) # shouldn't modify in place + self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) + self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_keys_only(self): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, unicode) for x in d2.keys())) + self.failIf(d is d2) # shouldn't modify in place + self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): def f1(a, b, c): @@ -194,7 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(object), []) # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(unicode.split), []) + self.assertEqual(get_func_args(six.text_type.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), [])