mirror of https://github.com/scrapy/scrapy.git
Merge pull request #1379 from scrapy/py3-utils-python
PY3 port scrapy.utils.python
This commit is contained in:
Mikhail Korobov
commit
335aa5f276
|
|
@ -23,8 +23,12 @@ def flatten(x):
|
|||
>>> [1, 2, [3,4], (5,6)]
|
||||
[1, 2, [3, 4], (5, 6)]
|
||||
>>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
|
||||
[1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]"""
|
||||
|
||||
[1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
|
||||
>>> flatten(["foo", "bar"])
|
||||
['foo', 'bar']
|
||||
>>> flatten(["foo", ["baz", 42], "bar"])
|
||||
['foo', 'baz', 42, 'bar']
|
||||
"""
|
||||
return list(iflatten(x))
|
||||
|
||||
|
||||
|
|
@ -32,15 +36,38 @@ def iflatten(x):
|
|||
"""iflatten(sequence) -> iterator
|
||||
|
||||
Similar to ``.flatten()``, but returns iterator instead"""
|
||||
|
||||
for el in x:
|
||||
if hasattr(el, "__iter__"):
|
||||
if is_listlike(el):
|
||||
for el_ in flatten(el):
|
||||
yield el_
|
||||
else:
|
||||
yield el
|
||||
|
||||
|
||||
def is_listlike(x):
|
||||
"""
|
||||
>>> is_listlike("foo")
|
||||
False
|
||||
>>> is_listlike(5)
|
||||
False
|
||||
>>> is_listlike(b"foo")
|
||||
False
|
||||
>>> is_listlike([b"foo"])
|
||||
True
|
||||
>>> is_listlike((b"foo",))
|
||||
True
|
||||
>>> is_listlike({})
|
||||
True
|
||||
>>> is_listlike(set())
|
||||
True
|
||||
>>> is_listlike((x for x in range(3)))
|
||||
True
|
||||
>>> is_listlike(six.moves.xrange(5))
|
||||
True
|
||||
"""
|
||||
return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
|
||||
|
||||
|
||||
def unique(list_, key=lambda x: x):
|
||||
"""efficient function to uniquify a list preserving item order"""
|
||||
seen = set()
|
||||
|
|
@ -115,11 +142,14 @@ def re_rsearch(pattern, text, chunk_size=1024):
|
|||
yield (text[offset:], offset)
|
||||
yield (text, 0)
|
||||
|
||||
pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern
|
||||
if isinstance(pattern, six.string_types):
|
||||
pattern = re.compile(pattern)
|
||||
|
||||
for chunk, offset in _chunk_iter():
|
||||
matches = [match for match in pattern.finditer(chunk)]
|
||||
if matches:
|
||||
return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
|
||||
start, end = matches[-1].span()
|
||||
return offset + start, offset + end
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -135,14 +165,16 @@ def memoizemethod_noargs(method):
|
|||
return cache[self]
|
||||
return new_method
|
||||
|
||||
_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])
|
||||
_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
|
||||
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
|
||||
|
||||
|
||||
def isbinarytext(text):
|
||||
"""Return True if the given text is considered binary, or false
|
||||
"""Return True if the given text is considered binary, or False
|
||||
otherwise, by looking for binary bytes at their chars
|
||||
"""
|
||||
assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__
|
||||
if not isinstance(text, bytes):
|
||||
raise TypeError("text must be bytes, got '%s'" % type(text).__name__)
|
||||
return any(c in _BINARYCHARS for c in text)
|
||||
|
||||
|
||||
|
|
@ -246,20 +278,22 @@ class WeakKeyCache(object):
|
|||
return self._weakdict[key]
|
||||
|
||||
|
||||
@deprecated
|
||||
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
|
||||
"""Return a (new) dict with the unicode keys (and values if, keys_only is
|
||||
"""Return a (new) dict with unicode keys (and values when "keys_only" is
|
||||
False) of the given dict converted to strings. `dct_or_tuples` can be a
|
||||
dict or a list of tuples, like any dict constructor supports.
|
||||
"""
|
||||
d = {}
|
||||
for k, v in six.iteritems(dict(dct_or_tuples)):
|
||||
k = k.encode(encoding) if isinstance(k, unicode) else k
|
||||
k = k.encode(encoding) if isinstance(k, six.text_type) else k
|
||||
if not keys_only:
|
||||
v = v.encode(encoding) if isinstance(v, unicode) else v
|
||||
v = v.encode(encoding) if isinstance(v, six.text_type) else v
|
||||
d[k] = v
|
||||
return d
|
||||
|
||||
|
||||
@deprecated
|
||||
def is_writable(path):
|
||||
"""Return True if the given path can be written (if it exists) or created
|
||||
(if it doesn't exist)
|
||||
|
|
@ -270,6 +304,7 @@ def is_writable(path):
|
|||
return os.access(os.path.dirname(path), os.W_OK)
|
||||
|
||||
|
||||
@deprecated
|
||||
def setattr_default(obj, name, value):
|
||||
"""Set attribute value, but only if it's not already set. Similar to
|
||||
setdefault() for dicts.
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ tests/test_utils_defer.py
|
|||
tests/test_utils_iterators.py
|
||||
tests/test_utils_jsonrpc.py
|
||||
tests/test_utils_log.py
|
||||
tests/test_utils_python.py
|
||||
tests/test_utils_reqser.py
|
||||
tests/test_utils_request.py
|
||||
tests/test_utils_response.py
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import functools
|
|||
import operator
|
||||
import unittest
|
||||
from itertools import count
|
||||
import six
|
||||
|
||||
from scrapy.utils.python import (
|
||||
memoizemethod_noargs, isbinarytext, equal_attributes,
|
||||
|
|
@ -12,10 +13,10 @@ __doctests__ = ['scrapy.utils.python']
|
|||
|
||||
class ToUnicodeTest(unittest.TestCase):
|
||||
def test_converting_an_utf8_encoded_string_to_unicode(self):
|
||||
self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e')
|
||||
self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e')
|
||||
|
||||
def test_converting_a_latin_1_encoded_string_to_unicode(self):
|
||||
self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e')
|
||||
self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e')
|
||||
|
||||
def test_converting_a_unicode_to_unicode_should_return_the_same_object(self):
|
||||
self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e')
|
||||
|
|
@ -23,28 +24,34 @@ class ToUnicodeTest(unittest.TestCase):
|
|||
def test_converting_a_strange_object_should_raise_TypeError(self):
|
||||
self.assertRaises(TypeError, to_unicode, 423)
|
||||
|
||||
def test_check_errors_argument_works(self):
|
||||
self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace'))
|
||||
def test_errors_argument(self):
|
||||
self.assertEqual(
|
||||
to_unicode(b'a\xedb', 'utf-8', errors='replace'),
|
||||
u'a\ufffdb'
|
||||
)
|
||||
|
||||
|
||||
class ToBytesTest(unittest.TestCase):
|
||||
def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self):
|
||||
self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49')
|
||||
self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49')
|
||||
|
||||
def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self):
|
||||
self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49')
|
||||
self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49')
|
||||
|
||||
def test_converting_a_regular_string_to_string_should_return_the_same_object(self):
|
||||
self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e')
|
||||
def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self):
|
||||
self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e')
|
||||
|
||||
def test_converting_a_strange_object_should_raise_TypeError(self):
|
||||
self.assertRaises(TypeError, to_bytes, unittest)
|
||||
|
||||
def test_check_errors_argument_works(self):
|
||||
self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace'))
|
||||
def test_errors_argument(self):
|
||||
self.assertEqual(
|
||||
to_bytes(u'a\ufffdb', 'latin-1', errors='replace'),
|
||||
b'a?b'
|
||||
)
|
||||
|
||||
|
||||
class UtilsPythonTestCase(unittest.TestCase):
|
||||
class MemoizedMethodTest(unittest.TestCase):
|
||||
def test_memoizemethod_noargs(self):
|
||||
class A(object):
|
||||
|
||||
|
|
@ -62,19 +69,23 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
assert one is two
|
||||
assert one is not three
|
||||
|
||||
|
||||
class IsBinaryTextTest(unittest.TestCase):
|
||||
def test_isbinarytext(self):
|
||||
assert not isbinarytext(b"hello")
|
||||
|
||||
# basic tests
|
||||
assert not isbinarytext("hello")
|
||||
|
||||
# utf-16 strings contain null bytes
|
||||
def test_utf_16_strings_contain_null_bytes(self):
|
||||
assert not isbinarytext(u"hello".encode('utf-16'))
|
||||
|
||||
# one with encoding
|
||||
assert not isbinarytext("<div>Price \xa3</div>")
|
||||
def test_one_with_encoding(self):
|
||||
assert not isbinarytext(b"<div>Price \xa3</div>")
|
||||
|
||||
# finally some real binary bytes
|
||||
assert isbinarytext("\x02\xa3")
|
||||
def test_real_binary_bytes(self):
|
||||
assert isbinarytext(b"\x02\xa3")
|
||||
|
||||
|
||||
|
||||
class UtilsPythonTestCase(unittest.TestCase):
|
||||
|
||||
def test_equal_attributes(self):
|
||||
class Obj:
|
||||
|
|
@ -134,29 +145,32 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
del k
|
||||
self.assertFalse(len(wk._weakdict))
|
||||
|
||||
@unittest.skipUnless(six.PY2, "deprecated function")
|
||||
def test_stringify_dict(self):
|
||||
d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
|
||||
d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
|
||||
d2 = stringify_dict(d, keys_only=False)
|
||||
self.assertEqual(d, d2)
|
||||
self.failIf(d is d2) # shouldn't modify in place
|
||||
self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
|
||||
self.failIf(any(isinstance(x, unicode) for x in d2.values()))
|
||||
self.failIf(d is d2) # shouldn't modify in place
|
||||
self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
|
||||
self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
|
||||
|
||||
@unittest.skipUnless(six.PY2, "deprecated function")
|
||||
def test_stringify_dict_tuples(self):
|
||||
tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
|
||||
d = dict(tuples)
|
||||
d2 = stringify_dict(tuples, keys_only=False)
|
||||
self.assertEqual(d, d2)
|
||||
self.failIf(d is d2) # shouldn't modify in place
|
||||
self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys())
|
||||
self.failIf(any(isinstance(x, unicode) for x in d2.values()))
|
||||
self.failIf(d is d2) # shouldn't modify in place
|
||||
self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
|
||||
self.failIf(any(isinstance(x, six.text_type) for x in d2.values()))
|
||||
|
||||
@unittest.skipUnless(six.PY2, "deprecated function")
|
||||
def test_stringify_dict_keys_only(self):
|
||||
d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
|
||||
d2 = stringify_dict(d)
|
||||
self.assertEqual(d, d2)
|
||||
self.failIf(d is d2) # shouldn't modify in place
|
||||
self.failIf(any(isinstance(x, unicode) for x in d2.keys()))
|
||||
self.failIf(d is d2) # shouldn't modify in place
|
||||
self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()))
|
||||
|
||||
def test_get_func_args(self):
|
||||
def f1(a, b, c):
|
||||
|
|
@ -194,7 +208,7 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
self.assertEqual(get_func_args(object), [])
|
||||
|
||||
# TODO: how do we fix this to return the actual argument names?
|
||||
self.assertEqual(get_func_args(unicode.split), [])
|
||||
self.assertEqual(get_func_args(six.text_type.split), [])
|
||||
self.assertEqual(get_func_args(" ".join), [])
|
||||
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue