From 887936ebf6d39a977a3558e249fdb875fc3f21ce Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 23 Jul 2015 17:26:20 +0200 Subject: [PATCH 1/4] PY3 port flatten and iflatten --- scrapy/utils/python.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5d5d5b004..f691a302f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -23,8 +23,12 @@ def flatten(x): >>> [1, 2, [3,4], (5,6)] [1, 2, [3, 4], (5, 6)] >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) - [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]""" - + [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10] + >>> flatten(["foo", "bar"]) + ['foo', 'bar'] + >>> flatten(["foo", ["baz", 42], "bar"]) + ['foo', 'baz', 42, 'bar'] + """ return list(iflatten(x)) @@ -32,15 +36,38 @@ def iflatten(x): """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" - for el in x: - if hasattr(el, "__iter__"): + if is_listlike(el): for el_ in flatten(el): yield el_ else: yield el +def is_listlike(x): + """ + >>> is_listlike("foo") + False + >>> is_listlike(5) + False + >>> is_listlike(b"foo") + False + >>> is_listlike([b"foo"]) + True + >>> is_listlike((b"foo",)) + True + >>> is_listlike({}) + True + >>> is_listlike(set()) + True + >>> is_listlike((x for x in range(3))) + True + >>> is_listlike(six.moves.xrange(5)) + True + """ + return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) + + def unique(list_, key=lambda x: x): """efficient function to uniquify a list preserving item order""" seen = set() From 41bcae5d00a3207fa15c7e2150a92cd103c7552b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 23 Jul 2015 17:35:23 +0200 Subject: [PATCH 2/4] TST fix to_bytes and to_unicode tests in Python 3.x --- tests/test_utils_python.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index fa77356de..a5f183e6e 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -12,10 +12,10 @@ __doctests__ = ['scrapy.utils.python'] class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): - self.assertEqual(to_unicode('lel\xc3\xb1e'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') def test_converting_a_latin_1_encoded_string_to_unicode(self): - self.assertEqual(to_unicode('lel\xf1e', 'latin-1'), u'lel\xf1e') + self.assertEqual(to_unicode(b'lel\xf1e', 'latin-1'), u'lel\xf1e') def test_converting_a_unicode_to_unicode_should_return_the_same_object(self): self.assertEqual(to_unicode(u'\xf1e\xf1e\xf1e'), u'\xf1e\xf1e\xf1e') @@ -24,24 +24,24 @@ class ToUnicodeTest(unittest.TestCase): self.assertRaises(TypeError, to_unicode, 423) def test_check_errors_argument_works(self): - self.assertIn(u'\ufffd', to_unicode('a\xedb', 'utf-8', errors='replace')) + self.assertIn(u'\ufffd', to_unicode(b'a\xedb', 'utf-8', errors='replace')) class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_an_utf_8_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49'), '\xc2\xa3 49') + self.assertEqual(to_bytes(u'\xa3 49'), b'\xc2\xa3 49') def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): - self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), '\xa3 49') + self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49') def test_converting_a_regular_string_to_string_should_return_the_same_object(self): - self.assertEqual(to_bytes('lel\xf1e'), 'lel\xf1e') + self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_bytes, unittest) def test_check_errors_argument_works(self): - self.assertIn('?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace')) + self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace')) class UtilsPythonTestCase(unittest.TestCase): From 407349865279795387b9ff84afb9011e2303bb9d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 23 Jul 2015 18:33:56 +0200 Subject: [PATCH 3/4] port utils.python * stringify_dict is deprecated * is_writable is deprecated * setattr_default is deprecated * get_spec is untested * re_rsearch is untested * retry_on_eintr is untested --- scrapy/utils/python.py | 24 +++++++++++++------- tests/test_utils_python.py | 46 ++++++++++++++++++++++---------------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index f691a302f..57016811f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -142,11 +142,14 @@ def re_rsearch(pattern, text, chunk_size=1024): yield (text[offset:], offset) yield (text, 0) - pattern = re.compile(pattern) if isinstance(pattern, basestring) else pattern + if isinstance(pattern, six.string_types): + pattern = re.compile(pattern) + for chunk, offset in _chunk_iter(): matches = [match for match in pattern.finditer(chunk)] if matches: - return (offset + matches[-1].span()[0], offset + matches[-1].span()[1]) + start, end = matches[-1].span() + return offset + start, offset + end return None @@ -162,14 +165,16 @@ def memoizemethod_noargs(method): return cache[self] return new_method -_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"]) +_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} +_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} def isbinarytext(text): - """Return True if the given text is considered binary, or false + """Return True if the given text is considered binary, or False otherwise, by looking for binary bytes at their chars """ - assert isinstance(text, str), "text must be str, got '%s'" % type(text).__name__ + if not isinstance(text, bytes): + raise TypeError("text must be bytes, got '%s'" % type(text).__name__) return any(c in _BINARYCHARS for c in text) @@ -273,20 +278,22 @@ class WeakKeyCache(object): return self._weakdict[key] +@deprecated def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): - """Return a (new) dict with the unicode keys (and values if, keys_only is + """Return a (new) dict with unicode keys (and values when "keys_only" is False) of the given dict converted to strings. `dct_or_tuples` can be a dict or a list of tuples, like any dict constructor supports. """ d = {} for k, v in six.iteritems(dict(dct_or_tuples)): - k = k.encode(encoding) if isinstance(k, unicode) else k + k = k.encode(encoding) if isinstance(k, six.text_type) else k if not keys_only: - v = v.encode(encoding) if isinstance(v, unicode) else v + v = v.encode(encoding) if isinstance(v, six.text_type) else v d[k] = v return d +@deprecated def is_writable(path): """Return True if the given path can be written (if it exists) or created (if it doesn't exist) @@ -297,6 +304,7 @@ def is_writable(path): return os.access(os.path.dirname(path), os.W_OK) +@deprecated def setattr_default(obj, name, value): """Set attribute value, but only if it's not already set. Similar to setdefault() for dicts. diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index a5f183e6e..3b99fec5b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -2,6 +2,7 @@ import functools import operator import unittest from itertools import count +import six from scrapy.utils.python import ( memoizemethod_noargs, isbinarytext, equal_attributes, @@ -44,7 +45,7 @@ class ToBytesTest(unittest.TestCase): self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace')) -class UtilsPythonTestCase(unittest.TestCase): +class MemoizedMethodTest(unittest.TestCase): def test_memoizemethod_noargs(self): class A(object): @@ -62,19 +63,23 @@ class UtilsPythonTestCase(unittest.TestCase): assert one is two assert one is not three + +class IsBinaryTextTest(unittest.TestCase): def test_isbinarytext(self): + assert not isbinarytext(b"hello") - # basic tests - assert not isbinarytext("hello") - - # utf-16 strings contain null bytes + def utf_16_strings_contain_null_bytes(self): assert not isbinarytext(u"hello".encode('utf-16')) - # one with encoding - assert not isbinarytext("
Price \xa3
") + def test_one_with_encoding(self): + assert not isbinarytext(b"
Price \xa3
") - # finally some real binary bytes - assert isbinarytext("\x02\xa3") + def test_finally_some_real_binary_bytes(self): + assert isbinarytext(b"\x02\xa3") + + + +class UtilsPythonTestCase(unittest.TestCase): def test_equal_attributes(self): class Obj: @@ -134,29 +139,32 @@ class UtilsPythonTestCase(unittest.TestCase): del k self.assertFalse(len(wk._weakdict)) + @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict(self): - d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} + d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, unicode) for x in d2.keys())) - self.failIf(any(isinstance(x, unicode) for x in d2.values())) + self.failIf(d is d2) # shouldn't modify in place + self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) + self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_tuples(self): tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')] d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, unicode) for x in d2.keys()), d2.keys()) - self.failIf(any(isinstance(x, unicode) for x in d2.values())) + self.failIf(d is d2) # shouldn't modify in place + self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) + self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_keys_only(self): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, unicode) for x in d2.keys())) + self.failIf(d is d2) # shouldn't modify in place + self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): def f1(a, b, c): @@ -194,7 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(object), []) # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(unicode.split), []) + self.assertEqual(get_func_args(six.text_type.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) From a7b4a3e7f95fed8f1d4b9bddf79f7e9a91555fdb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 23 Jul 2015 20:03:33 +0200 Subject: [PATCH 4/4] cleanup * run test_utils_python in Python 3; * make tests for 'errors' argument more explicit * add missing test_ prefix utf_16_strings_contain_null_bytes; * cleanup test names. --- tests/py3-ignores.txt | 1 - tests/test_utils_python.py | 20 +++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 842217dd0..aa091835c 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -59,7 +59,6 @@ tests/test_utils_defer.py tests/test_utils_iterators.py tests/test_utils_jsonrpc.py tests/test_utils_log.py -tests/test_utils_python.py tests/test_utils_reqser.py tests/test_utils_request.py tests/test_utils_response.py diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3b99fec5b..ca394ebf5 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -24,8 +24,11 @@ class ToUnicodeTest(unittest.TestCase): def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_unicode, 423) - def test_check_errors_argument_works(self): - self.assertIn(u'\ufffd', to_unicode(b'a\xedb', 'utf-8', errors='replace')) + def test_errors_argument(self): + self.assertEqual( + to_unicode(b'a\xedb', 'utf-8', errors='replace'), + u'a\ufffdb' + ) class ToBytesTest(unittest.TestCase): @@ -35,14 +38,17 @@ class ToBytesTest(unittest.TestCase): def test_converting_a_unicode_object_to_a_latin_1_encoded_string(self): self.assertEqual(to_bytes(u'\xa3 49', 'latin-1'), b'\xa3 49') - def test_converting_a_regular_string_to_string_should_return_the_same_object(self): + def test_converting_a_regular_bytes_to_bytes_should_return_the_same_object(self): self.assertEqual(to_bytes(b'lel\xf1e'), b'lel\xf1e') def test_converting_a_strange_object_should_raise_TypeError(self): self.assertRaises(TypeError, to_bytes, unittest) - def test_check_errors_argument_works(self): - self.assertIn(b'?', to_bytes(u'a\ufffdb', 'latin-1', errors='replace')) + def test_errors_argument(self): + self.assertEqual( + to_bytes(u'a\ufffdb', 'latin-1', errors='replace'), + b'a?b' + ) class MemoizedMethodTest(unittest.TestCase): @@ -68,13 +74,13 @@ class IsBinaryTextTest(unittest.TestCase): def test_isbinarytext(self): assert not isbinarytext(b"hello") - def utf_16_strings_contain_null_bytes(self): + def test_utf_16_strings_contain_null_bytes(self): assert not isbinarytext(u"hello".encode('utf-16')) def test_one_with_encoding(self): assert not isbinarytext(b"
Price \xa3
") - def test_finally_some_real_binary_bytes(self): + def test_real_binary_bytes(self): assert isbinarytext(b"\x02\xa3")