diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5d5d5b004..f691a302f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -23,8 +23,12 @@ def flatten(x): >>> [1, 2, [3,4], (5,6)] [1, 2, [3, 4], (5, 6)] >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) - [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]""" - + [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10] + >>> flatten(["foo", "bar"]) + ['foo', 'bar'] + >>> flatten(["foo", ["baz", 42], "bar"]) + ['foo', 'baz', 42, 'bar'] + """ return list(iflatten(x)) @@ -32,15 +36,38 @@ def iflatten(x): """iflatten(sequence) -> iterator Similar to ``.flatten()``, but returns iterator instead""" - for el in x: - if hasattr(el, "__iter__"): + if is_listlike(el): for el_ in flatten(el): yield el_ else: yield el +def is_listlike(x): + """ + >>> is_listlike("foo") + False + >>> is_listlike(5) + False + >>> is_listlike(b"foo") + False + >>> is_listlike([b"foo"]) + True + >>> is_listlike((b"foo",)) + True + >>> is_listlike({}) + True + >>> is_listlike(set()) + True + >>> is_listlike((x for x in range(3))) + True + >>> is_listlike(six.moves.xrange(5)) + True + """ + return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) + + def unique(list_, key=lambda x: x): """efficient function to uniquify a list preserving item order""" seen = set()