diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a16d42eae..e72a5d042 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,15 +1,19 @@ """Helper functions which doesn't fit anywhere else""" - import re import hashlib - from importlib import import_module from pkgutil import iter_modules +import six from w3lib.html import remove_entities + from scrapy.utils.python import flatten from scrapy.item import BaseItem + +_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes + + def arg_to_iter(arg): """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -18,11 +22,12 @@ def arg_to_iter(arg): """ if arg is None: return [] - elif not isinstance(arg, (dict, BaseItem)) and hasattr(arg, '__iter__'): + elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'): return arg else: return [arg] + def load_object(path): """Load an object given its absolute object path, and return it. @@ -48,6 +53,7 @@ def load_object(path): return obj + def walk_modules(path, load=False): """Loads a module and all its submodules from a the given module path and returns them. If *any* module throws an exception while importing, that @@ -69,6 +75,7 @@ def walk_modules(path, load=False): mods.append(submod) return mods + def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: @@ -91,12 +98,13 @@ def extract_regex(regex, text, encoding='utf-8'): else: return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings] + def md5sum(file): """Calculate the md5 checksum of a file-like object without reading its whole content in memory. - >>> from StringIO import StringIO - >>> md5sum(StringIO('file content to hash')) + >>> from io import BytesIO + >>> md5sum(BytesIO(b'file content to hash')) '784406af91dd5a54fbb9c84c2236595a' """ m = hashlib.md5() diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 6340ad683..685dbc17d 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -57,7 +57,6 @@ tests/test_toplevel.py tests/test_utils_defer.py tests/test_utils_iterators.py tests/test_utils_jsonrpc.py -tests/test_utils_misc/__init__.py tests/test_utils_python.py tests/test_utils_reqser.py tests/test_utils_request.py