From 6d06824488f6d84b5efcd420f215eb12daf8a8de Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Fri, 30 Jul 2010 15:53:24 -0300 Subject: [PATCH] utils: Add walk_packages utility function --HG-- rename : scrapy/tests/test_utils_misc.py => scrapy/tests/test_utils_misc/__init__.py --- scrapy/tests/test_utils_misc.py | 29 ---------- scrapy/tests/test_utils_misc/__init__.py | 54 +++++++++++++++++++ .../test_walk_modules/__init__.py | 0 .../test_walk_modules/mod/__init__.py | 0 .../test_walk_modules/mod/mod0.py | 0 .../test_utils_misc/test_walk_modules/mod1.py | 0 scrapy/utils/misc.py | 13 +++++ 7 files changed, 67 insertions(+), 29 deletions(-) delete mode 100644 scrapy/tests/test_utils_misc.py create mode 100644 scrapy/tests/test_utils_misc/__init__.py create mode 100644 scrapy/tests/test_utils_misc/test_walk_modules/__init__.py create mode 100644 scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py create mode 100644 scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py create mode 100644 scrapy/tests/test_utils_misc/test_walk_modules/mod1.py diff --git a/scrapy/tests/test_utils_misc.py b/scrapy/tests/test_utils_misc.py deleted file mode 100644 index 806225c7c..000000000 --- a/scrapy/tests/test_utils_misc.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest -from cStringIO import StringIO - -from scrapy.utils.misc import load_object, arg_to_iter - -class UtilsMiscTestCase(unittest.TestCase): - - def test_load_object(self): - obj = load_object('scrapy.utils.misc.load_object') - assert obj is load_object - self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') - self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') - - def test_arg_to_iter(self): - assert hasattr(arg_to_iter(None), '__iter__') - assert hasattr(arg_to_iter(100), '__iter__') - assert hasattr(arg_to_iter('lala'), '__iter__') - assert hasattr(arg_to_iter([1,2,3]), '__iter__') - assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') - - self.assertEqual(list(arg_to_iter(None)), []) - self.assertEqual(list(arg_to_iter('lala')), ['lala']) - self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) - self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) - self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/tests/test_utils_misc/__init__.py b/scrapy/tests/test_utils_misc/__init__.py new file mode 100644 index 000000000..60a95d5bf --- /dev/null +++ b/scrapy/tests/test_utils_misc/__init__.py @@ -0,0 +1,54 @@ +import unittest +from cStringIO import StringIO + +from scrapy.utils.misc import load_object, arg_to_iter, walk_modules + +class UtilsMiscTestCase(unittest.TestCase): + + def test_load_object(self): + obj = load_object('scrapy.utils.misc.load_object') + assert obj is load_object + self.assertRaises(ImportError, load_object, 'nomodule999.mod.function') + self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999') + + def test_walk_modules(self): + mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules') + expected = [ + 'scrapy.tests.test_utils_misc.test_walk_modules', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod1', + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + + mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod') + expected = [ + 'scrapy.tests.test_utils_misc.test_walk_modules.mod', + 'scrapy.tests.test_utils_misc.test_walk_modules.mod.mod0', + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + + mods = walk_modules('scrapy.tests.test_utils_misc.test_walk_modules.mod1') + expected = [ + 'scrapy.tests.test_utils_misc.test_walk_modules.mod1', + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + + self.assertRaises(ImportError, list, walk_modules('nomodule999')) + + def test_arg_to_iter(self): + assert hasattr(arg_to_iter(None), '__iter__') + assert hasattr(arg_to_iter(100), '__iter__') + assert hasattr(arg_to_iter('lala'), '__iter__') + assert hasattr(arg_to_iter([1,2,3]), '__iter__') + assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') + + self.assertEqual(list(arg_to_iter(None)), []) + self.assertEqual(list(arg_to_iter('lala')), ['lala']) + self.assertEqual(list(arg_to_iter(100)), [100]) + self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter([1,2,3])), [1,2,3]) + self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/__init__.py b/scrapy/tests/test_utils_misc/test_walk_modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py b/scrapy/tests/test_utils_misc/test_walk_modules/mod/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py b/scrapy/tests/test_utils_misc/test_walk_modules/mod/mod0.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/tests/test_utils_misc/test_walk_modules/mod1.py b/scrapy/tests/test_utils_misc/test_walk_modules/mod1.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ac24591c2..0f9473441 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -2,6 +2,7 @@ import re import hashlib +from pkgutil import walk_packages from scrapy.utils.python import flatten from scrapy.utils.markup import remove_entities @@ -44,6 +45,18 @@ def load_object(path): return obj +def walk_modules(path): + """Loads a module and all its submodules given its absolute path and + returns them. + + path ie: 'scrapy.contrib.downloadermiddelware.redirect' + """ + mod = __import__(path, {}, {}, ['']) + if hasattr(mod, '__path__'): + for _, path, _ in walk_packages(mod.__path__, mod.__name__ + '.'): + yield __import__(path, {}, {}, ['']) + yield mod + def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: