From 65c3de8e6acd0a29e6fcfeab600a43d4fdb4d65e Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 30 Jul 2010 17:05:55 -0300 Subject: [PATCH] Rewritten walk_modules function to support eggs, and added tests --- scrapy/tests/test_utils_misc/__init__.py | 19 ++++++++++++++++- scrapy/tests/test_utils_misc/test.egg | Bin 0 -> 2231 bytes scrapy/utils/misc.py | 25 +++++++++++++++-------- 3 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 scrapy/tests/test_utils_misc/test.egg diff --git a/scrapy/tests/test_utils_misc/__init__.py b/scrapy/tests/test_utils_misc/__init__.py index 60a95d5bf..5c8a1d7d9 100644 --- a/scrapy/tests/test_utils_misc/__init__.py +++ b/scrapy/tests/test_utils_misc/__init__.py @@ -1,3 +1,5 @@ +import sys +import os import unittest from cStringIO import StringIO @@ -34,7 +36,22 @@ class UtilsMiscTestCase(unittest.TestCase): ] self.assertEquals(set([m.__name__ for m in mods]), set(expected)) - self.assertRaises(ImportError, list, walk_modules('nomodule999')) + self.assertRaises(ImportError, walk_modules, 'nomodule999') + + def test_walk_modules_egg(self): + egg = os.path.join(os.path.dirname(__file__), 'test.egg') + sys.path.append(egg) + try: + mods = walk_modules('testegg') + expected = [ + 'testegg.spiders', + 'testegg.spiders.a', + 'testegg.spiders.b', + 'testegg' + ] + self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + finally: + sys.path.remove(egg) def test_arg_to_iter(self): assert hasattr(arg_to_iter(None), '__iter__') diff --git a/scrapy/tests/test_utils_misc/test.egg b/scrapy/tests/test_utils_misc/test.egg new file mode 100644 index 0000000000000000000000000000000000000000..238517694b306bb2daa59a6734d26f14f0a55c98 GIT binary patch literal 2231 zcmWIWW@Zs#U|`^22&(^QGnwtW5i^j-2*iRw?CS2W>*?p_uV0l}pj(`nmim~9Apk{( zeE$yJ9H7d3KnyYmt|Qn#G|1UCSg)j_MB7hSQ}=@QiYHp0x_X|z{@45a&!0K1)7kGX zbeTEn?wLJXYW_0hm&f|so7elyO!$96A^m#7-br0TJ}0MKX}Yp!%9TA&PW_m3X3v|X zH>XyZlmC!?yikHx%H-Z_!@9(aG<&F^p)s&TtHWG0I?Xd>q_zq z;&W2VQgguWgE;Fsy5EpPLl#+gN@_uBUP@|Sa%Fr@W?ptN$PkQ>Nncy5R0!0y8HiEB zGQb-WsGogxJ$*cNJg@M2>uQ}lbAEG>!4=~NpS;ibZVl3aI>~7wm)_a4zB;S61aW-* z{OvRU_s5@3ocFz|>*3??%3(QeTFMjgg`Z13Cpzg`tM%-Qp z$_}zH7YTV^`qW({sDu_aq7k>lIWc@BYy_ z{q^pe4gN;VA_hzwnFYi*GjxiuwI%RQ=$Oyu!Pl!{#vtUmbf%@|FHg;xM^&pVXI3s1 z-6blP_AyQE*UFbKf2poKs;XdRXJ@RxS;a1H{en#`%qvc+ncFWlQNPNUQet>Q1Iby} z>;BmQUBbWwB7i^`h_N_}8A;1CG%d(JDK5xNNi8bYPXc-cSq~_s`&S32qydfZ0Af)j zYmoJTz3=Uyctd?ABbP+u4aSb{$OacKX6FrT9!LC{RodM|q?tOGTskvH_kAwtbV~K8Gd3;-x#BdJbDbz+q2_2xXAN&_kI`)B0F0Fm(HYSO!Zdqo=iQ zZ+w1=W*{N3<>lwkoE|A8zaHYX99WodS|pL8f)wVD(Yz*&-3RJ9@E%(EbLf9ZPADjBEn75(HsFGq7O6Vgj-O*vb=x0o#xa zKrSheO~6*VAWV3L!vqvxU@Hd@Ml=I!XqYcxjsTYm$gaSaV-ZHkuwXX=*#wXy(DN_C lga9liAm?Ueldxqmgh}7=n-t*9$_7%*3xxSVpBu4)cmNz0f*$|? literal 0 HcmV?d00001 diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 0f9473441..e61b7aeb4 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -2,7 +2,7 @@ import re import hashlib -from pkgutil import walk_packages +from pkgutil import iter_modules from scrapy.utils.python import flatten from scrapy.utils.markup import remove_entities @@ -45,17 +45,26 @@ def load_object(path): return obj -def walk_modules(path): - """Loads a module and all its submodules given its absolute path and - returns them. +def walk_modules(path, load=False): + """Loads a module and all its submodules from a the given module path and + returns them. If *any* module throws an exception while importing, that + exception is thrown back. - path ie: 'scrapy.contrib.downloadermiddelware.redirect' + For example: walk_modules('scrapy.utils') """ + + mods = [] mod = __import__(path, {}, {}, ['']) + mods.append(mod) if hasattr(mod, '__path__'): - for _, path, _ in walk_packages(mod.__path__, mod.__name__ + '.'): - yield __import__(path, {}, {}, ['']) - yield mod + for _, subpath, ispkg in iter_modules(mod.__path__): + fullpath = path + '.' + subpath + if ispkg: + mods += walk_modules(fullpath) + else: + submod = __import__(fullpath, {}, {}, ['']) + mods.append(submod) + return mods def extract_regex(regex, text, encoding='utf-8'): """Extract a list of unicode strings from the given text/encoding using the following policies: