Fixed bug with runspider command that appeared after the introduction of new spider manager in r2121. Also factored out common code shared by new spider manager and runspider command, with tests included.

scrapy/commands/runspider.py

@@ -1,18 +1,17 @@
 import sys
 import os
 
-from scrapy.xlib.pydispatch import dispatcher
-from scrapy.contrib.exporter import XmlItemExporter
+from scrapy import log
+from scrapy.utils.spider import iter_spider_classes
 from scrapy.command import ScrapyCommand
 from scrapy.core.manager import scrapymanager
-from scrapy import signals
 
 def _import_file(filepath):
     abspath = os.path.abspath(filepath)
     dirname, file = os.path.split(abspath)
     fname, fext = os.path.splitext(file)
     if fext != '.py':
-        raise ValueError("Only Python files supported: %s" % abspath)
+        raise ValueError("Not a Python source file: %s" % abspath)
     if dirname:
         sys.path = [dirname] + sys.path
     try:
@@ -22,7 +21,6 @@ def _import_file(filepath):
             sys.path.pop(0)
     return module
 
-
 class Command(ScrapyCommand):
 
     requires_project = False
@@ -34,28 +32,25 @@ class Command(ScrapyCommand):
         return "Run a spider"
 
     def long_desc(self):
-        return "Run the spider defined in the given file. The file must be a " \
-            "Python module which defines a SPIDER variable with a instance of " \
-            "the spider to run."
-
-    def add_options(self, parser):
-        super(Command, self).add_options(parser)
-        parser.add_option("--output", dest="output", metavar="FILE",
-            help="store scraped items to FILE in XML format")
+        return "Run the spider defined in the given file"
 
     def run(self, args, opts):
         if len(args) != 1:
             return False
-        if opts.output:
-            file = open(opts.output, 'w+b')
-            exporter = XmlItemExporter(file)
-            dispatcher.connect(exporter.export_item, signal=signals.item_passed)
-            exporter.start_exporting()
-        module = _import_file(args[0])
-
+        filename = args[0]
+        if not os.path.exists(filename):
+            log.msg("File not found: %s\n" % filename, log.ERROR)
+            return
+        try:
+            module = _import_file(filename)
+        except (ImportError, ValueError), e:
+            log.msg("Unable to load %r: %s\n" % (filename, e), log.ERROR)
+            return
+        spclasses = list(iter_spider_classes(module))
+        if not spclasses:
+            log.msg("No spider found in file: %s\n" % filename, log.ERROR)
+            return
+        spider = spclasses.pop()()
         # schedule spider and start engine
-        scrapymanager.queue.append_spider(module.SPIDER)
+        scrapymanager.queue.append_spider(spider)
         scrapymanager.start()
-
-        if opts.output:
-            exporter.finish_exporting()

scrapy/contrib/spidermanager.py

@@ -3,12 +3,10 @@ SpiderManager is the class which locates and manages all website-specific
 spiders
 """
 
-import inspect
-
 from scrapy import log
 from scrapy.conf import settings
 from scrapy.utils.misc import walk_modules
-from scrapy.spider import BaseSpider
+from scrapy.utils.spider import iter_spider_classes
 
 
 class SpiderManager(object):
@@ -68,11 +66,8 @@ class SpiderManager(object):
         self.loaded = True
 
     def _load_spiders(self, module):
-        for obj in vars(module).itervalues():
-            if inspect.isclass(obj) and issubclass(obj, BaseSpider):
-                name = getattr(obj, 'name', None)
-                if name is not None:
-                    self._spiders[name] = obj
+        for spcls in iter_spider_classes(module):
+            self._spiders[spcls.name] = spcls
 
     def close_spider(self, spider):
         pass

scrapy/tests/test_utils_spider.py

@@ -1,8 +1,19 @@
 import unittest
 from scrapy.http import Request
 from scrapy.item import BaseItem
-from scrapy.utils.spider import iterate_spider_output
+from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
 
+from scrapy.contrib.spiders import CrawlSpider
+
+
+class MyBaseSpider(CrawlSpider):
+    pass # abstract spider
+
+class MySpider1(MyBaseSpider):
+    name = 'myspider1'
+
+class MySpider2(MyBaseSpider):
+    name = 'myspider2'
 
 class UtilsSpidersTestCase(unittest.TestCase):
 
@@ -16,6 +27,10 @@ class UtilsSpidersTestCase(unittest.TestCase):
         self.assertEqual(list(iterate_spider_output(o)), [o])
         self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])
 
+    def test_iter_spider_classes(self):
+        import scrapy.tests.test_utils_spider
+        it = iter_spider_classes(scrapy.tests.test_utils_spider)
+        self.assertEqual(set(it), set([MySpider1, MySpider2]))
 
 if __name__ == "__main__":
     unittest.main()

scrapy/utils/spider.py

@@ -1,6 +1,23 @@
+import inspect
+
 from scrapy.item import BaseItem
 from scrapy.utils.misc import  arg_to_iter
 
 
 def iterate_spider_output(result):
     return [result] if isinstance(result, BaseItem) else arg_to_iter(result)
+
+def iter_spider_classes(module):
+    """Return an iterator over all spider classes defined in the given module
+    that can be instantiated (ie. which have name)
+    """
+    # this needs to be imported here until get rid of the spider manager
+    # singleton in scrapy.spider.spiders
+    from scrapy.spider import BaseSpider
+
+    for obj in vars(module).itervalues():
+        if inspect.isclass(obj) and \
+           issubclass(obj, BaseSpider) and \
+           obj.__module__ == module.__name__ and \
+           getattr(obj, 'name', None):
+            yield obj