Support passing spider arguments in crawl command with -a option. Closes #216

scrapy/command.py

@@ -10,6 +10,7 @@ from optparse import OptionGroup
 
 import scrapy
 from scrapy.conf import settings
+from scrapy.utils.conf import arglist_to_dict
 
 class ScrapyCommand(object):
 
@@ -74,7 +75,7 @@ class ScrapyCommand(object):
             help="write lsprof profiling stats to FILE")
         group.add_option("--pidfile", dest="pidfile", metavar="FILE", \
             help="write process ID to FILE")
-        group.add_option("--set", dest="set", action="append", default=[], \
+        group.add_option("--set", dest="set", action="append", default=[], metavar="NAME=VALUE", \
             help="set/override setting (may be repeated)")
         group.add_option("--settings", dest="settings", metavar="MODULE",
             help="python path to the Scrapy project settings")
@@ -84,14 +85,11 @@ class ScrapyCommand(object):
         if opts.settings:
             settings.set_settings_module(opts.settings)
 
-        for setting in opts.set:
-            if '=' in setting:
-                name, val = setting.split('=', 1)
-                settings.overrides[name] = val
-            else:
-                sys.stderr.write("%s: invalid argument --set %s - proper format " \
-                    "is --set SETTING=VALUE'\n" % (sys.argv[0], setting))
-                sys.exit(2)
+        try:
+            settings.overrides.update(arglist_to_dict(opts.set))
+        except ValueError:
+            sys.stderr.write("Invalid --set value, use --set NAME=VALUE\n")
+            sys.exit(2)
 
         if opts.version:
             print "Scrapy %s" % scrapy.__version__

scrapy/commands/crawl.py

@@ -1,8 +1,11 @@
+import sys
+
 from scrapy import log
 from scrapy.command import ScrapyCommand
 from scrapy.conf import settings
 from scrapy.http import Request
 from scrapy.utils.url import is_url
+from scrapy.utils.conf import arglist_to_dict
 
 from collections import defaultdict
 
@@ -20,11 +23,18 @@ class Command(ScrapyCommand):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--spider", dest="spider", default=None, \
             help="always use this spider when arguments are urls")
+        parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \
+            help="set spider argument (may be repeated)")
         parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \
             help="don't follow links (for use with URLs only)")
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
+        try:
+            opts.spargs = arglist_to_dict(opts.spargs)
+        except ValueError:
+            sys.stderr.write("Invalid --arg value, use --arg NAME=VALUE\n")
+            sys.exit(2)
         if opts.nofollow:
             settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
 
@@ -32,18 +42,18 @@ class Command(ScrapyCommand):
         q = self.crawler.queue
         urls, names = self._split_urls_and_names(args)
         for name in names:
-            q.append_spider_name(name)
+            q.append_spider_name(name, **opts.spargs)
 
         if opts.spider:
             try:
-                spider = self.crawler.spiders.create(opts.spider)
+                spider = self.crawler.spiders.create(opts.spider, **opts.spargs)
                 for url in urls:
                     q.append_url(url, spider)
             except KeyError:
                 log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
         else:
             for name, urls in self._group_urls_by_spider(urls):
-                spider = self.crawler.spiders.create(name)
+                spider = self.crawler.spiders.create(name, **opts.spargs)
                 for url in urls:
                     q.append_url(url, spider)
         self.crawler.start()

scrapy/tests/test_utils_conf.py

@@ -1,6 +1,6 @@
 import unittest
 
-from scrapy.utils.conf import build_component_list
+from scrapy.utils.conf import build_component_list, arglist_to_dict
 
 class UtilsConfTestCase(unittest.TestCase):
 
@@ -13,5 +13,10 @@ class UtilsConfTestCase(unittest.TestCase):
         custom = ['a', 'b', 'c']
         self.assertEqual(build_component_list(base, custom), custom)
 
+    def test_arglist_to_dict(self):
+        self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
+            {'arg1': 'val1', 'arg2': 'val2'})
+
+
 if __name__ == "__main__":
     unittest.main()

scrapy/utils/conf.py

@@ -15,6 +15,11 @@ def build_component_list(base, custom):
     return [k for k, v in sorted(compdict.items(), key=itemgetter(1)) \
         if v is not None]
 
+def arglist_to_dict(arglist):
+    """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a
+    dict"""
+    return dict(x.split('=', 1) for x in arglist)
+
 def closest_scrapy_cfg(path='.', prevpath=None):
     if path == prevpath:
         return ''