diff --git a/scrapy/command.py b/scrapy/command.py index 62291a353..39bdb325d 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -10,6 +10,7 @@ from optparse import OptionGroup import scrapy from scrapy.conf import settings +from scrapy.utils.conf import arglist_to_dict class ScrapyCommand(object): @@ -74,7 +75,7 @@ class ScrapyCommand(object): help="write lsprof profiling stats to FILE") group.add_option("--pidfile", dest="pidfile", metavar="FILE", \ help="write process ID to FILE") - group.add_option("--set", dest="set", action="append", default=[], \ + group.add_option("--set", dest="set", action="append", default=[], metavar="NAME=VALUE", \ help="set/override setting (may be repeated)") group.add_option("--settings", dest="settings", metavar="MODULE", help="python path to the Scrapy project settings") @@ -84,14 +85,11 @@ class ScrapyCommand(object): if opts.settings: settings.set_settings_module(opts.settings) - for setting in opts.set: - if '=' in setting: - name, val = setting.split('=', 1) - settings.overrides[name] = val - else: - sys.stderr.write("%s: invalid argument --set %s - proper format " \ - "is --set SETTING=VALUE'\n" % (sys.argv[0], setting)) - sys.exit(2) + try: + settings.overrides.update(arglist_to_dict(opts.set)) + except ValueError: + sys.stderr.write("Invalid --set value, use --set NAME=VALUE\n") + sys.exit(2) if opts.version: print "Scrapy %s" % scrapy.__version__ diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index db6009206..ea2855788 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,11 @@ +import sys + from scrapy import log from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.http import Request from scrapy.utils.url import is_url +from scrapy.utils.conf import arglist_to_dict from collections import defaultdict @@ -20,11 +23,18 @@ class Command(ScrapyCommand): ScrapyCommand.add_options(self, parser) parser.add_option("--spider", dest="spider", default=None, \ help="always use this spider when arguments are urls") + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ + help="set spider argument (may be repeated)") parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", \ help="don't follow links (for use with URLs only)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + sys.stderr.write("Invalid --arg value, use --arg NAME=VALUE\n") + sys.exit(2) if opts.nofollow: settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False @@ -32,18 +42,18 @@ class Command(ScrapyCommand): q = self.crawler.queue urls, names = self._split_urls_and_names(args) for name in names: - q.append_spider_name(name) + q.append_spider_name(name, **opts.spargs) if opts.spider: try: - spider = self.crawler.spiders.create(opts.spider) + spider = self.crawler.spiders.create(opts.spider, **opts.spargs) for url in urls: q.append_url(url, spider) except KeyError: log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) else: for name, urls in self._group_urls_by_spider(urls): - spider = self.crawler.spiders.create(name) + spider = self.crawler.spiders.create(name, **opts.spargs) for url in urls: q.append_url(url, spider) self.crawler.start() diff --git a/scrapy/tests/test_utils_conf.py b/scrapy/tests/test_utils_conf.py index 44a940fe7..35277a423 100644 --- a/scrapy/tests/test_utils_conf.py +++ b/scrapy/tests/test_utils_conf.py @@ -1,6 +1,6 @@ import unittest -from scrapy.utils.conf import build_component_list +from scrapy.utils.conf import build_component_list, arglist_to_dict class UtilsConfTestCase(unittest.TestCase): @@ -13,5 +13,10 @@ class UtilsConfTestCase(unittest.TestCase): custom = ['a', 'b', 'c'] self.assertEqual(build_component_list(base, custom), custom) + def test_arglist_to_dict(self): + self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']), + {'arg1': 'val1', 'arg2': 'val2'}) + + if __name__ == "__main__": unittest.main() diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index afba9d88b..0e21fa798 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -15,6 +15,11 @@ def build_component_list(base, custom): return [k for k, v in sorted(compdict.items(), key=itemgetter(1)) \ if v is not None] +def arglist_to_dict(arglist): + """Convert a list of arguments like ['arg1=val1', 'arg2=val2', ...] to a + dict""" + return dict(x.split('=', 1) for x in arglist) + def closest_scrapy_cfg(path='.', prevpath=None): if path == prevpath: return ''