diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 97f818cb2..ab850dcb3 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -5,7 +5,7 @@ import os from optparse import OptionGroup from twisted.python import failure -from scrapy.utils.conf import arglist_to_dict +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError @@ -104,3 +104,27 @@ class ScrapyCommand: Entry point for running commands """ raise NotImplementedError + + +class BaseRunSpiderCommand(ScrapyCommand): + """ + Common class used to share functionality between the crawl and runspider commands + """ + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", action="append", + help="dump scraped items into FILE (use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b2f9484b..e1724c1e6 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,9 +1,8 @@ -from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -class Command(ScrapyCommand): +class Command(BaseRunSpiderCommand): requires_project = True @@ -13,25 +12,6 @@ class Command(ScrapyCommand): def short_desc(self): return "Run a spider" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) < 1: raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 62510609a..befee021b 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,9 +3,8 @@ import os from importlib import import_module from scrapy.utils.spider import iter_spider_classes -from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.commands import BaseRunSpiderCommand def _import_file(filepath): @@ -24,7 +23,7 @@ def _import_file(filepath): return module -class Command(ScrapyCommand): +class Command(BaseRunSpiderCommand): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} @@ -38,25 +37,6 @@ class Command(ScrapyCommand): def long_desc(self): return "Run the spider defined in the given file" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) != 1: raise UsageError()