diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py new file mode 100644 index 000000000..7da7494ac --- /dev/null +++ b/scrapy/commands/common_commands.py @@ -0,0 +1,29 @@ +from scrapy.commands import ScrapyCommand +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.exceptions import UsageError + + +class CommonCommands(ScrapyCommand): + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-a", dest="spargs", action="append", default=[], + metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", action="append", + help="dump scraped items into FILE" + + "(use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + feeds = feed_process_params_from_cli( + self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b2f9484b..b477d7c71 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,9 +1,8 @@ -from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError +from scrapy.commands.common_commands import CommonCommands -class Command(ScrapyCommand): +class Command(CommonCommands): requires_project = True @@ -13,30 +12,12 @@ class Command(ScrapyCommand): def short_desc(self): return "Run a spider" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError( + "running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 62510609a..9959f6b0d 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,9 +3,8 @@ import os from importlib import import_module from scrapy.utils.spider import iter_spider_classes -from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.commands.common_commands import CommonCommands def _import_file(filepath): @@ -24,7 +23,7 @@ def _import_file(filepath): return module -class Command(ScrapyCommand): +class Command(CommonCommands): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} @@ -38,25 +37,6 @@ class Command(ScrapyCommand): def long_desc(self): return "Run the spider defined in the given file" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) != 1: raise UsageError()