diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f0cd06a55..1cd40469b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,7 +1,9 @@ +import os from scrapy.command import ScrapyCommand from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError + class Command(ScrapyCommand): requires_project = True @@ -14,12 +16,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", \ - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \ - help="format to use for dumping items with -o (default: %default)") + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", + help="dump scraped items into FILE (use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -33,8 +35,13 @@ class Command(ScrapyCommand): else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() + if not opts.output_format: + opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) + raise UsageError("Unrecognized output format '%s', set one" + " using the '-t' switch or as a file extension" + " from the supported list %s" % (opts.output_format, + tuple(valid_output_formats))) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 14d67e24b..2d537dee7 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -7,6 +7,7 @@ from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict + def _import_file(filepath): abspath = os.path.abspath(filepath) dirname, file = os.path.split(abspath) @@ -22,6 +23,7 @@ def _import_file(filepath): sys.path.pop(0) return module + class Command(ScrapyCommand): requires_project = False @@ -37,12 +39,12 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", \ - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \ - help="format to use for dumping items with -o (default: %default)") + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", + help="dump scraped items into FILE (use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -56,8 +58,13 @@ class Command(ScrapyCommand): else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() + if not opts.output_format: + opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats)) + raise UsageError("Unrecognized output format '%s', set one" + " using the '-t' switch or as a file extension" + " from the supported list %s" % (opts.output_format, + tuple(valid_output_formats))) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts):