From 8e0b2bd343865fdb195f713151d93fc1fc103b94 Mon Sep 17 00:00:00 2001 From: Denys Butenko Date: Wed, 19 Mar 2014 19:00:55 +0200 Subject: [PATCH 1/5] Resolved issue #546. Output format parsing from filename extension. --- scrapy/commands/crawl.py | 12 ++++++------ scrapy/commands/runspider.py | 13 ++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f0cd06a55..00d63dbe1 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -2,6 +2,7 @@ from scrapy.command import ScrapyCommand from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError + class Command(ScrapyCommand): requires_project = True @@ -14,12 +15,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", \ - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \ - help="format to use for dumping items with -o (default: %default)") + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", + help="dump scraped items into FILE (use - for stdout)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -33,6 +32,7 @@ class Command(ScrapyCommand): else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() + opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 14d67e24b..1bbcef880 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -37,12 +37,10 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", \ - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \ - help="format to use for dumping items with -o (default: %default)") + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", + help="dump scraped items into FILE (use - for stdout)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -56,8 +54,9 @@ class Command(ScrapyCommand): else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() + opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format,valid_output_formats)) + raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): From 9822eb78cc52c641416d0c0b8d5ef94d2e364de0 Mon Sep 17 00:00:00 2001 From: Denys Butenko Date: Wed, 19 Mar 2014 21:18:01 +0200 Subject: [PATCH 2/5] Added back `-t` option. If `--output-format` not defined parse from extension `--output` --- scrapy/commands/crawl.py | 5 ++++- scrapy/commands/runspider.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 00d63dbe1..53d2137dc 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -19,6 +19,8 @@ class Command(ScrapyCommand): help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", help="dump scraped items into FILE (use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", + help="format to use for dumping items with -o (default: %default)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -32,7 +34,8 @@ class Command(ScrapyCommand): else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() - opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") + if not opts.output_format: + opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 1bbcef880..414f4c43f 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -41,6 +41,8 @@ class Command(ScrapyCommand): help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", help="dump scraped items into FILE (use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", + help="format to use for dumping items with -o (default: %default)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -54,7 +56,8 @@ class Command(ScrapyCommand): else: self.settings.overrides['FEED_URI'] = opts.output valid_output_formats = self.settings['FEED_EXPORTERS'].keys() + self.settings['FEED_EXPORTERS_BASE'].keys() - opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") + if not opts.output_format: + opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) self.settings.overrides['FEED_FORMAT'] = opts.output_format From e64d6e3f3cf263abc9d84701113d42397aeba11b Mon Sep 17 00:00:00 2001 From: Denys Butenko Date: Wed, 19 Mar 2014 21:32:32 +0200 Subject: [PATCH 3/5] Fix default value. --- scrapy/commands/crawl.py | 4 ++-- scrapy/commands/runspider.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 53d2137dc..2fd102cc3 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -19,8 +19,8 @@ class Command(ScrapyCommand): help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", - help="format to use for dumping items with -o (default: %default)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 414f4c43f..a9b429ec4 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -41,8 +41,8 @@ class Command(ScrapyCommand): help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", - help="format to use for dumping items with -o (default: %default)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) From aa79abc5bd71fc684e9ad096ec0ff69292e3caf9 Mon Sep 17 00:00:00 2001 From: Denys Butenko Date: Thu, 20 Mar 2014 12:36:05 +0200 Subject: [PATCH 4/5] Add import os for crawl. --- scrapy/commands/crawl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 2fd102cc3..6d56b5ea3 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,3 +1,4 @@ +import os from scrapy.command import ScrapyCommand from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError From 7e353761a33261794f28f6cb1c8a15fe374388b8 Mon Sep 17 00:00:00 2001 From: Denys Butenko Date: Wed, 26 Mar 2014 12:27:52 +0200 Subject: [PATCH 5/5] Added more verbose error message for unrecognized output format. PEP8. --- scrapy/commands/crawl.py | 5 ++++- scrapy/commands/runspider.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 6d56b5ea3..1cd40469b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -38,7 +38,10 @@ class Command(ScrapyCommand): if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) + raise UsageError("Unrecognized output format '%s', set one" + " using the '-t' switch or as a file extension" + " from the supported list %s" % (opts.output_format, + tuple(valid_output_formats))) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a9b429ec4..2d537dee7 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -7,6 +7,7 @@ from scrapy.command import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.conf import arglist_to_dict + def _import_file(filepath): abspath = os.path.abspath(filepath) dirname, file = os.path.split(abspath) @@ -22,6 +23,7 @@ def _import_file(filepath): sys.path.pop(0) return module + class Command(ScrapyCommand): requires_project = False @@ -59,7 +61,10 @@ class Command(ScrapyCommand): if not opts.output_format: opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") if opts.output_format not in valid_output_formats: - raise UsageError('Invalid/unrecognized output format: %s, Expected %s' % (opts.output_format, valid_output_formats)) + raise UsageError("Unrecognized output format '%s', set one" + " using the '-t' switch or as a file extension" + " from the supported list %s" % (opts.output_format, + tuple(valid_output_formats))) self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts):