From ade5efdc617a3f6c33b1c968862e0cbdc277d0d0 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 22 Oct 2011 20:53:49 -0200 Subject: [PATCH] added -o option to scrapy crawl, a convenient shortcut for using feed exports --- docs/faq.rst | 6 +++--- docs/intro/overview.rst | 2 +- docs/intro/tutorial.rst | 2 +- scrapy/commands/crawl.py | 7 +++++++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index a6fc4927d..abbef29ea 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -197,15 +197,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file? To dump into a JSON file:: - scrapy crawl myspider -s FEED_URI=items.json -s FEED_FORMAT=json + scrapy crawl myspider -o items.json -t json To dump into a CSV file:: - scrapy crawl myspider -s FEED_URI=items.csv -s FEED_FORMAT=csv + scrapy crawl myspider -o items.csv -t csv To dump into a XML file:: - scrapy crawl myspider -s FEED_URI=items.xml -s FEED_FORMAT=xml + scrapy crawl myspider -o items.xml -t xml For more information see :ref:`topics-feed-exports` diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index db2cafe2d..5828f0530 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -161,7 +161,7 @@ Run the spider to extract the data Finally, we'll run the spider to crawl the site an output file ``scraped_data.json`` with the scraped data in JSON format:: - scrapy crawl mininova.org -s FEED_URI=scraped_data.json -s FEED_FORMAT=json + scrapy crawl mininova.org -o scraped_data.json -t json This uses :ref:`feed exports ` to generate the JSON file. You can easily change the export format (XML or CSV, for example) or the diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a47b187f0..840515763 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -420,7 +420,7 @@ Storing the scraped data The simplest way to store the scraped data is by using the :ref:`Feed exports `, with the following command:: - scrapy crawl dmoz -s FEED_URI=items.json -s FEED_FORMAT=json + scrapy crawl dmoz -o items.json -t json That will generate a ``items.json`` file containing all scraped items, serialized in `JSON`_. diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index f97f907ee..424aa0224 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -16,6 +16,10 @@ class Command(ScrapyCommand): ScrapyCommand.add_options(self, parser) parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", \ help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", \ + help="store scraped items into FILE (using feed exports)") + parser.add_option("-t", "--output-format", metavar="FORMAT", default="jsonlines", \ + help="format to use in feed exports (default: %default)") def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) @@ -23,6 +27,9 @@ class Command(ScrapyCommand): opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + self.settings.overrides['FEED_URI'] = opts.output + self.settings.overrides['FEED_FORMAT'] = opts.output_format def run(self, args, opts): if len(args) < 1: