diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 588f33755..b9ecead6c 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -137,7 +137,7 @@ Finally, here's the spider code:: class MininovaSpider(CrawlSpider): - name = 'mininova.org' + name = 'mininova' allowed_domains = ['mininova.org'] start_urls = ['http://www.mininova.org/today'] rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] @@ -160,7 +160,7 @@ Run the spider to extract the data Finally, we'll run the spider to crawl the site an output file ``scraped_data.json`` with the scraped data in JSON format:: - scrapy crawl mininova.org -o scraped_data.json -t json + scrapy crawl mininova -o scraped_data.json -t json This uses :ref:`feed exports ` to generate the JSON file. You can easily change the export format (XML or CSV, for example) or the diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f5e6139c3..bdc948a3c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -56,7 +56,7 @@ Settings attribute. Example:: - scrapy crawl domain.com -s LOG_FILE=scrapy.log + scrapy crawl myspider -s LOG_FILE=scrapy.log 2. Project settings module -------------------------- diff --git a/scrapy/command.py b/scrapy/command.py index 039f53442..5fe884abe 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -1,7 +1,6 @@ """ Base class for Scrapy commands """ - import os import warnings from optparse import OptionGroup @@ -10,6 +9,7 @@ from twisted.python import failure from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError, ScrapyDeprecationWarning + class ScrapyCommand(object): requires_project = False @@ -21,7 +21,7 @@ class ScrapyCommand(object): exitcode = 0 def __init__(self): - self.settings = None # set in scrapy.cmdline + self.settings = None # set in scrapy.cmdline def set_crawler(self, crawler): assert not hasattr(self, '_crawler'), "crawler already set" @@ -38,12 +38,14 @@ class ScrapyCommand(object): old_start = crawler.start self.crawler_process.started = False + def wrapped_start(): if self.crawler_process.started: old_start() else: self.crawler_process.started = True self.crawler_process.start() + crawler.start = wrapped_start self.set_crawler(crawler) @@ -81,22 +83,22 @@ class ScrapyCommand(object): Populate option parse with options available for this command """ group = OptionGroup(parser, "Global Options") - group.add_option("--logfile", metavar="FILE", \ + group.add_option("--logfile", metavar="FILE", help="log file. if omitted stderr will be used") - group.add_option("-L", "--loglevel", metavar="LEVEL", \ - default=None, \ + group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, help="log level (default: %s)" % self.settings['LOG_LEVEL']) - group.add_option("--nolog", action="store_true", \ + group.add_option("--nolog", action="store_true", help="disable logging completely") - group.add_option("--profile", metavar="FILE", default=None, \ + group.add_option("--profile", metavar="FILE", default=None, help="write python cProfile stats to FILE") - group.add_option("--lsprof", metavar="FILE", default=None, \ + group.add_option("--lsprof", metavar="FILE", default=None, help="write lsprof profiling stats to FILE") - group.add_option("--pidfile", metavar="FILE", \ + group.add_option("--pidfile", metavar="FILE", help="write process ID to FILE") - group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", \ + group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", help="set/override setting (may be repeated)") group.add_option("--pdb", action="store_true", help="enable pdb on failure") + parser.add_option_group(group) def process_options(self, args, opts): diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index b781b1a64..cafbac6f8 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -10,7 +10,7 @@ class Command(ScrapyCommand): return "[options] " def short_desc(self): - return "Start crawling from a spider or URL" + return "Start crawling from a spider" def add_options(self, parser): ScrapyCommand.add_options(self, parser)