Merge pull request #432 from darkrho/crawl-url

Removed URL reference in crawl command and .tld suffix in docs for spider names

docs/intro/overview.rst

@@ -137,7 +137,7 @@ Finally, here's the spider code::
 
     class MininovaSpider(CrawlSpider):
 
-        name = 'mininova.org'
+        name = 'mininova'
         allowed_domains = ['mininova.org']
         start_urls = ['http://www.mininova.org/today']
         rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')]
@@ -160,7 +160,7 @@ Run the spider to extract the data
 Finally, we'll run the spider to crawl the site an output file
 ``scraped_data.json`` with the scraped data in JSON format::
 
-    scrapy crawl mininova.org -o scraped_data.json -t json
+    scrapy crawl mininova -o scraped_data.json -t json
 
 This uses :ref:`feed exports <topics-feed-exports>` to generate the JSON file.
 You can easily change the export format (XML or CSV, for example) or the

docs/topics/settings.rst

@@ -56,7 +56,7 @@ Settings attribute.
 
 Example::
 
-    scrapy crawl domain.com -s LOG_FILE=scrapy.log
+    scrapy crawl myspider -s LOG_FILE=scrapy.log
 
 2. Project settings module
 --------------------------

scrapy/command.py

@@ -1,7 +1,6 @@
 """
 Base class for Scrapy commands
 """
-
 import os
 import warnings
 from optparse import OptionGroup
@@ -10,6 +9,7 @@ from twisted.python import failure
 from scrapy.utils.conf import arglist_to_dict
 from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
 
+
 class ScrapyCommand(object):
 
     requires_project = False
@@ -21,7 +21,7 @@ class ScrapyCommand(object):
     exitcode = 0
 
     def __init__(self):
-        self.settings = None # set in scrapy.cmdline
+        self.settings = None  # set in scrapy.cmdline
 
     def set_crawler(self, crawler):
         assert not hasattr(self, '_crawler'), "crawler already set"
@@ -38,12 +38,14 @@ class ScrapyCommand(object):
 
             old_start = crawler.start
             self.crawler_process.started = False
+
             def wrapped_start():
                 if self.crawler_process.started:
                     old_start()
                 else:
                     self.crawler_process.started = True
                     self.crawler_process.start()
+
             crawler.start = wrapped_start
 
             self.set_crawler(crawler)
@@ -81,22 +83,22 @@ class ScrapyCommand(object):
         Populate option parse with options available for this command
         """
         group = OptionGroup(parser, "Global Options")
-        group.add_option("--logfile", metavar="FILE", \
+        group.add_option("--logfile", metavar="FILE",
             help="log file. if omitted stderr will be used")
-        group.add_option("-L", "--loglevel", metavar="LEVEL", \
-            default=None, \
+        group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
             help="log level (default: %s)" % self.settings['LOG_LEVEL'])
-        group.add_option("--nolog", action="store_true", \
+        group.add_option("--nolog", action="store_true",
             help="disable logging completely")
-        group.add_option("--profile", metavar="FILE", default=None, \
+        group.add_option("--profile", metavar="FILE", default=None,
             help="write python cProfile stats to FILE")
-        group.add_option("--lsprof", metavar="FILE", default=None, \
+        group.add_option("--lsprof", metavar="FILE", default=None,
             help="write lsprof profiling stats to FILE")
-        group.add_option("--pidfile", metavar="FILE", \
+        group.add_option("--pidfile", metavar="FILE",
             help="write process ID to FILE")
-        group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", \
+        group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
             help="set/override setting (may be repeated)")
         group.add_option("--pdb", action="store_true", help="enable pdb on failure")
+
         parser.add_option_group(group)
 
     def process_options(self, args, opts):

scrapy/commands/crawl.py

@@ -10,7 +10,7 @@ class Command(ScrapyCommand):
         return "[options] <spider>"
 
     def short_desc(self):
-        return "Start crawling from a spider or URL"
+        return "Start crawling from a spider"
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)