From 924219dd5a2c272fd5e010d917e46916c81d763b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 28 Aug 2009 20:32:55 -0300 Subject: [PATCH] - added reference documentation about scrapy-ctl.py script - yet another refactor to cmdline module - removed help command (use -h to help now) --HG-- rename : docs/experimental/scripts.rst => docs/topics/scrapy-ctl.rst --- docs/experimental/index.rst | 1 - docs/experimental/scripts.rst | 141 ------------------------------- docs/index.rst | 4 + docs/topics/scrapy-ctl.rst | 103 ++++++++++++++++++++++ scrapy/command/cmdline.py | 84 +++++++++--------- scrapy/command/commands/help.py | 40 --------- scrapy/command/commands/shell.py | 5 +- 7 files changed, 155 insertions(+), 223 deletions(-) delete mode 100644 docs/experimental/scripts.rst create mode 100644 docs/topics/scrapy-ctl.rst delete mode 100644 scrapy/command/commands/help.py diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index bfdc7cb0d..838759ac4 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -21,5 +21,4 @@ it's properly merged) . Use at your own risk. exporters images - scripts djangoitems diff --git a/docs/experimental/scripts.rst b/docs/experimental/scripts.rst deleted file mode 100644 index ca5305625..000000000 --- a/docs/experimental/scripts.rst +++ /dev/null @@ -1,141 +0,0 @@ -.. _topics-scripts: - -================== -Management scripts -================== - -Scrapy is controlled by the ``scrapy-ctl.py`` command. - -.. _topics-scripts-scrapy-ctl: - -scrapy-ctl.py -============= - -Usage: ``scrapy-ctl.py `` - -This script is located in every project's root folder. - - -Available subcommands ---------------------- - -crawl -~~~~~ -Usage: ``crawl [options] ...`` - -Start crawling a domain or URL - - -``--nopipeline`` -"""""""""""""""" -disable scraped item pipeline - -``--restrict`` -"""""""""""""" -restrict crawling only to the given urls - -``-n, --nofollow`` -"""""""""""""""""" -don't follow links (for use with URLs only) - -``-c, --callback`` -"""""""""""""""""" -use the provided callback for starting to crawl the given url - - -fetch -~~~~~ -Usage: ``fetch `` - -Fetch a URL using the Scrapy downloader and print its content to stdout. You -may want to use --nolog to disable logging. - - -``--headers`` -""""""""""""" -print HTTP headers instead of body - - -genspider -~~~~~~~~~ -Usage: ``genspider [options] `` - - -``--template`` -"""""""""""""" -Default: ``crawl`` - -uses a custom template. - -``--force`` -""""""""""" -if the spider already exists, overwrite it with the template. - -``--list`` -~~~~~~~~~~ -list available templates - -``--dump`` -"""""""""""""" -dump ``--template`` to stdout - - -help -~~~~ -Usage: ``help `` - -Provides extended help for the given command. - - -list -~~~~ -List available spiders. - - -parse -~~~~~ -Usage: ``parse [options] `` - -Parse the given URL (using the spider) and print the results. - - -``--nolinks`` -""""""""""""" -don't show extracted links - -``--noitems`` -""""""""""""" -don't show scraped items - -``--nocolour`` -"""""""""""""" -avoid using pygments to colorize the output - -``-r, --rules`` -""""""""""""""" -try to match and parse the url with the defined rules (if any) - -``-c, --callbacks`` -""""""""""""""""""" -use the provided callback(s) for parsing the url (separated with commas) - - -shell -~~~~~ -Usage: ``shell [options] `` - -Interactive console for scraping the given url. For scraping local files you -can use a URL like ``file://path/to/file.html``. See :ref:`topics-shell` for -usage documentation. - - -start -~~~~~ -Start the Scrapy manager but don't run any spider (idle mode) - -startproject -~~~~~~~~~~~~ -Usage: ``startproject `` - -Starts a new project with name ``project_name`` - diff --git a/docs/index.rst b/docs/index.rst index 6363c6024..9c28c212e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -164,11 +164,15 @@ Reference .. toctree:: :hidden: + topics/scrapy-ctl topics/request-response topics/settings topics/signals topics/exceptions +:ref:`topics-scrapy-ctl` + Understand the command used to control your Scrapy project. + :ref:`topics-request-response` Understand the classes used to represent HTTP requests and responses. diff --git a/docs/topics/scrapy-ctl.rst b/docs/topics/scrapy-ctl.rst new file mode 100644 index 000000000..411659ba9 --- /dev/null +++ b/docs/topics/scrapy-ctl.rst @@ -0,0 +1,103 @@ +.. _topics-scrapy-ctl: + +============= +scrapy-ctl.py +============= + +Scrapy is controlled through the ``scrapy-ctl.py`` control script. The script +provides several commands, for different purposes. Each command supports its +own particular syntax. In other words, each command supports a different set of +arguments and options. + +This page doesn't describe each command and its syntax, but provides an +introduction to how the ``scrapy-ctl.py`` script is used. After you learn how +to use it, you can get help for each particular command using the same +``scrapy-ctl.py`` script. + +Global and project-specific ``scrapy-ctl.py`` +============================================= + +There is one global ``scrapy-ctl.py`` script shipped with Scrapy and another +``scrapy-ctl.py`` script automatically created inside your Scrapy project. The +project-specific ``scrapy-ctl.py`` is just a thin wrapper around the global +``scrapy-ctl.py`` which populates the settings of your project, so you don't +have to specify them every time through the ``--settings`` argument. + +Using the ``scrapy-ctl.py`` script +================================== + +The first thing you would do with the ``scrapy-ctl.py`` script is create your +Scrapy project:: + + scrapy-ctl.py startproject myproject + +That will create a Scrapy project under the ``myproject`` directory and will +put a new ``scrapy-ctl.py`` inside that directory. + +So, you go inside the new project directory:: + + cd myproject + +And you're ready to use your project's ``scrapy-ctl.py``. For example, to +create a new spider:: + + python scrapy-ctl.py genspider mydomain mydomain.com + +This is the same as using the global ``scrapy-ctl.py`` script and passing the +project settings module in the ``--settings`` argument:: + + scrapy-ctl.py --settings=myproject.settings genspider mydomain mydomain.com + +You'll typically use the project-specific ``scrapy-ctl.py``, for convenience. + +See all available commands +-------------------------- + +To see all available commands type:: + + scrapy-ctl.py -h + +That will print a summary of all available Scrapy commands. + +The first line will print the currently active project (if any). + +Example (active project):: + + Scrapy 0.7.0 - project: myproject + + Usage + ===== + + ... + +Example (no active project):: + + Scrapy 0.7.0 - no active project + + Usage + ===== + + ... + + +Get help for a particular command +--------------------------------- + +To get help about a particular command, including its description, usage and +available options type:: + + scrapy-ctl.py -h + +Example:: + + scrapy-ctl.py crawl -h + +Using ``scrapy-ctl.py`` outside your project +============================================ + +Not all commands must be run from "inside" a Scrapy project. You can, for +example, use the ``fetch`` command to download a page (using Scrapy built-in +downloader) from outside a project. Other commands that can be used outside a +project are ``startproject`` (obviously) and ``shell``, to launch a +:ref:`Scrapy Shell `. + diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index 55c85b468..6c4dbd69a 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -10,10 +10,18 @@ from scrapy import log from scrapy.spider import spiders from scrapy.xlib import lsprofcalltree from scrapy.conf import settings +from scrapy.command.models import ScrapyCommand # This dict holds information about the executed command for later use command_executed = {} +def save_command_executed(cmdname, cmd, args, opts): + """Save command executed info for later reference""" + command_executed['name'] = cmdname + command_executed['class'] = cmd + command_executed['args'] = args[:] + command_executed['opts'] = opts.__dict__.copy() + def find_commands(dir): try: return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \ @@ -45,20 +53,26 @@ def get_command_name(argv): if not arg.startswith('-'): return arg -def usage(): - s = "Usage\n" - s += "=====\n" - s += "scrapy-ctl.py [options] [args]\n" - s += " Run a command\n\n" - s += "scrapy-ctl.py -h\n" - s += " Print command help and options\n\n" - s += "Available commands\n" - s += "===================\n" +def print_usage(inside_project): + if inside_project: + print "Scrapy %s - project: %s\n" % (scrapy.__version__, \ + settings['BOT_NAME']) + else: + print "Scrapy %s - no active project\n" % scrapy.__version__ + print "Usage" + print "=====\n" + print "To run a command:" + print " scrapy-ctl.py [options] [args]\n" + print "To get help:" + print " scrapy-ctl.py -h\n" + print "Available commands" + print "==================\n" cmds = get_commands_dict() for cmdname, cmdclass in sorted(cmds.iteritems()): - s += "%s %s\n" % (cmdname, cmdclass.syntax()) - s += " %s\n" % cmdclass.short_desc() - return s + if inside_project or not cmdclass.requires_project: + print "%s %s" % (cmdname, cmdclass.syntax()) + print " %s" % cmdclass.short_desc() + print def update_default_settings(module, cmdname): if not module: @@ -82,44 +96,38 @@ def execute(argv=None): update_default_settings('scrapy.conf.commands', cmdname) update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname) - if not cmdname: - print "Scrapy %s\n" % scrapy.__version__ - print usage() - sys.exit(2) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ conflict_handler='resolve', add_help_option=False) if cmdname in cmds: cmd = cmds[cmdname] cmd.add_options(parser) + opts, args = parser.parse_args(args=argv[1:]) + cmd.process_options(args, opts) parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) parser.description = cmd.long_desc() + if cmd.requires_project and not settings.settings_module: + print "Error running: scrapy-ctl.py %s\n" % cmdname + print "Cannot find project settings module in python path: %s" % \ + settings.settings_module_path + sys.exit(1) + if opts.help: + parser.print_help() + sys.exit() + elif not cmdname: + cmd = ScrapyCommand() + cmd.add_options(parser) + opts, args = parser.parse_args(args=argv) + cmd.process_options(args, opts) + print_usage(settings.settings_module) + sys.exit(2) else: - print "Scrapy %s\n" % scrapy.__version__ print "Unknown command: %s\n" % cmdname - print 'Type "%s -h" for help' % argv[0] + print 'Use "scrapy-ctl.py -h" for help' sys.exit(2) - (opts, args) = parser.parse_args(args=argv[1:]) - del args[0] # args[0] is cmdname - - if opts.help: - parser.print_help() - sys.exit() - - # storing command executed info for later reference - command_executed['name'] = cmdname - command_executed['class'] = cmd - command_executed['args'] = args[:] - command_executed['opts'] = opts.__dict__.copy() - - cmd.process_options(args, opts) - if cmd.requires_project and not settings.settings_module: - print "Error running: scrapy-ctl.py %s\n" % cmdname - print "Cannot find project settings module in python path: %s" % \ - settings.settings_module_path - sys.exit(1) + del args[0] # remove command name from args + save_command_executed(cmdname, cmd, args, opts) spiders.load() log.start() ret = run_command(cmd, args, opts) diff --git a/scrapy/command/commands/help.py b/scrapy/command/commands/help.py deleted file mode 100644 index 915a22041..000000000 --- a/scrapy/command/commands/help.py +++ /dev/null @@ -1,40 +0,0 @@ -import textwrap - -from scrapy.command import ScrapyCommand, cmdline - -class Command(ScrapyCommand): - - requires_project = False - - def syntax(self): - return "" - - def short_desc(self): - return "Provides extended help for the given command" - - def run(self, args, opts): - if not args: - return False - - commands = cmdline.get_commands_dict() - - cmdname = args[0] - if cmdname in commands: - cmd = commands[cmdname] - help = getattr(cmd, 'help', None) or getattr(cmd, 'long_desc', None) - title = "%s command" % cmdname - print title - print "-" * len(title) - print - print cmd.short_desc() - print - print "usage: %s %s" % (cmdname, cmd.syntax()) - print - print "\n".join(textwrap.wrap(help())) - print - print "For a list of supported arguments use:" - print - print " scrapy-ctl.py %s -h" % cmdname - print - else: - print "Unknown command: %s" % cmdname diff --git a/scrapy/command/commands/shell.py b/scrapy/command/commands/shell.py index 3f8c9e241..364821a18 100644 --- a/scrapy/command/commands/shell.py +++ b/scrapy/command/commands/shell.py @@ -12,14 +12,13 @@ class Command(ScrapyCommand): requires_project = False def syntax(self): - return "[url]" + return "[url|file]" def short_desc(self): return "Interactive scraping console" def long_desc(self): - return "Interactive console for scraping the given url. For scraping " \ - "local files you can use a URL like file://path/to/file.html" + return "Interactive console for scraping the given url" def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be