mirror of https://github.com/scrapy/scrapy.git
- added reference documentation about scrapy-ctl.py script
- yet another refactor to cmdline module - removed help command (use -h to help now) --HG-- rename : docs/experimental/scripts.rst => docs/topics/scrapy-ctl.rst
This commit is contained in:
Pablo Hoffman
parent
566758eeb7
commit
924219dd5a
|
|
@ -21,5 +21,4 @@ it's properly merged) . Use at your own risk.
|
|||
|
||||
exporters
|
||||
images
|
||||
scripts
|
||||
djangoitems
|
||||
|
|
|
|||
|
|
@ -1,141 +0,0 @@
|
|||
.. _topics-scripts:
|
||||
|
||||
==================
|
||||
Management scripts
|
||||
==================
|
||||
|
||||
Scrapy is controlled by the ``scrapy-ctl.py`` command.
|
||||
|
||||
.. _topics-scripts-scrapy-ctl:
|
||||
|
||||
scrapy-ctl.py
|
||||
=============
|
||||
|
||||
Usage: ``scrapy-ctl.py <command>``
|
||||
|
||||
This script is located in every project's root folder.
|
||||
|
||||
|
||||
Available subcommands
|
||||
---------------------
|
||||
|
||||
crawl
|
||||
~~~~~
|
||||
Usage: ``crawl [options] <domain|url> ...``
|
||||
|
||||
Start crawling a domain or URL
|
||||
|
||||
|
||||
``--nopipeline``
|
||||
""""""""""""""""
|
||||
disable scraped item pipeline
|
||||
|
||||
``--restrict``
|
||||
""""""""""""""
|
||||
restrict crawling only to the given urls
|
||||
|
||||
``-n, --nofollow``
|
||||
""""""""""""""""""
|
||||
don't follow links (for use with URLs only)
|
||||
|
||||
``-c, --callback``
|
||||
""""""""""""""""""
|
||||
use the provided callback for starting to crawl the given url
|
||||
|
||||
|
||||
fetch
|
||||
~~~~~
|
||||
Usage: ``fetch <url>``
|
||||
|
||||
Fetch a URL using the Scrapy downloader and print its content to stdout. You
|
||||
may want to use --nolog to disable logging.
|
||||
|
||||
|
||||
``--headers``
|
||||
"""""""""""""
|
||||
print HTTP headers instead of body
|
||||
|
||||
|
||||
genspider
|
||||
~~~~~~~~~
|
||||
Usage: ``genspider [options] <spider_module_name> <spider_domain_name>``
|
||||
|
||||
|
||||
``--template``
|
||||
""""""""""""""
|
||||
Default: ``crawl``
|
||||
|
||||
uses a custom template.
|
||||
|
||||
``--force``
|
||||
"""""""""""
|
||||
if the spider already exists, overwrite it with the template.
|
||||
|
||||
``--list``
|
||||
~~~~~~~~~~
|
||||
list available templates
|
||||
|
||||
``--dump``
|
||||
""""""""""""""
|
||||
dump ``--template`` to stdout
|
||||
|
||||
|
||||
help
|
||||
~~~~
|
||||
Usage: ``help <command>``
|
||||
|
||||
Provides extended help for the given command.
|
||||
|
||||
|
||||
list
|
||||
~~~~
|
||||
List available spiders.
|
||||
|
||||
|
||||
parse
|
||||
~~~~~
|
||||
Usage: ``parse [options] <url>``
|
||||
|
||||
Parse the given URL (using the spider) and print the results.
|
||||
|
||||
|
||||
``--nolinks``
|
||||
"""""""""""""
|
||||
don't show extracted links
|
||||
|
||||
``--noitems``
|
||||
"""""""""""""
|
||||
don't show scraped items
|
||||
|
||||
``--nocolour``
|
||||
""""""""""""""
|
||||
avoid using pygments to colorize the output
|
||||
|
||||
``-r, --rules``
|
||||
"""""""""""""""
|
||||
try to match and parse the url with the defined rules (if any)
|
||||
|
||||
``-c, --callbacks``
|
||||
"""""""""""""""""""
|
||||
use the provided callback(s) for parsing the url (separated with commas)
|
||||
|
||||
|
||||
shell
|
||||
~~~~~
|
||||
Usage: ``shell [options] <url>``
|
||||
|
||||
Interactive console for scraping the given url. For scraping local files you
|
||||
can use a URL like ``file://path/to/file.html``. See :ref:`topics-shell` for
|
||||
usage documentation.
|
||||
|
||||
|
||||
start
|
||||
~~~~~
|
||||
Start the Scrapy manager but don't run any spider (idle mode)
|
||||
|
||||
startproject
|
||||
~~~~~~~~~~~~
|
||||
Usage: ``startproject <project_name>``
|
||||
|
||||
Starts a new project with name ``project_name``
|
||||
|
||||
|
|
@ -164,11 +164,15 @@ Reference
|
|||
.. toctree::
|
||||
:hidden:
|
||||
|
||||
topics/scrapy-ctl
|
||||
topics/request-response
|
||||
topics/settings
|
||||
topics/signals
|
||||
topics/exceptions
|
||||
|
||||
:ref:`topics-scrapy-ctl`
|
||||
Understand the command used to control your Scrapy project.
|
||||
|
||||
:ref:`topics-request-response`
|
||||
Understand the classes used to represent HTTP requests and responses.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
.. _topics-scrapy-ctl:
|
||||
|
||||
=============
|
||||
scrapy-ctl.py
|
||||
=============
|
||||
|
||||
Scrapy is controlled through the ``scrapy-ctl.py`` control script. The script
|
||||
provides several commands, for different purposes. Each command supports its
|
||||
own particular syntax. In other words, each command supports a different set of
|
||||
arguments and options.
|
||||
|
||||
This page doesn't describe each command and its syntax, but provides an
|
||||
introduction to how the ``scrapy-ctl.py`` script is used. After you learn how
|
||||
to use it, you can get help for each particular command using the same
|
||||
``scrapy-ctl.py`` script.
|
||||
|
||||
Global and project-specific ``scrapy-ctl.py``
|
||||
=============================================
|
||||
|
||||
There is one global ``scrapy-ctl.py`` script shipped with Scrapy and another
|
||||
``scrapy-ctl.py`` script automatically created inside your Scrapy project. The
|
||||
project-specific ``scrapy-ctl.py`` is just a thin wrapper around the global
|
||||
``scrapy-ctl.py`` which populates the settings of your project, so you don't
|
||||
have to specify them every time through the ``--settings`` argument.
|
||||
|
||||
Using the ``scrapy-ctl.py`` script
|
||||
==================================
|
||||
|
||||
The first thing you would do with the ``scrapy-ctl.py`` script is create your
|
||||
Scrapy project::
|
||||
|
||||
scrapy-ctl.py startproject myproject
|
||||
|
||||
That will create a Scrapy project under the ``myproject`` directory and will
|
||||
put a new ``scrapy-ctl.py`` inside that directory.
|
||||
|
||||
So, you go inside the new project directory::
|
||||
|
||||
cd myproject
|
||||
|
||||
And you're ready to use your project's ``scrapy-ctl.py``. For example, to
|
||||
create a new spider::
|
||||
|
||||
python scrapy-ctl.py genspider mydomain mydomain.com
|
||||
|
||||
This is the same as using the global ``scrapy-ctl.py`` script and passing the
|
||||
project settings module in the ``--settings`` argument::
|
||||
|
||||
scrapy-ctl.py --settings=myproject.settings genspider mydomain mydomain.com
|
||||
|
||||
You'll typically use the project-specific ``scrapy-ctl.py``, for convenience.
|
||||
|
||||
See all available commands
|
||||
--------------------------
|
||||
|
||||
To see all available commands type::
|
||||
|
||||
scrapy-ctl.py -h
|
||||
|
||||
That will print a summary of all available Scrapy commands.
|
||||
|
||||
The first line will print the currently active project (if any).
|
||||
|
||||
Example (active project)::
|
||||
|
||||
Scrapy 0.7.0 - project: myproject
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
...
|
||||
|
||||
Example (no active project)::
|
||||
|
||||
Scrapy 0.7.0 - no active project
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
...
|
||||
|
||||
|
||||
Get help for a particular command
|
||||
---------------------------------
|
||||
|
||||
To get help about a particular command, including its description, usage and
|
||||
available options type::
|
||||
|
||||
scrapy-ctl.py <command> -h
|
||||
|
||||
Example::
|
||||
|
||||
scrapy-ctl.py crawl -h
|
||||
|
||||
Using ``scrapy-ctl.py`` outside your project
|
||||
============================================
|
||||
|
||||
Not all commands must be run from "inside" a Scrapy project. You can, for
|
||||
example, use the ``fetch`` command to download a page (using Scrapy built-in
|
||||
downloader) from outside a project. Other commands that can be used outside a
|
||||
project are ``startproject`` (obviously) and ``shell``, to launch a
|
||||
:ref:`Scrapy Shell <topics-shell>`.
|
||||
|
||||
|
|
@ -10,10 +10,18 @@ from scrapy import log
|
|||
from scrapy.spider import spiders
|
||||
from scrapy.xlib import lsprofcalltree
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command.models import ScrapyCommand
|
||||
|
||||
# This dict holds information about the executed command for later use
|
||||
command_executed = {}
|
||||
|
||||
def save_command_executed(cmdname, cmd, args, opts):
|
||||
"""Save command executed info for later reference"""
|
||||
command_executed['name'] = cmdname
|
||||
command_executed['class'] = cmd
|
||||
command_executed['args'] = args[:]
|
||||
command_executed['opts'] = opts.__dict__.copy()
|
||||
|
||||
def find_commands(dir):
|
||||
try:
|
||||
return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \
|
||||
|
|
@ -45,20 +53,26 @@ def get_command_name(argv):
|
|||
if not arg.startswith('-'):
|
||||
return arg
|
||||
|
||||
def usage():
|
||||
s = "Usage\n"
|
||||
s += "=====\n"
|
||||
s += "scrapy-ctl.py <command> [options] [args]\n"
|
||||
s += " Run a command\n\n"
|
||||
s += "scrapy-ctl.py <command> -h\n"
|
||||
s += " Print command help and options\n\n"
|
||||
s += "Available commands\n"
|
||||
s += "===================\n"
|
||||
def print_usage(inside_project):
|
||||
if inside_project:
|
||||
print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
|
||||
settings['BOT_NAME'])
|
||||
else:
|
||||
print "Scrapy %s - no active project\n" % scrapy.__version__
|
||||
print "Usage"
|
||||
print "=====\n"
|
||||
print "To run a command:"
|
||||
print " scrapy-ctl.py <command> [options] [args]\n"
|
||||
print "To get help:"
|
||||
print " scrapy-ctl.py <command> -h\n"
|
||||
print "Available commands"
|
||||
print "==================\n"
|
||||
cmds = get_commands_dict()
|
||||
for cmdname, cmdclass in sorted(cmds.iteritems()):
|
||||
s += "%s %s\n" % (cmdname, cmdclass.syntax())
|
||||
s += " %s\n" % cmdclass.short_desc()
|
||||
return s
|
||||
if inside_project or not cmdclass.requires_project:
|
||||
print "%s %s" % (cmdname, cmdclass.syntax())
|
||||
print " %s" % cmdclass.short_desc()
|
||||
print
|
||||
|
||||
def update_default_settings(module, cmdname):
|
||||
if not module:
|
||||
|
|
@ -82,44 +96,38 @@ def execute(argv=None):
|
|||
update_default_settings('scrapy.conf.commands', cmdname)
|
||||
update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname)
|
||||
|
||||
if not cmdname:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print usage()
|
||||
sys.exit(2)
|
||||
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
conflict_handler='resolve', add_help_option=False)
|
||||
|
||||
if cmdname in cmds:
|
||||
cmd = cmds[cmdname]
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
cmd.process_options(args, opts)
|
||||
parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
|
||||
parser.description = cmd.long_desc()
|
||||
if cmd.requires_project and not settings.settings_module:
|
||||
print "Error running: scrapy-ctl.py %s\n" % cmdname
|
||||
print "Cannot find project settings module in python path: %s" % \
|
||||
settings.settings_module_path
|
||||
sys.exit(1)
|
||||
if opts.help:
|
||||
parser.print_help()
|
||||
sys.exit()
|
||||
elif not cmdname:
|
||||
cmd = ScrapyCommand()
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv)
|
||||
cmd.process_options(args, opts)
|
||||
print_usage(settings.settings_module)
|
||||
sys.exit(2)
|
||||
else:
|
||||
print "Scrapy %s\n" % scrapy.__version__
|
||||
print "Unknown command: %s\n" % cmdname
|
||||
print 'Type "%s -h" for help' % argv[0]
|
||||
print 'Use "scrapy-ctl.py -h" for help'
|
||||
sys.exit(2)
|
||||
|
||||
(opts, args) = parser.parse_args(args=argv[1:])
|
||||
del args[0] # args[0] is cmdname
|
||||
|
||||
if opts.help:
|
||||
parser.print_help()
|
||||
sys.exit()
|
||||
|
||||
# storing command executed info for later reference
|
||||
command_executed['name'] = cmdname
|
||||
command_executed['class'] = cmd
|
||||
command_executed['args'] = args[:]
|
||||
command_executed['opts'] = opts.__dict__.copy()
|
||||
|
||||
cmd.process_options(args, opts)
|
||||
if cmd.requires_project and not settings.settings_module:
|
||||
print "Error running: scrapy-ctl.py %s\n" % cmdname
|
||||
print "Cannot find project settings module in python path: %s" % \
|
||||
settings.settings_module_path
|
||||
sys.exit(1)
|
||||
del args[0] # remove command name from args
|
||||
save_command_executed(cmdname, cmd, args, opts)
|
||||
spiders.load()
|
||||
log.start()
|
||||
ret = run_command(cmd, args, opts)
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
import textwrap
|
||||
|
||||
from scrapy.command import ScrapyCommand, cmdline
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
requires_project = False
|
||||
|
||||
def syntax(self):
|
||||
return "<command>"
|
||||
|
||||
def short_desc(self):
|
||||
return "Provides extended help for the given command"
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
return False
|
||||
|
||||
commands = cmdline.get_commands_dict()
|
||||
|
||||
cmdname = args[0]
|
||||
if cmdname in commands:
|
||||
cmd = commands[cmdname]
|
||||
help = getattr(cmd, 'help', None) or getattr(cmd, 'long_desc', None)
|
||||
title = "%s command" % cmdname
|
||||
print title
|
||||
print "-" * len(title)
|
||||
print
|
||||
print cmd.short_desc()
|
||||
print
|
||||
print "usage: %s %s" % (cmdname, cmd.syntax())
|
||||
print
|
||||
print "\n".join(textwrap.wrap(help()))
|
||||
print
|
||||
print "For a list of supported arguments use:"
|
||||
print
|
||||
print " scrapy-ctl.py %s -h" % cmdname
|
||||
print
|
||||
else:
|
||||
print "Unknown command: %s" % cmdname
|
||||
|
|
@ -12,14 +12,13 @@ class Command(ScrapyCommand):
|
|||
requires_project = False
|
||||
|
||||
def syntax(self):
|
||||
return "[url]"
|
||||
return "[url|file]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Interactive scraping console"
|
||||
|
||||
def long_desc(self):
|
||||
return "Interactive console for scraping the given url. For scraping " \
|
||||
"local files you can use a URL like file://path/to/file.html"
|
||||
return "Interactive console for scraping the given url"
|
||||
|
||||
def update_vars(self, vars):
|
||||
"""You can use this function to update the Scrapy objects that will be
|
||||
|
|
|
|||
Loading…
Reference in New Issue