From 6b20f1f7d93df14620f471f84adf17058262e782 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 8 Aug 2009 04:02:49 -0300 Subject: [PATCH] added "Global Options" group to command line options, improved help display, splitted --profile option in --profile + --lsprof --- scrapy/command/cmdline.py | 43 ++++++++++++++++++++++++--------------- scrapy/command/models.py | 35 ++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index bd72ef49e..9e722d1ac 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -8,6 +8,7 @@ import cProfile import scrapy from scrapy import log from scrapy.spider import spiders +from scrapy.xlib import lsprofcalltree from scrapy.conf import settings, SETTINGS_MODULE def find_commands(dir): @@ -48,10 +49,15 @@ def getcmdname(argv): if not arg.startswith('-'): return arg -def usage(argv): - s = "usage: %s [options] [args]\n" % argv[0] - s += " %s -h\n\n" % argv[0] - s += "Available commands:\n\n" +def usage(prog): + s = "Usage\n" + s += "=====\n" + s += "%s [options] [args]\n" % prog + s += " Run a command\n\n" + s += "%s -h\n" % prog + s += " Print command help and options\n\n" + s += "Available commands\n" + s += "===================\n" cmds = builtin_commands_dict() cmds.update(custom_commands_dict()) @@ -103,10 +109,11 @@ def execute_with_args(argv): if not cmdname: print "Scrapy %s\n" % scrapy.__version__ - print usage(argv) + print usage(argv[0]) sys.exit(2) - parser = optparse.OptionParser(conflict_handler='resolve') + parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ + conflict_handler='resolve', add_help_option=False) if cmdname in cmds: cmd = cmds[cmdname] @@ -122,6 +129,10 @@ def execute_with_args(argv): (opts, args) = parser.parse_args(args=argv[1:]) del args[0] # args[0] is cmdname + if opts.help: + parser.print_help() + sys.exit() + # storing command executed info for later reference command_executed['name'] = cmdname command_executed['class'] = cmd @@ -131,20 +142,20 @@ def execute_with_args(argv): cmd.process_options(args, opts) spiders.load() log.start() - if opts.profile: - log.msg("Profiling enabled. Analyze later with: python -m pstats %s" % opts.profile) + if opts.profile or opts.lsprof: + if opts.profile: + log.msg("writing cProfile stats to %r" % opts.profile) + if opts.lsprof: + log.msg("writing lsprof stats to %r" % opts.lsprof) loc = locals() p = cProfile.Profile() p.runctx('ret = cmd.run(args, opts)', globals(), loc) - p.dump_stats(opts.profile) - try: - from scrapy.xlib import lsprofcalltree - fn = opts.profile + ".cachegrind" - k = lsprofcalltree.KCacheGrind(p) - with open(fn, 'w') as f: + if opts.profile: + p.dump_stats(opts.profile) + k = lsprofcalltree.KCacheGrind(p) + if opts.lsprof: + with open(opts.lsprof, 'w') as f: k.output(f) - except ImportError: - pass ret = loc['ret'] else: ret = cmd.run(args, opts) diff --git a/scrapy/command/models.py b/scrapy/command/models.py index fc8e3aec2..a4ed4147c 100644 --- a/scrapy/command/models.py +++ b/scrapy/command/models.py @@ -6,6 +6,8 @@ from __future__ import with_statement import os import sys +from optparse import OptionGroup + from scrapy.conf import settings class ScrapyCommand(object): @@ -22,16 +24,14 @@ class ScrapyCommand(object): return "" def long_desc(self): - """ - A long description of the command. Return short description when not + """A long description of the command. Return short description when not available. It cannot contain newlines, since contents will be formatted by optparser which removes newlines and wraps text. """ return self.short_desc() def help(self): - """ - An extensive help for the command. It will be shown when using the + """An extensive help for the command. It will be shown when using the "help" command. It can contain newlines, since not post-formatting will be applied to its contents. """ @@ -41,22 +41,30 @@ class ScrapyCommand(object): """ Populate option parse with options available for this command """ - parser.add_option("--logfile", dest="logfile", metavar="FILE", \ + group = OptionGroup(parser, "Global Options") + group.add_option("-h", "--help", action="store_true", dest="help", \ + help="print command help and options") + group.add_option("--logfile", dest="logfile", metavar="FILE", \ help="log file. if omitted stderr will be used") - parser.add_option("-L", "--loglevel", dest="loglevel", metavar="LEVEL", \ + group.add_option("-L", "--loglevel", dest="loglevel", metavar="LEVEL", \ default=None, \ - help="log level. use SILENT level to diasble all log messages") - parser.add_option("--default-spider", dest="default_spider", default=None, \ + help="log level (default: %s)" % settings['LOGLEVEL']) + group.add_option("--nolog", action="store_true", dest="nolog", \ + help="disable logging completely") + group.add_option("--default-spider", dest="default_spider", default=None, \ help="use this spider when arguments are urls and no spider is found") - parser.add_option("--spider", dest="spider", default=None, \ + group.add_option("--spider", dest="spider", default=None, \ help="always use this spider when arguments are urls") - parser.add_option("--profile", dest="profile", metavar="FILE", default=None, \ + group.add_option("--profile", dest="profile", metavar="FILE", default=None, \ help="write python cProfile stats to FILE") - parser.add_option("--pidfile", dest="pidfile", metavar="FILE", \ + group.add_option("--lsprof", dest="lsprof", metavar="FILE", default=None, \ + help="write lsprof profiling stats to FILE") + group.add_option("--pidfile", dest="pidfile", metavar="FILE", \ help="write process ID to FILE") - parser.add_option("--set", dest="settings", action="append", \ + group.add_option("--set", dest="settings", action="append", \ metavar="SETTING=VALUE", default=[], \ help="set/override setting (may be repeated)") + parser.add_option_group(group) def process_options(self, args, opts): if opts.logfile: @@ -67,6 +75,9 @@ class ScrapyCommand(object): settings.overrides['LOG_ENABLED'] = True settings.overrides['LOGLEVEL'] = opts.loglevel + if opts.nolog: + settings.overrides['LOG_ENABLED'] = False + if opts.default_spider: from scrapy.spider import spiders spiders.default_domain = opts.default_spider