diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index c7e859515..8d1a31d16 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -6,11 +6,10 @@ import optparse import cProfile import scrapy -from scrapy import log from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree from scrapy.conf import settings -from scrapy.command import ScrapyCommand +from scrapy.exceptions import UsageError def _find_commands(dir): try: @@ -19,47 +18,61 @@ def _find_commands(dir): except OSError: return [] -def _get_commands_from_module(module): +def _get_commands_from_module(module, inproject): d = {} mod = __import__(module, {}, {}, ['']) for cmdname in _find_commands(mod.__path__[0]): modname = '%s.%s' % (module, cmdname) command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) if callable(command): - d[cmdname] = command() + if inproject or not command.requires_project: + d[cmdname] = command() else: - print 'WARNING: Module %r does not define a Command class' % modname + raise RuntimeError("Module %r does not define a Command class" % modname) return d -def _get_commands_dict(): - cmds = _get_commands_from_module('scrapy.commands') +def _get_commands_dict(inproject): + cmds = _get_commands_from_module('scrapy.commands', inproject) cmds_module = settings['COMMANDS_MODULE'] if cmds_module: - cmds.update(_get_commands_from_module(cmds_module)) + cmds.update(_get_commands_from_module(cmds_module, inproject)) return cmds -def _get_command_name(argv): +def _pop_command_name(argv): + i = 0 for arg in argv[1:]: if not arg.startswith('-'): + del argv[i] return arg + i += 1 -def _print_usage(inside_project): - if inside_project: +def _print_header(inproject): + if inproject: print "Scrapy %s - project: %s\n" % (scrapy.__version__, \ settings['BOT_NAME']) else: print "Scrapy %s - no active project\n" % scrapy.__version__ + +def _print_commands(inproject): + _print_header(inproject) print "Usage:" print " scrapy [options] [args]\n" print "Available commands:" - cmds = _get_commands_dict() + cmds = _get_commands_dict(inproject) for cmdname, cmdclass in sorted(cmds.iteritems()): - if inside_project or not cmdclass.requires_project: - print " %-13s %s" % (cmdname, cmdclass.short_desc()) + print " %-13s %s" % (cmdname, cmdclass.short_desc()) print - print 'Use "scrapy -h" for more info about a command' + print 'Use "scrapy -h" to see more info about a command' -def check_deprecated_scrapy_ctl(argv): +def _print_unknown_command(cmdname, inproject): + _print_header(inproject) + print "Unknown command: %s\n" % cmdname + print 'Use "scrapy" to see available commands' + if not inproject: + print + print "More commands are available in project mode" + +def _check_deprecated_scrapy_ctl(argv, inproject): """Check if Scrapy was called using the deprecated scrapy-ctl command and warn in that case, also creating a scrapy.cfg if it doesn't exist. """ @@ -68,7 +81,7 @@ def check_deprecated_scrapy_ctl(argv): import warnings warnings.warn("`scrapy-ctl.py` command-line tool is deprecated and will be removed in Scrapy 0.11, use `scrapy` instead", DeprecationWarning, stacklevel=3) - if settings.settings_module: + if inproject: projpath = os.path.abspath(os.path.dirname(os.path.dirname(settings.settings_module.__file__))) cfg_path = os.path.join(projpath, 'scrapy.cfg') if not os.path.exists(cfg_path): @@ -77,72 +90,64 @@ def check_deprecated_scrapy_ctl(argv): f.write("[default]" + os.linesep) f.write("settings = %s" % settings.settings_module_path + os.linesep) +def _run_print_help(parser, func, *a, **kw): + try: + func(*a, **kw) + except UsageError, e: + if str(e): + parser.error(str(e)) + if e.print_help: + parser.print_help() + sys.exit(2) + def execute(argv=None): if argv is None: argv = sys.argv crawler = CrawlerProcess(settings) crawler.install() - check_deprecated_scrapy_ctl(argv) # TODO: remove for Scrapy 0.11 - cmds = _get_commands_dict() - cmdname = _get_command_name(argv) + inproject = bool(settings.settings_module) + _check_deprecated_scrapy_ctl(argv, inproject) # TODO: remove for Scrapy 0.11 + cmds = _get_commands_dict(inproject) + cmdname = _pop_command_name(argv) parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ - conflict_handler='resolve', add_help_option=False) - if cmdname in cmds: - cmd = cmds[cmdname] - cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) - cmd.process_options(args, opts) - parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) - parser.description = cmd.long_desc() - if cmd.requires_project and not settings.settings_module: - print "Error running: scrapy %s\n" % cmdname - print "Cannot find project settings module in python path: %s" % \ - settings.settings_module_path - sys.exit(1) - if opts.help: - parser.print_help() - sys.exit() - elif not cmdname: - cmd = ScrapyCommand() - cmd.add_options(parser) - opts, args = parser.parse_args(args=argv) - cmd.process_options(args, opts) - _print_usage(settings.settings_module) - sys.exit(2) - else: - print "Unknown command: %s\n" % cmdname - print 'Use "scrapy -h" for help' + conflict_handler='resolve') + if not cmdname: + _print_commands(inproject) + sys.exit(0) + elif cmdname not in cmds: + _print_unknown_command(cmdname, inproject) sys.exit(2) + cmd = cmds[cmdname] + parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax()) + parser.description = cmd.long_desc() settings.defaults.update(cmd.default_settings) - del args[0] # remove command name from args - log.start() cmd.set_crawler(crawler) - ret = _run_command(cmd, args, opts) - if ret is False: - parser.print_help() + cmd.add_options(parser) + opts, args = parser.parse_args(args=argv[1:]) + _run_print_help(parser, cmd.process_options, args, opts) + _run_print_help(parser, _run_command, cmd, args, opts) def _run_command(cmd, args, opts): if opts.profile or opts.lsprof: - return _run_command_profiled(cmd, args, opts) + _run_command_profiled(cmd, args, opts) else: - return cmd.run(args, opts) + cmd.run(args, opts) def _run_command_profiled(cmd, args, opts): if opts.profile: - log.msg("writing cProfile stats to %r" % opts.profile) + sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile) if opts.lsprof: - log.msg("writing lsprof stats to %r" % opts.lsprof) + sys.stderr.write("scrapy: writing lsprof stats to %r\n" % opts.lsprof) loc = locals() p = cProfile.Profile() - p.runctx('ret = cmd.run(args, opts)', globals(), loc) + p.runctx('cmd.run(args, opts)', globals(), loc) if opts.profile: p.dump_stats(opts.profile) k = lsprofcalltree.KCacheGrind(p) if opts.lsprof: with open(opts.lsprof, 'w') as f: k.output(f) - return loc['ret'] if __name__ == '__main__': execute() diff --git a/scrapy/command.py b/scrapy/command.py index 39bdb325d..a9c447e38 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -9,8 +9,10 @@ import sys from optparse import OptionGroup import scrapy +from scrapy import log from scrapy.conf import settings from scrapy.utils.conf import arglist_to_dict +from scrapy.exceptions import UsageError class ScrapyCommand(object): @@ -24,6 +26,8 @@ class ScrapyCommand(object): @property def crawler(self): + if not log.started: + log.start() self._crawler.configure() return self._crawler @@ -58,10 +62,6 @@ class ScrapyCommand(object): Populate option parse with options available for this command """ group = OptionGroup(parser, "Global Options") - group.add_option("-h", "--help", action="store_true", dest="help", \ - help="print command help and options") - group.add_option("--version", action="store_true", dest="version", \ - help="print Scrapy version and exit") group.add_option("--logfile", dest="logfile", metavar="FILE", \ help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", dest="loglevel", metavar="LEVEL", \ @@ -77,23 +77,13 @@ class ScrapyCommand(object): help="write process ID to FILE") group.add_option("--set", dest="set", action="append", default=[], metavar="NAME=VALUE", \ help="set/override setting (may be repeated)") - group.add_option("--settings", dest="settings", metavar="MODULE", - help="python path to the Scrapy project settings") parser.add_option_group(group) def process_options(self, args, opts): - if opts.settings: - settings.set_settings_module(opts.settings) - try: settings.overrides.update(arglist_to_dict(opts.set)) except ValueError: - sys.stderr.write("Invalid --set value, use --set NAME=VALUE\n") - sys.exit(2) - - if opts.version: - print "Scrapy %s" % scrapy.__version__ - sys.exit() + raise UsageError("Invalid --set value, use --set NAME=VALUE", print_help=False) if opts.logfile: settings.overrides['LOG_ENABLED'] = True @@ -115,4 +105,3 @@ class ScrapyCommand(object): Entry point for running commands """ raise NotImplementedError - diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index bc99c8f23..d27ba7f9b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -6,6 +6,7 @@ from scrapy.conf import settings from scrapy.http import Request from scrapy.utils.url import is_url from scrapy.utils.conf import arglist_to_dict +from scrapy.exceptions import UsageError from collections import defaultdict @@ -33,8 +34,7 @@ class Command(ScrapyCommand): try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: - sys.stderr.write("Invalid -a value, use -a NAME=VALUE\n") - sys.exit(2) + raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.nofollow: settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 980348b12..4d1abe14e 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from scrapy.command import ScrapyCommand from scrapy.http import Request from scrapy.spider import BaseSpider from scrapy.utils.url import is_url +from scrapy.exceptions import UsageError class Command(ScrapyCommand): @@ -35,7 +36,7 @@ class Command(ScrapyCommand): def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): - return False + raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 35bb0f23e..e3f869f4c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -7,6 +7,7 @@ import scrapy from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.utils.template import render_templatefile, string_camelcase +from scrapy.exceptions import UsageError def sanitize_module_name(module_name): """Sanitize the given module name, by replacing dashes and points @@ -54,7 +55,7 @@ class Command(ScrapyCommand): print open(template_file, 'r').read() return if len(args) != 2: - return False + raise UsageError() name, domain = args[0:2] module = sanitize_module_name(name) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 839b226ab..68f727e48 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -4,6 +4,7 @@ from scrapy.item import BaseItem from scrapy.utils import display from scrapy.utils.spider import iterate_spider_output from scrapy.utils.url import is_url +from scrapy.exceptions import UsageError from scrapy import log class Command(ScrapyCommand): @@ -93,7 +94,7 @@ class Command(ScrapyCommand): def run(self, args, opts): if not len(args) == 1 or not is_url(args[0]): - return False + raise UsageError() response, spider = self.get_response_and_spider(args[0], opts) if not response: return diff --git a/scrapy/commands/queue.py b/scrapy/commands/queue.py index 03379d8f6..fb6ee594e 100644 --- a/scrapy/commands/queue.py +++ b/scrapy/commands/queue.py @@ -1,6 +1,6 @@ from scrapy.command import ScrapyCommand from scrapy.commands import runserver -from scrapy.utils.misc import load_object +from scrapy.exceptions import UsageError from scrapy.conf import settings class Command(runserver.Command): @@ -23,7 +23,7 @@ class Command(runserver.Command): def run(self, args, opts): if len(args) < 1: - return False + raise UsageError() cmd = args[0] botname = settings['BOT_NAME'] @@ -31,7 +31,7 @@ class Command(runserver.Command): if cmd == 'add': if len(args) < 2: - return False + raise UsageError() msg = dict(x for x in [x.split('=', 1) for x in opts.spargs]) for x in args[1:]: msg.update(name=x) @@ -44,4 +44,4 @@ class Command(runserver.Command): queue.clear() print "Cleared %s queue" % botname else: - return False + raise UsageError() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 84253c66d..e9804e061 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -1,9 +1,9 @@ import sys import os -from scrapy import log from scrapy.utils.spider import iter_spider_classes from scrapy.command import ScrapyCommand +from scrapy.exceptions import UsageError def _import_file(filepath): abspath = os.path.abspath(filepath) @@ -33,22 +33,22 @@ class Command(ScrapyCommand): def long_desc(self): return "Run the spider defined in the given file" + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + def run(self, args, opts): if len(args) != 1: - return False + raise UsageError() filename = args[0] if not os.path.exists(filename): - log.msg("File not found: %s\n" % filename, log.ERROR) - return + raise UsageError("File not found: %s\n" % filename) try: module = _import_file(filename) except (ImportError, ValueError), e: - log.msg("Unable to load %r: %s\n" % (filename, e), log.ERROR) - return + raise UsageError("Unable to load %r: %s\n" % (filename, e)) spclasses = list(iter_spider_classes(module)) if not spclasses: - log.msg("No spider found in file: %s\n" % filename, log.ERROR) - return + raise UsageError("No spider found in file: %s\n" % filename) spider = spclasses.pop()() # schedule spider and start engine self.crawler.queue.append_spider(spider) diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index b398f78e0..0b39630ac 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -8,6 +8,7 @@ import scrapy from scrapy.command import ScrapyCommand from scrapy.utils.template import render_templatefile, string_camelcase from scrapy.utils.py26 import ignore_patterns, copytree +from scrapy.exceptions import UsageError TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project') @@ -33,7 +34,7 @@ class Command(ScrapyCommand): def run(self, args, opts): if len(args) != 1: - return False + raise UsageError() project_name = args[0] if not re.search(r'^[_a-zA-Z]\w*$', project_name): print 'Error: Project names must begin with a letter and contain only\n' \ diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py new file mode 100644 index 000000000..4d4557357 --- /dev/null +++ b/scrapy/commands/version.py @@ -0,0 +1,10 @@ +import scrapy +from scrapy.command import ScrapyCommand + +class Command(ScrapyCommand): + + def short_desc(self): + return "Print Scrapy version" + + def run(self, args, opts): + print "Scrapy %s" % scrapy.__version__ diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 48204f148..3294dc4da 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -38,3 +38,11 @@ class DropItem(Exception): class NotSupported(Exception): """Indicates a feature or method is not supported""" pass + +# Commands + +class UsageError(Exception): + """To indicate a command-line usage error""" + def __init__(self, *a, **kw): + self.print_help = kw.pop('print_help', True) + super(UsageError, self).__init__(*a, **kw) diff --git a/scrapy/tests/test_cmdline/__init__.py b/scrapy/tests/test_cmdline/__init__.py index 697ebb512..3588436b9 100644 --- a/scrapy/tests/test_cmdline/__init__.py +++ b/scrapy/tests/test_cmdline/__init__.py @@ -23,11 +23,6 @@ class CmdlineTest(unittest.TestCase): self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ 'default + loaded + started') - def test_override_settings_using_settings_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', \ - '--settings', 'scrapy.tests.test_cmdline.settings2'), \ - 'override + loaded + started') - def test_override_settings_using_set_arg(self): self.assertEqual(self._execute('settings', '--get', 'TEST1', '--set', 'TEST1=override'), \ 'override + loaded + started') diff --git a/scrapy/tests/test_cmdline/settings2.py b/scrapy/tests/test_cmdline/settings2.py deleted file mode 100644 index 4b5ba9e16..000000000 --- a/scrapy/tests/test_cmdline/settings2.py +++ /dev/null @@ -1,5 +0,0 @@ -EXTENSIONS = [ - 'scrapy.tests.test_cmdline.extensions.TestExtension' -] - -TEST1 = 'override' diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 7ece8a62b..47f67338a 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -69,7 +69,7 @@ class GenspiderCommandTest(CommandTest): def test_arguments(self): # only pass one argument. spider script shouldn't be created - self.assertEqual(0, self.call('genspider', 'test_name')) + self.assertEqual(2, self.call('genspider', 'test_name')) assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py')) # pass two arguments . spider script should be created self.assertEqual(0, self.call('genspider', 'test_name', 'test.com')) @@ -147,12 +147,12 @@ from scrapy.spider import BaseSpider """) p = self.proc('runspider', fname) log = p.stderr.read() - self.assert_("ERROR: No spider found in file" in log) + self.assert_("No spider found in file" in log) def test_runspider_file_not_found(self): p = self.proc('runspider', 'some_non_existent_file') log = p.stderr.read() - self.assert_("ERROR: File not found: some_non_existent_file" in log) + self.assert_("File not found: some_non_existent_file" in log) def test_runspider_unable_to_load(self): tmpdir = self.mktemp() @@ -162,5 +162,5 @@ from scrapy.spider import BaseSpider f.write("") p = self.proc('runspider', fname) log = p.stderr.read() - self.assert_("ERROR: Unable to load" in log) + self.assert_("Unable to load" in log)