diff --git a/bin/scrapy-ctl.py b/bin/scrapy-ctl.py index e8e56eaf2..918ea7fbd 100755 --- a/bin/scrapy-ctl.py +++ b/bin/scrapy-ctl.py @@ -1,4 +1,4 @@ #!/usr/bin/env python -from scrapy.command.cmdline import execute +from scrapy.cmdline import execute execute() diff --git a/examples/experimental/googledir/scrapy-ctl.py b/examples/experimental/googledir/scrapy-ctl.py index 552421ac3..60ca701cb 100644 --- a/examples/experimental/googledir/scrapy-ctl.py +++ b/examples/experimental/googledir/scrapy-ctl.py @@ -3,5 +3,5 @@ import os os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings') -from scrapy.command.cmdline import execute +from scrapy.cmdline import execute execute() diff --git a/examples/experimental/imdb/scrapy-ctl.py b/examples/experimental/imdb/scrapy-ctl.py index df57621b3..c1ff31537 100644 --- a/examples/experimental/imdb/scrapy-ctl.py +++ b/examples/experimental/imdb/scrapy-ctl.py @@ -3,5 +3,5 @@ import os os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'imdb.settings') -from scrapy.command.cmdline import execute +from scrapy.cmdline import execute execute() diff --git a/examples/googledir/scrapy-ctl.py b/examples/googledir/scrapy-ctl.py index 552421ac3..60ca701cb 100755 --- a/examples/googledir/scrapy-ctl.py +++ b/examples/googledir/scrapy-ctl.py @@ -3,5 +3,5 @@ import os os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings') -from scrapy.command.cmdline import execute +from scrapy.cmdline import execute execute() diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py new file mode 100644 index 000000000..2fe790a32 --- /dev/null +++ b/scrapy/cmdline.py @@ -0,0 +1,155 @@ +from __future__ import with_statement + +import sys +import os +import optparse +import cProfile + +import scrapy +from scrapy import log +from scrapy.xlib import lsprofcalltree +from scrapy.conf import settings +from scrapy.command.models import ScrapyCommand +from scrapy.utils.signal import send_catch_log + +# Signal that carries information about the command which was executed +# args: cmdname, cmdobj, args, opts +command_executed = object() + +def _find_commands(dir): + try: + return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \ + f.endswith('.py')] + except OSError: + return [] + +def _get_commands_from_module(module): + d = {} + mod = __import__(module, {}, {}, ['']) + for cmdname in _find_commands(mod.__path__[0]): + modname = '%s.%s' % (module, cmdname) + command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) + if callable(command): + d[cmdname] = command() + else: + print 'WARNING: Module %r does not define a Command class' % modname + return d + +def _get_commands_dict(): + cmds = _get_commands_from_module('scrapy.commands') + cmds_module = settings['COMMANDS_MODULE'] + if cmds_module: + cmds.update(_get_commands_from_module(cmds_module)) + return cmds + +def _get_command_name(argv): + for arg in argv[1:]: + if not arg.startswith('-'): + return arg + +def _print_usage(inside_project): + if inside_project: + print "Scrapy %s - project: %s\n" % (scrapy.__version__, \ + settings['BOT_NAME']) + else: + print "Scrapy %s - no active project\n" % scrapy.__version__ + print "Usage" + print "=====\n" + print "To run a command:" + print " scrapy-ctl.py [options] [args]\n" + print "To get help:" + print " scrapy-ctl.py -h\n" + print "Available commands" + print "==================\n" + cmds = _get_commands_dict() + for cmdname, cmdclass in sorted(cmds.iteritems()): + if inside_project or not cmdclass.requires_project: + print "%s %s" % (cmdname, cmdclass.syntax()) + print " %s" % cmdclass.short_desc() + print + +def _update_default_settings(module, cmdname): + if not module: + return + try: + mod = __import__('%s.%s' % (module, cmdname), {}, {}, ['']) + except ImportError: + return + settingsdict = vars(mod) + for k, v in settingsdict.iteritems(): + if not k.startswith("_"): + settings.defaults[k] = v + +def execute(argv=None): + if argv is None: + argv = sys.argv + + cmds = _get_commands_dict() + + cmdname = _get_command_name(argv) + _update_default_settings('scrapy.conf.commands', cmdname) + _update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname) + + parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ + conflict_handler='resolve', add_help_option=False) + + if cmdname in cmds: + cmd = cmds[cmdname] + cmd.add_options(parser) + opts, args = parser.parse_args(args=argv[1:]) + cmd.process_options(args, opts) + parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) + parser.description = cmd.long_desc() + if cmd.requires_project and not settings.settings_module: + print "Error running: scrapy-ctl.py %s\n" % cmdname + print "Cannot find project settings module in python path: %s" % \ + settings.settings_module_path + sys.exit(1) + if opts.help: + parser.print_help() + sys.exit() + elif not cmdname: + cmd = ScrapyCommand() + cmd.add_options(parser) + opts, args = parser.parse_args(args=argv) + cmd.process_options(args, opts) + _print_usage(settings.settings_module) + sys.exit(2) + else: + print "Unknown command: %s\n" % cmdname + print 'Use "scrapy-ctl.py -h" for help' + sys.exit(2) + + del args[0] # remove command name from args + send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \ + args=args, opts=opts) + from scrapy.core.manager import scrapymanager + scrapymanager.configure(control_reactor=True) + ret = _run_command(cmd, args, opts) + if ret is False: + parser.print_help() + +def _run_command(cmd, args, opts): + if opts.profile or opts.lsprof: + return _run_command_profiled(cmd, args, opts) + else: + return cmd.run(args, opts) + +def _run_command_profiled(cmd, args, opts): + if opts.profile: + log.msg("writing cProfile stats to %r" % opts.profile) + if opts.lsprof: + log.msg("writing lsprof stats to %r" % opts.lsprof) + loc = locals() + p = cProfile.Profile() + p.runctx('ret = cmd.run(args, opts)', globals(), loc) + if opts.profile: + p.dump_stats(opts.profile) + k = lsprofcalltree.KCacheGrind(p) + if opts.lsprof: + with open(opts.lsprof, 'w') as f: + k.output(f) + return loc['ret'] + +if __name__ == '__main__': + execute() diff --git a/scrapy/command/cmdline.py b/scrapy/command/cmdline.py index 2fe790a32..b22b57b34 100644 --- a/scrapy/command/cmdline.py +++ b/scrapy/command/cmdline.py @@ -1,155 +1,10 @@ -from __future__ import with_statement +# TODO: remove this module for Scrapy 0.10 +import warnings -import sys -import os -import optparse -import cProfile +from scrapy import cmdline -import scrapy -from scrapy import log -from scrapy.xlib import lsprofcalltree -from scrapy.conf import settings -from scrapy.command.models import ScrapyCommand -from scrapy.utils.signal import send_catch_log - -# Signal that carries information about the command which was executed -# args: cmdname, cmdobj, args, opts -command_executed = object() - -def _find_commands(dir): - try: - return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \ - f.endswith('.py')] - except OSError: - return [] - -def _get_commands_from_module(module): - d = {} - mod = __import__(module, {}, {}, ['']) - for cmdname in _find_commands(mod.__path__[0]): - modname = '%s.%s' % (module, cmdname) - command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) - if callable(command): - d[cmdname] = command() - else: - print 'WARNING: Module %r does not define a Command class' % modname - return d - -def _get_commands_dict(): - cmds = _get_commands_from_module('scrapy.commands') - cmds_module = settings['COMMANDS_MODULE'] - if cmds_module: - cmds.update(_get_commands_from_module(cmds_module)) - return cmds - -def _get_command_name(argv): - for arg in argv[1:]: - if not arg.startswith('-'): - return arg - -def _print_usage(inside_project): - if inside_project: - print "Scrapy %s - project: %s\n" % (scrapy.__version__, \ - settings['BOT_NAME']) - else: - print "Scrapy %s - no active project\n" % scrapy.__version__ - print "Usage" - print "=====\n" - print "To run a command:" - print " scrapy-ctl.py [options] [args]\n" - print "To get help:" - print " scrapy-ctl.py -h\n" - print "Available commands" - print "==================\n" - cmds = _get_commands_dict() - for cmdname, cmdclass in sorted(cmds.iteritems()): - if inside_project or not cmdclass.requires_project: - print "%s %s" % (cmdname, cmdclass.syntax()) - print " %s" % cmdclass.short_desc() - print - -def _update_default_settings(module, cmdname): - if not module: - return - try: - mod = __import__('%s.%s' % (module, cmdname), {}, {}, ['']) - except ImportError: - return - settingsdict = vars(mod) - for k, v in settingsdict.iteritems(): - if not k.startswith("_"): - settings.defaults[k] = v - -def execute(argv=None): - if argv is None: - argv = sys.argv - - cmds = _get_commands_dict() - - cmdname = _get_command_name(argv) - _update_default_settings('scrapy.conf.commands', cmdname) - _update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname) - - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ - conflict_handler='resolve', add_help_option=False) - - if cmdname in cmds: - cmd = cmds[cmdname] - cmd.add_options(parser) - opts, args = parser.parse_args(args=argv[1:]) - cmd.process_options(args, opts) - parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) - parser.description = cmd.long_desc() - if cmd.requires_project and not settings.settings_module: - print "Error running: scrapy-ctl.py %s\n" % cmdname - print "Cannot find project settings module in python path: %s" % \ - settings.settings_module_path - sys.exit(1) - if opts.help: - parser.print_help() - sys.exit() - elif not cmdname: - cmd = ScrapyCommand() - cmd.add_options(parser) - opts, args = parser.parse_args(args=argv) - cmd.process_options(args, opts) - _print_usage(settings.settings_module) - sys.exit(2) - else: - print "Unknown command: %s\n" % cmdname - print 'Use "scrapy-ctl.py -h" for help' - sys.exit(2) - - del args[0] # remove command name from args - send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \ - args=args, opts=opts) - from scrapy.core.manager import scrapymanager - scrapymanager.configure(control_reactor=True) - ret = _run_command(cmd, args, opts) - if ret is False: - parser.print_help() - -def _run_command(cmd, args, opts): - if opts.profile or opts.lsprof: - return _run_command_profiled(cmd, args, opts) - else: - return cmd.run(args, opts) - -def _run_command_profiled(cmd, args, opts): - if opts.profile: - log.msg("writing cProfile stats to %r" % opts.profile) - if opts.lsprof: - log.msg("writing lsprof stats to %r" % opts.lsprof) - loc = locals() - p = cProfile.Profile() - p.runctx('ret = cmd.run(args, opts)', globals(), loc) - if opts.profile: - p.dump_stats(opts.profile) - k = lsprofcalltree.KCacheGrind(p) - if opts.lsprof: - with open(opts.lsprof, 'w') as f: - k.output(f) - return loc['ret'] - -if __name__ == '__main__': - execute() +def execute(): + warnings.warn("scrapy.command.cmdline.execute() is deprecated, modify your " \ + "project-ctl.py script to use scrapy.cmdline.execute() instead", \ + DeprecationWarning, stacklevel=2) + cmdline.execute() diff --git a/scrapy/templates/project/scrapy-ctl.py b/scrapy/templates/project/scrapy-ctl.py index 0c5176f10..931d31afe 100755 --- a/scrapy/templates/project/scrapy-ctl.py +++ b/scrapy/templates/project/scrapy-ctl.py @@ -3,5 +3,5 @@ import os os.environ.setdefault('SCRAPY_SETTINGS_MODULE', '${project_name}.settings') -from scrapy.command.cmdline import execute +from scrapy.cmdline import execute execute() diff --git a/scrapy/tests/test_cmdline/__init__.py b/scrapy/tests/test_cmdline/__init__.py index 671b8f2fb..7c88870c3 100644 --- a/scrapy/tests/test_cmdline/__init__.py +++ b/scrapy/tests/test_cmdline/__init__.py @@ -14,7 +14,7 @@ class CmdlineTest(unittest.TestCase): self.env['SCRAPY_SETTINGS_MODULE'] = 'scrapy.tests.test_cmdline.settings' def _execute(self, *new_args, **kwargs): - args = (sys.executable, '-m', 'scrapy.command.cmdline') + new_args + args = (sys.executable, '-m', 'scrapy.cmdline') + new_args proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs) comm = proc.communicate() return comm[0].strip() diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index 6eed6d22a..a76ba08d8 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -25,7 +25,7 @@ class ProjectTest(unittest.TestCase): def call(self, *new_args, **kwargs): out = os.tmpfile() - args = (sys.executable, '-m', 'scrapy.command.cmdline') + new_args + args = (sys.executable, '-m', 'scrapy.cmdline') + new_args return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, \ env=self.env, **kwargs)