mirror of https://github.com/scrapy/scrapy.git
moved scrapy.command.cmdline module to scrapy.cmdline (keeping backwards compatibility until 0.10)
--HG-- rename : scrapy/command/cmdline.py => scrapy/cmdline.py
This commit is contained in:
Pablo Hoffman
parent
56abafec61
commit
14bfeabede
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
from scrapy.cmdline import execute
|
||||
execute()
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@
|
|||
import os
|
||||
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings')
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
from scrapy.cmdline import execute
|
||||
execute()
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@
|
|||
import os
|
||||
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'imdb.settings')
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
from scrapy.cmdline import execute
|
||||
execute()
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@
|
|||
import os
|
||||
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', 'googledir.settings')
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
from scrapy.cmdline import execute
|
||||
execute()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,155 @@
|
|||
from __future__ import with_statement
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import cProfile
|
||||
|
||||
import scrapy
|
||||
from scrapy import log
|
||||
from scrapy.xlib import lsprofcalltree
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command.models import ScrapyCommand
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
|
||||
# Signal that carries information about the command which was executed
|
||||
# args: cmdname, cmdobj, args, opts
|
||||
command_executed = object()
|
||||
|
||||
def _find_commands(dir):
|
||||
try:
|
||||
return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \
|
||||
f.endswith('.py')]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def _get_commands_from_module(module):
|
||||
d = {}
|
||||
mod = __import__(module, {}, {}, [''])
|
||||
for cmdname in _find_commands(mod.__path__[0]):
|
||||
modname = '%s.%s' % (module, cmdname)
|
||||
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
|
||||
if callable(command):
|
||||
d[cmdname] = command()
|
||||
else:
|
||||
print 'WARNING: Module %r does not define a Command class' % modname
|
||||
return d
|
||||
|
||||
def _get_commands_dict():
|
||||
cmds = _get_commands_from_module('scrapy.commands')
|
||||
cmds_module = settings['COMMANDS_MODULE']
|
||||
if cmds_module:
|
||||
cmds.update(_get_commands_from_module(cmds_module))
|
||||
return cmds
|
||||
|
||||
def _get_command_name(argv):
|
||||
for arg in argv[1:]:
|
||||
if not arg.startswith('-'):
|
||||
return arg
|
||||
|
||||
def _print_usage(inside_project):
|
||||
if inside_project:
|
||||
print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
|
||||
settings['BOT_NAME'])
|
||||
else:
|
||||
print "Scrapy %s - no active project\n" % scrapy.__version__
|
||||
print "Usage"
|
||||
print "=====\n"
|
||||
print "To run a command:"
|
||||
print " scrapy-ctl.py <command> [options] [args]\n"
|
||||
print "To get help:"
|
||||
print " scrapy-ctl.py <command> -h\n"
|
||||
print "Available commands"
|
||||
print "==================\n"
|
||||
cmds = _get_commands_dict()
|
||||
for cmdname, cmdclass in sorted(cmds.iteritems()):
|
||||
if inside_project or not cmdclass.requires_project:
|
||||
print "%s %s" % (cmdname, cmdclass.syntax())
|
||||
print " %s" % cmdclass.short_desc()
|
||||
print
|
||||
|
||||
def _update_default_settings(module, cmdname):
|
||||
if not module:
|
||||
return
|
||||
try:
|
||||
mod = __import__('%s.%s' % (module, cmdname), {}, {}, [''])
|
||||
except ImportError:
|
||||
return
|
||||
settingsdict = vars(mod)
|
||||
for k, v in settingsdict.iteritems():
|
||||
if not k.startswith("_"):
|
||||
settings.defaults[k] = v
|
||||
|
||||
def execute(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv
|
||||
|
||||
cmds = _get_commands_dict()
|
||||
|
||||
cmdname = _get_command_name(argv)
|
||||
_update_default_settings('scrapy.conf.commands', cmdname)
|
||||
_update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname)
|
||||
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
conflict_handler='resolve', add_help_option=False)
|
||||
|
||||
if cmdname in cmds:
|
||||
cmd = cmds[cmdname]
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
cmd.process_options(args, opts)
|
||||
parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
|
||||
parser.description = cmd.long_desc()
|
||||
if cmd.requires_project and not settings.settings_module:
|
||||
print "Error running: scrapy-ctl.py %s\n" % cmdname
|
||||
print "Cannot find project settings module in python path: %s" % \
|
||||
settings.settings_module_path
|
||||
sys.exit(1)
|
||||
if opts.help:
|
||||
parser.print_help()
|
||||
sys.exit()
|
||||
elif not cmdname:
|
||||
cmd = ScrapyCommand()
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv)
|
||||
cmd.process_options(args, opts)
|
||||
_print_usage(settings.settings_module)
|
||||
sys.exit(2)
|
||||
else:
|
||||
print "Unknown command: %s\n" % cmdname
|
||||
print 'Use "scrapy-ctl.py -h" for help'
|
||||
sys.exit(2)
|
||||
|
||||
del args[0] # remove command name from args
|
||||
send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \
|
||||
args=args, opts=opts)
|
||||
from scrapy.core.manager import scrapymanager
|
||||
scrapymanager.configure(control_reactor=True)
|
||||
ret = _run_command(cmd, args, opts)
|
||||
if ret is False:
|
||||
parser.print_help()
|
||||
|
||||
def _run_command(cmd, args, opts):
|
||||
if opts.profile or opts.lsprof:
|
||||
return _run_command_profiled(cmd, args, opts)
|
||||
else:
|
||||
return cmd.run(args, opts)
|
||||
|
||||
def _run_command_profiled(cmd, args, opts):
|
||||
if opts.profile:
|
||||
log.msg("writing cProfile stats to %r" % opts.profile)
|
||||
if opts.lsprof:
|
||||
log.msg("writing lsprof stats to %r" % opts.lsprof)
|
||||
loc = locals()
|
||||
p = cProfile.Profile()
|
||||
p.runctx('ret = cmd.run(args, opts)', globals(), loc)
|
||||
if opts.profile:
|
||||
p.dump_stats(opts.profile)
|
||||
k = lsprofcalltree.KCacheGrind(p)
|
||||
if opts.lsprof:
|
||||
with open(opts.lsprof, 'w') as f:
|
||||
k.output(f)
|
||||
return loc['ret']
|
||||
|
||||
if __name__ == '__main__':
|
||||
execute()
|
||||
|
|
@ -1,155 +1,10 @@
|
|||
from __future__ import with_statement
|
||||
# TODO: remove this module for Scrapy 0.10
|
||||
import warnings
|
||||
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import cProfile
|
||||
from scrapy import cmdline
|
||||
|
||||
import scrapy
|
||||
from scrapy import log
|
||||
from scrapy.xlib import lsprofcalltree
|
||||
from scrapy.conf import settings
|
||||
from scrapy.command.models import ScrapyCommand
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
|
||||
# Signal that carries information about the command which was executed
|
||||
# args: cmdname, cmdobj, args, opts
|
||||
command_executed = object()
|
||||
|
||||
def _find_commands(dir):
|
||||
try:
|
||||
return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \
|
||||
f.endswith('.py')]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
def _get_commands_from_module(module):
|
||||
d = {}
|
||||
mod = __import__(module, {}, {}, [''])
|
||||
for cmdname in _find_commands(mod.__path__[0]):
|
||||
modname = '%s.%s' % (module, cmdname)
|
||||
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
|
||||
if callable(command):
|
||||
d[cmdname] = command()
|
||||
else:
|
||||
print 'WARNING: Module %r does not define a Command class' % modname
|
||||
return d
|
||||
|
||||
def _get_commands_dict():
|
||||
cmds = _get_commands_from_module('scrapy.commands')
|
||||
cmds_module = settings['COMMANDS_MODULE']
|
||||
if cmds_module:
|
||||
cmds.update(_get_commands_from_module(cmds_module))
|
||||
return cmds
|
||||
|
||||
def _get_command_name(argv):
|
||||
for arg in argv[1:]:
|
||||
if not arg.startswith('-'):
|
||||
return arg
|
||||
|
||||
def _print_usage(inside_project):
|
||||
if inside_project:
|
||||
print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
|
||||
settings['BOT_NAME'])
|
||||
else:
|
||||
print "Scrapy %s - no active project\n" % scrapy.__version__
|
||||
print "Usage"
|
||||
print "=====\n"
|
||||
print "To run a command:"
|
||||
print " scrapy-ctl.py <command> [options] [args]\n"
|
||||
print "To get help:"
|
||||
print " scrapy-ctl.py <command> -h\n"
|
||||
print "Available commands"
|
||||
print "==================\n"
|
||||
cmds = _get_commands_dict()
|
||||
for cmdname, cmdclass in sorted(cmds.iteritems()):
|
||||
if inside_project or not cmdclass.requires_project:
|
||||
print "%s %s" % (cmdname, cmdclass.syntax())
|
||||
print " %s" % cmdclass.short_desc()
|
||||
print
|
||||
|
||||
def _update_default_settings(module, cmdname):
|
||||
if not module:
|
||||
return
|
||||
try:
|
||||
mod = __import__('%s.%s' % (module, cmdname), {}, {}, [''])
|
||||
except ImportError:
|
||||
return
|
||||
settingsdict = vars(mod)
|
||||
for k, v in settingsdict.iteritems():
|
||||
if not k.startswith("_"):
|
||||
settings.defaults[k] = v
|
||||
|
||||
def execute(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv
|
||||
|
||||
cmds = _get_commands_dict()
|
||||
|
||||
cmdname = _get_command_name(argv)
|
||||
_update_default_settings('scrapy.conf.commands', cmdname)
|
||||
_update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname)
|
||||
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
conflict_handler='resolve', add_help_option=False)
|
||||
|
||||
if cmdname in cmds:
|
||||
cmd = cmds[cmdname]
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv[1:])
|
||||
cmd.process_options(args, opts)
|
||||
parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
|
||||
parser.description = cmd.long_desc()
|
||||
if cmd.requires_project and not settings.settings_module:
|
||||
print "Error running: scrapy-ctl.py %s\n" % cmdname
|
||||
print "Cannot find project settings module in python path: %s" % \
|
||||
settings.settings_module_path
|
||||
sys.exit(1)
|
||||
if opts.help:
|
||||
parser.print_help()
|
||||
sys.exit()
|
||||
elif not cmdname:
|
||||
cmd = ScrapyCommand()
|
||||
cmd.add_options(parser)
|
||||
opts, args = parser.parse_args(args=argv)
|
||||
cmd.process_options(args, opts)
|
||||
_print_usage(settings.settings_module)
|
||||
sys.exit(2)
|
||||
else:
|
||||
print "Unknown command: %s\n" % cmdname
|
||||
print 'Use "scrapy-ctl.py -h" for help'
|
||||
sys.exit(2)
|
||||
|
||||
del args[0] # remove command name from args
|
||||
send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \
|
||||
args=args, opts=opts)
|
||||
from scrapy.core.manager import scrapymanager
|
||||
scrapymanager.configure(control_reactor=True)
|
||||
ret = _run_command(cmd, args, opts)
|
||||
if ret is False:
|
||||
parser.print_help()
|
||||
|
||||
def _run_command(cmd, args, opts):
|
||||
if opts.profile or opts.lsprof:
|
||||
return _run_command_profiled(cmd, args, opts)
|
||||
else:
|
||||
return cmd.run(args, opts)
|
||||
|
||||
def _run_command_profiled(cmd, args, opts):
|
||||
if opts.profile:
|
||||
log.msg("writing cProfile stats to %r" % opts.profile)
|
||||
if opts.lsprof:
|
||||
log.msg("writing lsprof stats to %r" % opts.lsprof)
|
||||
loc = locals()
|
||||
p = cProfile.Profile()
|
||||
p.runctx('ret = cmd.run(args, opts)', globals(), loc)
|
||||
if opts.profile:
|
||||
p.dump_stats(opts.profile)
|
||||
k = lsprofcalltree.KCacheGrind(p)
|
||||
if opts.lsprof:
|
||||
with open(opts.lsprof, 'w') as f:
|
||||
k.output(f)
|
||||
return loc['ret']
|
||||
|
||||
if __name__ == '__main__':
|
||||
execute()
|
||||
def execute():
|
||||
warnings.warn("scrapy.command.cmdline.execute() is deprecated, modify your " \
|
||||
"project-ctl.py script to use scrapy.cmdline.execute() instead", \
|
||||
DeprecationWarning, stacklevel=2)
|
||||
cmdline.execute()
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@
|
|||
import os
|
||||
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', '${project_name}.settings')
|
||||
|
||||
from scrapy.command.cmdline import execute
|
||||
from scrapy.cmdline import execute
|
||||
execute()
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class CmdlineTest(unittest.TestCase):
|
|||
self.env['SCRAPY_SETTINGS_MODULE'] = 'scrapy.tests.test_cmdline.settings'
|
||||
|
||||
def _execute(self, *new_args, **kwargs):
|
||||
args = (sys.executable, '-m', 'scrapy.command.cmdline') + new_args
|
||||
args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
|
||||
proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
|
||||
comm = proc.communicate()
|
||||
return comm[0].strip()
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class ProjectTest(unittest.TestCase):
|
|||
|
||||
def call(self, *new_args, **kwargs):
|
||||
out = os.tmpfile()
|
||||
args = (sys.executable, '-m', 'scrapy.command.cmdline') + new_args
|
||||
args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
|
||||
return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, \
|
||||
env=self.env, **kwargs)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue