mirror of https://github.com/scrapy/scrapy.git
commit
3dff65ac2c
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import print_function
|
||||
import sys, os
|
||||
import sys
|
||||
import os
|
||||
import optparse
|
||||
import cProfile
|
||||
import inspect
|
||||
|
|
@ -14,6 +15,7 @@ from scrapy.utils.project import inside_project, get_project_settings
|
|||
from scrapy.utils.python import garbage_collect
|
||||
from scrapy.settings.deprecated import check_deprecated_settings
|
||||
|
||||
|
||||
def _iter_command_classes(module_name):
|
||||
# TODO: add `name` attribute to commands and and merge this function with
|
||||
# scrapy.utils.spider.iter_spider_classes
|
||||
|
|
@ -25,6 +27,7 @@ def _iter_command_classes(module_name):
|
|||
not obj == ScrapyCommand:
|
||||
yield obj
|
||||
|
||||
|
||||
def _get_commands_from_module(module, inproject):
|
||||
d = {}
|
||||
for cmd in _iter_command_classes(module):
|
||||
|
|
@ -33,6 +36,7 @@ def _get_commands_from_module(module, inproject):
|
|||
d[cmdname] = cmd()
|
||||
return d
|
||||
|
||||
|
||||
def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
|
||||
cmds = {}
|
||||
for entry_point in pkg_resources.iter_entry_points(group):
|
||||
|
|
@ -43,6 +47,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
|
|||
raise Exception("Invalid entry point %s" % entry_point.name)
|
||||
return cmds
|
||||
|
||||
|
||||
def _get_commands_dict(settings, inproject):
|
||||
cmds = _get_commands_from_module('scrapy.commands', inproject)
|
||||
cmds.update(_get_commands_from_entry_points(inproject))
|
||||
|
|
@ -51,6 +56,7 @@ def _get_commands_dict(settings, inproject):
|
|||
cmds.update(_get_commands_from_module(cmds_module, inproject))
|
||||
return cmds
|
||||
|
||||
|
||||
def _pop_command_name(argv):
|
||||
i = 0
|
||||
for arg in argv[1:]:
|
||||
|
|
@ -59,13 +65,15 @@ def _pop_command_name(argv):
|
|||
return arg
|
||||
i += 1
|
||||
|
||||
|
||||
def _print_header(settings, inproject):
|
||||
if inproject:
|
||||
print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
|
||||
settings['BOT_NAME']))
|
||||
settings['BOT_NAME']))
|
||||
else:
|
||||
print("Scrapy %s - no active project\n" % scrapy.__version__)
|
||||
|
||||
|
||||
def _print_commands(settings, inproject):
|
||||
_print_header(settings, inproject)
|
||||
print("Usage:")
|
||||
|
|
@ -80,11 +88,13 @@ def _print_commands(settings, inproject):
|
|||
print()
|
||||
print('Use "scrapy <command> -h" to see more info about a command')
|
||||
|
||||
|
||||
def _print_unknown_command(settings, cmdname, inproject):
|
||||
_print_header(settings, inproject)
|
||||
print("Unknown command: %s\n" % cmdname)
|
||||
print('Use "scrapy" to see available commands')
|
||||
|
||||
|
||||
def _run_print_help(parser, func, *a, **kw):
|
||||
try:
|
||||
func(*a, **kw)
|
||||
|
|
@ -95,6 +105,7 @@ def _run_print_help(parser, func, *a, **kw):
|
|||
parser.print_help()
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def execute(argv=None, settings=None):
|
||||
if argv is None:
|
||||
argv = sys.argv
|
||||
|
|
@ -104,7 +115,8 @@ def execute(argv=None, settings=None):
|
|||
# set EDITOR from environment if available
|
||||
try:
|
||||
editor = os.environ['EDITOR']
|
||||
except KeyError: pass
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
settings['EDITOR'] = editor
|
||||
check_deprecated_settings(settings)
|
||||
|
|
@ -113,7 +125,7 @@ def execute(argv=None, settings=None):
|
|||
cmds = _get_commands_dict(settings, inproject)
|
||||
cmdname = _pop_command_name(argv)
|
||||
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
|
||||
conflict_handler='resolve')
|
||||
conflict_handler='resolve')
|
||||
if not cmdname:
|
||||
_print_commands(settings, inproject)
|
||||
sys.exit(0)
|
||||
|
|
@ -134,12 +146,14 @@ def execute(argv=None, settings=None):
|
|||
_run_print_help(parser, _run_command, cmd, args, opts)
|
||||
sys.exit(cmd.exitcode)
|
||||
|
||||
|
||||
def _run_command(cmd, args, opts):
|
||||
if opts.profile:
|
||||
_run_command_profiled(cmd, args, opts)
|
||||
else:
|
||||
cmd.run(args, opts)
|
||||
|
||||
|
||||
def _run_command_profiled(cmd, args, opts):
|
||||
if opts.profile:
|
||||
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
|
||||
|
|
@ -149,6 +163,7 @@ def _run_command_profiled(cmd, args, opts):
|
|||
if opts.profile:
|
||||
p.dump_stats(opts.profile)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
execute()
|
||||
|
|
|
|||
Loading…
Reference in New Issue