RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Yet another scrapy.cmdline code refactoring by removing --settings and --version options, adding a version command and adding a UsageError exception for signaling usage errors. Updated all commands accordingly

This commit is contained in:
Pablo Hoffman 2010-08-28 18:06:51 -03:00
parent 7394aa926d
commit 59f09c50e4
14 changed files with 111 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
scrapy/cmdline.py
View File

@ -6,11 +6,10 @@ import optparse
import cProfile
import scrapy
from scrapy import log
from scrapy.crawler import CrawlerProcess
from scrapy.xlib import lsprofcalltree
from scrapy.conf import settings
from scrapy.command import ScrapyCommand
from scrapy.exceptions import UsageError
def _find_commands(dir):
try:
@ -19,47 +18,61 @@ def _find_commands(dir):
except OSError:
return []
def _get_commands_from_module(module):
def _get_commands_from_module(module, inproject):
d = {}
mod = __import__(module, {}, {}, [''])
for cmdname in _find_commands(mod.__path__[0]):
modname = '%s.%s' % (module, cmdname)
command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None)
if callable(command):
d[cmdname] = command()
if inproject or not command.requires_project:
d[cmdname] = command()
else:
print 'WARNING: Module %r does not define a Command class' % modname
raise RuntimeError("Module %r does not define a Command class" % modname)
return d
def _get_commands_dict():
cmds = _get_commands_from_module('scrapy.commands')
def _get_commands_dict(inproject):
cmds = _get_commands_from_module('scrapy.commands', inproject)
cmds_module = settings['COMMANDS_MODULE']
if cmds_module:
cmds.update(_get_commands_from_module(cmds_module))
cmds.update(_get_commands_from_module(cmds_module, inproject))
return cmds
def _get_command_name(argv):
def _pop_command_name(argv):
i = 0
for arg in argv[1:]:
if not arg.startswith('-'):
del argv[i]
return arg
i += 1
def _print_usage(inside_project):
if inside_project:
def _print_header(inproject):
if inproject:
print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
settings['BOT_NAME'])
else:
print "Scrapy %s - no active project\n" % scrapy.__version__
def _print_commands(inproject):
_print_header(inproject)
print "Usage:"
print " scrapy <command> [options] [args]\n"
print "Available commands:"
cmds = _get_commands_dict()
cmds = _get_commands_dict(inproject)
for cmdname, cmdclass in sorted(cmds.iteritems()):
if inside_project or not cmdclass.requires_project:
print " %-13s %s" % (cmdname, cmdclass.short_desc())
print " %-13s %s" % (cmdname, cmdclass.short_desc())
print
print 'Use "scrapy <command> -h" for more info about a command'
print 'Use "scrapy <command> -h" to see more info about a command'
def check_deprecated_scrapy_ctl(argv):
def _print_unknown_command(cmdname, inproject):
_print_header(inproject)
print "Unknown command: %s\n" % cmdname
print 'Use "scrapy" to see available commands'
if not inproject:
print
print "More commands are available in project mode"
def _check_deprecated_scrapy_ctl(argv, inproject):
"""Check if Scrapy was called using the deprecated scrapy-ctl command and
warn in that case, also creating a scrapy.cfg if it doesn't exist.
"""
@ -68,7 +81,7 @@ def check_deprecated_scrapy_ctl(argv):
import warnings
warnings.warn("`scrapy-ctl.py` command-line tool is deprecated and will be removed in Scrapy 0.11, use `scrapy` instead",
DeprecationWarning, stacklevel=3)
if settings.settings_module:
if inproject:
projpath = os.path.abspath(os.path.dirname(os.path.dirname(settings.settings_module.__file__)))
cfg_path = os.path.join(projpath, 'scrapy.cfg')
if not os.path.exists(cfg_path):
@ -77,72 +90,64 @@ def check_deprecated_scrapy_ctl(argv):
f.write("[default]" + os.linesep)
f.write("settings = %s" % settings.settings_module_path + os.linesep)
def _run_print_help(parser, func, *a, **kw):
try:
func(*a, **kw)
except UsageError, e:
if str(e):
parser.error(str(e))
if e.print_help:
parser.print_help()
sys.exit(2)
def execute(argv=None):
if argv is None:
argv = sys.argv
crawler = CrawlerProcess(settings)
crawler.install()
check_deprecated_scrapy_ctl(argv) # TODO: remove for Scrapy 0.11
cmds = _get_commands_dict()
cmdname = _get_command_name(argv)
inproject = bool(settings.settings_module)
_check_deprecated_scrapy_ctl(argv, inproject) # TODO: remove for Scrapy 0.11
cmds = _get_commands_dict(inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
conflict_handler='resolve', add_help_option=False)
if cmdname in cmds:
cmd = cmds[cmdname]
cmd.add_options(parser)
opts, args = parser.parse_args(args=argv[1:])
cmd.process_options(args, opts)
parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax())
parser.description = cmd.long_desc()
if cmd.requires_project and not settings.settings_module:
print "Error running: scrapy %s\n" % cmdname
print "Cannot find project settings module in python path: %s" % \
settings.settings_module_path
sys.exit(1)
if opts.help:
parser.print_help()
sys.exit()
elif not cmdname:
cmd = ScrapyCommand()
cmd.add_options(parser)
opts, args = parser.parse_args(args=argv)
cmd.process_options(args, opts)
_print_usage(settings.settings_module)
sys.exit(2)
else:
print "Unknown command: %s\n" % cmdname
print 'Use "scrapy -h" for help'
conflict_handler='resolve')
if not cmdname:
_print_commands(inproject)
sys.exit(0)
elif cmdname not in cmds:
_print_unknown_command(cmdname, inproject)
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = "scrapy %s %s" % (cmdname, cmd.syntax())
parser.description = cmd.long_desc()
settings.defaults.update(cmd.default_settings)
del args[0] # remove command name from args
log.start()
cmd.set_crawler(crawler)
ret = _run_command(cmd, args, opts)
if ret is False:
parser.print_help()
cmd.add_options(parser)
opts, args = parser.parse_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
_run_print_help(parser, _run_command, cmd, args, opts)
def _run_command(cmd, args, opts):
if opts.profile or opts.lsprof:
return _run_command_profiled(cmd, args, opts)
_run_command_profiled(cmd, args, opts)
else:
return cmd.run(args, opts)
cmd.run(args, opts)
def _run_command_profiled(cmd, args, opts):
if opts.profile:
log.msg("writing cProfile stats to %r" % opts.profile)
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
if opts.lsprof:
log.msg("writing lsprof stats to %r" % opts.lsprof)
sys.stderr.write("scrapy: writing lsprof stats to %r\n" % opts.lsprof)
loc = locals()
p = cProfile.Profile()
p.runctx('ret = cmd.run(args, opts)', globals(), loc)
p.runctx('cmd.run(args, opts)', globals(), loc)
if opts.profile:
p.dump_stats(opts.profile)
k = lsprofcalltree.KCacheGrind(p)
if opts.lsprof:
with open(opts.lsprof, 'w') as f:
k.output(f)
return loc['ret']
if __name__ == '__main__':
execute()

21
scrapy/command.py
View File

@ -9,8 +9,10 @@ import sys
from optparse import OptionGroup
import scrapy
from scrapy import log
from scrapy.conf import settings
from scrapy.utils.conf import arglist_to_dict
from scrapy.exceptions import UsageError
class ScrapyCommand(object):
@ -24,6 +26,8 @@ class ScrapyCommand(object):
@property
def crawler(self):
if not log.started:
log.start()
self._crawler.configure()
return self._crawler
@ -58,10 +62,6 @@ class ScrapyCommand(object):
Populate option parse with options available for this command
"""
group = OptionGroup(parser, "Global Options")
group.add_option("-h", "--help", action="store_true", dest="help", \
help="print command help and options")
group.add_option("--version", action="store_true", dest="version", \
help="print Scrapy version and exit")
group.add_option("--logfile", dest="logfile", metavar="FILE", \
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", dest="loglevel", metavar="LEVEL", \
@ -77,23 +77,13 @@ class ScrapyCommand(object):
help="write process ID to FILE")
group.add_option("--set", dest="set", action="append", default=[], metavar="NAME=VALUE", \
help="set/override setting (may be repeated)")
group.add_option("--settings", dest="settings", metavar="MODULE",
help="python path to the Scrapy project settings")
parser.add_option_group(group)
def process_options(self, args, opts):
if opts.settings:
settings.set_settings_module(opts.settings)
try:
settings.overrides.update(arglist_to_dict(opts.set))
except ValueError:
sys.stderr.write("Invalid --set value, use --set NAME=VALUE\n")
sys.exit(2)
if opts.version:
print "Scrapy %s" % scrapy.__version__
sys.exit()
raise UsageError("Invalid --set value, use --set NAME=VALUE", print_help=False)
if opts.logfile:
settings.overrides['LOG_ENABLED'] = True
@ -115,4 +105,3 @@ class ScrapyCommand(object):
Entry point for running commands
"""
raise NotImplementedError

4
scrapy/commands/crawl.py
View File

@ -6,6 +6,7 @@ from scrapy.conf import settings
from scrapy.http import Request
from scrapy.utils.url import is_url
from scrapy.utils.conf import arglist_to_dict
from scrapy.exceptions import UsageError
from collections import defaultdict
@ -33,8 +34,7 @@ class Command(ScrapyCommand):
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
sys.stderr.write("Invalid -a value, use -a NAME=VALUE\n")
sys.exit(2)
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.nofollow:
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False

3
scrapy/commands/fetch.py
View File

@ -5,6 +5,7 @@ from scrapy.command import ScrapyCommand
from scrapy.http import Request
from scrapy.spider import BaseSpider
from scrapy.utils.url import is_url
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
@ -35,7 +36,7 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
return False
raise UsageError()
cb = lambda x: self._print_response(x, opts)
request = Request(args[0], callback=cb, dont_filter=True)

3
scrapy/commands/genspider.py
View File

@ -7,6 +7,7 @@ import scrapy
from scrapy.command import ScrapyCommand
from scrapy.conf import settings
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.exceptions import UsageError
def sanitize_module_name(module_name):
"""Sanitize the given module name, by replacing dashes and points
@ -54,7 +55,7 @@ class Command(ScrapyCommand):
print open(template_file, 'r').read()
return
if len(args) != 2:
return False
raise UsageError()
name, domain = args[0:2]
module = sanitize_module_name(name)

3
scrapy/commands/parse.py
View File

@ -4,6 +4,7 @@ from scrapy.item import BaseItem
from scrapy.utils import display
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.url import is_url
from scrapy.exceptions import UsageError
from scrapy import log
class Command(ScrapyCommand):
@ -93,7 +94,7 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if not len(args) == 1 or not is_url(args[0]):
return False
raise UsageError()
response, spider = self.get_response_and_spider(args[0], opts)
if not response:
return

8
scrapy/commands/queue.py
View File

@ -1,6 +1,6 @@
from scrapy.command import ScrapyCommand
from scrapy.commands import runserver
from scrapy.utils.misc import load_object
from scrapy.exceptions import UsageError
from scrapy.conf import settings
class Command(runserver.Command):
@ -23,7 +23,7 @@ class Command(runserver.Command):
def run(self, args, opts):
if len(args) < 1:
return False
raise UsageError()
cmd = args[0]
botname = settings['BOT_NAME']
@ -31,7 +31,7 @@ class Command(runserver.Command):
if cmd == 'add':
if len(args) < 2:
return False
raise UsageError()
msg = dict(x for x in [x.split('=', 1) for x in opts.spargs])
for x in args[1:]:
msg.update(name=x)
@ -44,4 +44,4 @@ class Command(runserver.Command):
queue.clear()
print "Cleared %s queue" % botname
else:
return False
raise UsageError()

16
scrapy/commands/runspider.py
View File

@ -1,9 +1,9 @@
import sys
import os
from scrapy import log
from scrapy.utils.spider import iter_spider_classes
from scrapy.command import ScrapyCommand
from scrapy.exceptions import UsageError
def _import_file(filepath):
abspath = os.path.abspath(filepath)
@ -33,22 +33,22 @@ class Command(ScrapyCommand):
def long_desc(self):
return "Run the spider defined in the given file"
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
def run(self, args, opts):
if len(args) != 1:
return False
raise UsageError()
filename = args[0]
if not os.path.exists(filename):
log.msg("File not found: %s\n" % filename, log.ERROR)
return
raise UsageError("File not found: %s\n" % filename)
try:
module = _import_file(filename)
except (ImportError, ValueError), e:
log.msg("Unable to load %r: %s\n" % (filename, e), log.ERROR)
return
raise UsageError("Unable to load %r: %s\n" % (filename, e))
spclasses = list(iter_spider_classes(module))
if not spclasses:
log.msg("No spider found in file: %s\n" % filename, log.ERROR)
return
raise UsageError("No spider found in file: %s\n" % filename)
spider = spclasses.pop()()
# schedule spider and start engine
self.crawler.queue.append_spider(spider)

3
scrapy/commands/startproject.py
View File

@ -8,6 +8,7 @@ import scrapy
from scrapy.command import ScrapyCommand
from scrapy.utils.template import render_templatefile, string_camelcase
from scrapy.utils.py26 import ignore_patterns, copytree
from scrapy.exceptions import UsageError
TEMPLATES_PATH = join(scrapy.__path__[0], 'templates', 'project')
@ -33,7 +34,7 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1:
return False
raise UsageError()
project_name = args[0]
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
print 'Error: Project names must begin with a letter and contain only\n' \

10
scrapy/commands/version.py Normal file
View File

@ -0,0 +1,10 @@
import scrapy
from scrapy.command import ScrapyCommand
class Command(ScrapyCommand):
def short_desc(self):
return "Print Scrapy version"
def run(self, args, opts):
print "Scrapy %s" % scrapy.__version__

8
scrapy/exceptions.py
View File

@ -38,3 +38,11 @@ class DropItem(Exception):
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
pass
# Commands
class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)

5
scrapy/tests/test_cmdline/__init__.py
View File

@ -23,11 +23,6 @@ class CmdlineTest(unittest.TestCase):
self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
'default + loaded + started')
def test_override_settings_using_settings_arg(self):
self.assertEqual(self._execute('settings', '--get', 'TEST1', \
'--settings', 'scrapy.tests.test_cmdline.settings2'), \
'override + loaded + started')
def test_override_settings_using_set_arg(self):
self.assertEqual(self._execute('settings', '--get', 'TEST1', '--set', 'TEST1=override'), \
'override + loaded + started')

5
scrapy/tests/test_cmdline/settings2.py
View File

@ -1,5 +0,0 @@
EXTENSIONS = [
'scrapy.tests.test_cmdline.extensions.TestExtension'
]
TEST1 = 'override'

8
scrapy/tests/test_commands.py
View File

@ -69,7 +69,7 @@ class GenspiderCommandTest(CommandTest):
def test_arguments(self):
# only pass one argument. spider script shouldn't be created
self.assertEqual(0, self.call('genspider', 'test_name'))
self.assertEqual(2, self.call('genspider', 'test_name'))
assert not exists(join(self.proj_mod_path, 'spiders', 'test_name.py'))
# pass two arguments <name> <domain>. spider script should be created
self.assertEqual(0, self.call('genspider', 'test_name', 'test.com'))
@ -147,12 +147,12 @@ from scrapy.spider import BaseSpider
""")
p = self.proc('runspider', fname)
log = p.stderr.read()
self.assert_("ERROR: No spider found in file" in log)
self.assert_("No spider found in file" in log)
def test_runspider_file_not_found(self):
p = self.proc('runspider', 'some_non_existent_file')
log = p.stderr.read()
self.assert_("ERROR: File not found: some_non_existent_file" in log)
self.assert_("File not found: some_non_existent_file" in log)
def test_runspider_unable_to_load(self):
tmpdir = self.mktemp()
@ -162,5 +162,5 @@ from scrapy.spider import BaseSpider
f.write("")
p = self.proc('runspider', fname)
log = p.stderr.read()
self.assert_("ERROR: Unable to load" in log)
self.assert_("Unable to load" in log)