RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move from optparse to argparse (#5374)

This commit is contained in:
Alex 2022-02-09 11:52:07 -08:00 committed by GitHub
parent 1356836e7e
commit e2e2ffd0d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 168 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/cmdline.py
View File

@ -1,13 +1,13 @@
import sys import sys
import os import os
import optparse import argparse
import cProfile import cProfile
import inspect import inspect
import pkg_resources import pkg_resources
import scrapy import scrapy
from scrapy.crawler import CrawlerProcess from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings from scrapy.utils.project import inside_project, get_project_settings
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
inproject = inside_project() inproject = inside_project()
cmds = _get_commands_dict(settings, inproject) cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv) cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname: if not cmdname:
_print_commands(settings, inproject) _print_commands(settings, inproject)
sys.exit(0) sys.exit(0)
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
sys.exit(2) sys.exit(2)
cmd = cmds[cmdname] cmd = cmds[cmdname]
parser.usage = f"scrapy {cmdname} {cmd.syntax()}" parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
parser.description = cmd.long_desc() usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command') settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings cmd.settings = settings
cmd.add_options(parser) cmd.add_options(parser)
opts, args = parser.parse_args(args=argv[1:]) opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts) _run_print_help(parser, cmd.process_options, args, opts)
cmd.crawler_process = CrawlerProcess(settings) cmd.crawler_process = CrawlerProcess(settings)

75
scrapy/commands/__init__.py
View File

@ -2,7 +2,7 @@
Base class for Scrapy commands Base class for Scrapy commands
""" """
import os import os
from optparse import OptionGroup import argparse
from typing import Any, Dict from typing import Any, Dict
from twisted.python import failure from twisted.python import failure
@ -59,22 +59,20 @@ class ScrapyCommand:
""" """
Populate option parse with options available for this command Populate option parse with options available for this command
""" """
group = OptionGroup(parser, "Global Options") group = parser.add_argument_group(title='Global Options')
group.add_option("--logfile", metavar="FILE", group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used") help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})") help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true", group.add_argument("--nolog", action="store_true",
help="disable logging completely") help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None, group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE") help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE", group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE") help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)") help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure") group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)
def process_options(self, args, opts): def process_options(self, args, opts):
try: try:
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
""" """
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)") help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append", parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)") help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append", parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file") help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT", parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items") help="format to use for dumping items")
def process_options(self, args, opts): def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts) ScrapyCommand.process_options(self, args, opts)
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.overwrite_output, opts.overwrite_output,
) )
self.settings.set('FEEDS', feeds, priority='cmdline') self.settings.set('FEEDS', feeds, priority='cmdline')
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
return super()._join_parts(parts)
def format_part_strings(self, part_strings):
"""
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

8
scrapy/commands/check.py
View File

@ -49,10 +49,10 @@ class Command(ScrapyCommand):
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true", parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them") help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true', parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders") help="print contract tests for all spiders")
def run(self, args, opts): def run(self, args, opts):
# load contracts # load contracts

10
scrapy/commands/fetch.py
View File

@ -26,11 +26,11 @@ class Command(ScrapyCommand):
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", help="use this spider") parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true", parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body") help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is") help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix): def _print_headers(self, headers, prefix):
for key, values in headers.items(): for key, values in headers.items():

20
scrapy/commands/genspider.py
View File

@ -44,16 +44,16 @@ class Command(ScrapyCommand):
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true", parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates") help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true", parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it") help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output") help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic", parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.") help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true", parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template") help="If the spider already exists, overwrite it with the template")
def run(self, args, opts): def run(self, args, opts):
if opts.list: if opts.list:

44
scrapy/commands/parse.py
View File

@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser): def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser) BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None, parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one") help="use this spider without looking for one")
parser.add_option("--pipelines", action="store_true", parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines") help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true", parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)") help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true", parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items") help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true", parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output") help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true", parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback") help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback", parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback") help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta", parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string") help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs", parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string") help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1, parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %default]") help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true", parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one") help="print each depth level one by one")
@property @property
def max_level(self): def max_level(self):

20
scrapy/commands/settings.py
View File

@ -18,16 +18,16 @@ class Command(ScrapyCommand):
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING", parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value") help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING", parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean") help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING", parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer") help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float") help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING", parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list") help="print setting value, interpreted as a list")
def run(self, args, opts): def run(self, args, opts):
settings = self.crawler_process.settings settings = self.crawler_process.settings

12
scrapy/commands/shell.py
View File

@ -33,12 +33,12 @@ class Command(ScrapyCommand):
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code", parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit") help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider", parser.add_argument("--spider", dest="spider",
help="use this spider") help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is") help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars): def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be """You can use this function to update the Scrapy objects that will be

4
scrapy/commands/version.py
View File

@ -16,8 +16,8 @@ class Command(ScrapyCommand):
def add_options(self, parser): def add_options(self, parser):
ScrapyCommand.add_options(self, parser) ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true", parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)") help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts): def run(self, args, opts):
if opts.verbose: if opts.verbose:

3
scrapy/commands/view.py
View File

@ -1,3 +1,4 @@
import argparse
from scrapy.commands import fetch from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser from scrapy.utils.response import open_in_browser
@ -12,7 +13,7 @@ class Command(fetch.Command):
def add_options(self, parser): def add_options(self, parser):
super().add_options(parser) super().add_options(parser)
parser.remove_option("--headers") parser.add_argument('--headers', help=argparse.SUPPRESS)
def _print_response(self, response, opts): def _print_response(self, response, opts):
open_in_browser(response) open_in_browser(response)

19
tests/test_command_parse.py
View File

@ -1,6 +1,9 @@
import os import os
import argparse
from os.path import join, abspath, isfile, exists from os.path import join, abspath, isfile, exists
from twisted.internet import defer from twisted.internet import defer
from scrapy.commands import parse
from scrapy.settings import Settings
from scrapy.utils.testsite import SiteTest from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest from scrapy.utils.testproc import ProcessTest
from scrapy.utils.python import to_unicode from scrapy.utils.python import to_unicode
@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
content = '[\n{},\n{"foo": "bar"}\n]' content = '[\n{},\n{"foo": "bar"}\n]'
with open(file_path, 'r') as f: with open(file_path, 'r') as f:
self.assertEqual(f.read(), content) self.assertEqual(f.read(), content)
def test_parse_add_options(self):
command = parse.Command()
command.settings = Settings()
parser = argparse.ArgumentParser(
prog='scrapy', formatter_class=argparse.HelpFormatter,
conflict_handler='resolve', prefix_chars='-'
)
command.add_options(parser)
namespace = parser.parse_args(
['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
)
self.assertTrue(namespace.nolinks)
self.assertEqual(namespace.depth, 2)
self.assertEqual(namespace.spider, self.spider_name)
self.assertTrue(namespace.verbose)

38
tests/test_commands.py
View File

@ -1,6 +1,6 @@
import inspect import inspect
import json import json
import optparse import argparse
import os import os
import platform import platform
import re import re
@ -23,7 +23,7 @@ from twisted.python.versions import Version
from twisted.trial import unittest from twisted.trial import unittest
import scrapy import scrapy
from scrapy.commands import ScrapyCommand from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter
from scrapy.commands.startproject import IGNORE from scrapy.commands.startproject import IGNORE
from scrapy.settings import Settings from scrapy.settings import Settings
from scrapy.utils.python import to_unicode from scrapy.utils.python import to_unicode
@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):
def setUp(self): def setUp(self):
self.command = ScrapyCommand() self.command = ScrapyCommand()
self.command.settings = Settings() self.command.settings = Settings()
self.parser = optparse.OptionParser( self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve')
conflict_handler='resolve',
)
self.command.add_options(self.parser) self.command.add_options(self.parser)
def test_settings_json_string(self): def test_settings_json_string(self):
feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py']) opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
self.command.process_options(args, opts) self.command.process_options(args, opts)
self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
def test_help_formatter(self):
formatter = ScrapyHelpFormatter(prog='scrapy')
part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
'\n', 'optional arguments:\n', '\n', 'Global Options:\n']
self.assertEqual(
formatter._join_parts(part_strings),
('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n'
'Optional Arguments\n==================\n\n'
'Global Options\n--------------\n')
)
class ProjectTest(unittest.TestCase): class ProjectTest(unittest.TestCase):
project_name = 'testproject' project_name = 'testproject'
@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest):
self.assertNotIn('Unhandled Error', log) self.assertNotIn('Unhandled Error', log)
class ViewCommandTest(CommandTest):
def test_methods(self):
command = view.Command()
command.settings = Settings()
parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
formatter_class=ScrapyHelpFormatter,
conflict_handler='resolve')
command.add_options(parser)
self.assertEqual(command.short_desc(),
"Open URL in browser, as seen by Scrapy")
self.assertIn("URL using the Scrapy downloader and show its",
command.long_desc())
class CrawlCommandTest(CommandTest): class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()): def crawl(self, code, args=()):