RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move from optparse to argparse (#5374)

This commit is contained in:
Alex 2022-02-09 11:52:07 -08:00 committed by GitHub
parent 1356836e7e
commit e2e2ffd0d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 168 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/cmdline.py
View File

@ -1,13 +1,13 @@
import sys
import os
import optparse
import argparse
import cProfile
import inspect
import pkg_resources
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.commands import ScrapyCommand
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
@ -123,8 +123,6 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
sys.exit(0)
@ -133,12 +131,14 @@ def execute(argv=None, settings=None):
sys.exit(2)
cmd = cmds[cmdname]
parser.usage = f"scrapy {cmdname} {cmd.syntax()}"
parser.description = cmd.long_desc()
parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
usage=f"scrapy {cmdname} {cmd.syntax()}",
conflict_handler='resolve',
description=cmd.long_desc())
settings.setdict(cmd.default_settings, priority='command')
cmd.settings = settings
cmd.add_options(parser)
opts, args = parser.parse_args(args=argv[1:])
opts, args = parser.parse_known_args(args=argv[1:])
_run_print_help(parser, cmd.process_options, args, opts)
cmd.crawler_process = CrawlerProcess(settings)

75
scrapy/commands/__init__.py
View File

@ -2,7 +2,7 @@
Base class for Scrapy commands
"""
import os
from optparse import OptionGroup
import argparse
from typing import Any, Dict
from twisted.python import failure
@ -59,22 +59,20 @@ class ScrapyCommand:
"""
Populate option parse with options available for this command
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_option("--nolog", action="store_true",
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)
group = parser.add_argument_group(title='Global Options')
group.add_argument("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
group.add_argument("-L", "--loglevel", metavar="LEVEL", default=None,
help=f"log level (default: {self.settings['LOG_LEVEL']})")
group.add_argument("--nolog", action="store_true",
help="disable logging completely")
group.add_argument("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
group.add_argument("--pidfile", metavar="FILE",
help="write process ID to FILE")
group.add_argument("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
def process_options(self, args, opts):
try:
@ -114,14 +112,14 @@ class BaseRunSpiderCommand(ScrapyCommand):
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
parser.add_argument("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_argument("-o", "--output", metavar="FILE", action="append",
help="append scraped items to the end of FILE (use - for stdout)")
parser.add_argument("-O", "--overwrite-output", metavar="FILE", action="append",
help="dump scraped items into FILE, overwriting any existing file")
parser.add_argument("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
@ -137,3 +135,30 @@ class BaseRunSpiderCommand(ScrapyCommand):
opts.overwrite_output,
)
self.settings.set('FEEDS', feeds, priority='cmdline')
class ScrapyHelpFormatter(argparse.HelpFormatter):
"""
Help Formatter for scrapy command line help messages.
"""
def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
super().__init__(prog, indent_increment=indent_increment,
max_help_position=max_help_position, width=width)
def _join_parts(self, part_strings):
parts = self.format_part_strings(part_strings)
return super()._join_parts(parts)
def format_part_strings(self, part_strings):
"""
Underline and title case command line help message headers.
"""
if part_strings and part_strings[0].startswith("usage: "):
part_strings[0] = "Usage\n=====\n " + part_strings[0][len('usage: '):]
headings = [i for i in range(len(part_strings)) if part_strings[i].endswith(':\n')]
for index in headings[::-1]:
char = '-' if "Global Options" in part_strings[index] else '='
part_strings[index] = part_strings[index][:-2].title()
underline = ''.join(["\n", (char * len(part_strings[index])), "\n"])
part_strings.insert(index + 1, underline)
return part_strings

8
scrapy/commands/check.py
View File

@ -49,10 +49,10 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_option("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="only list contracts, without checking them")
parser.add_argument("-v", "--verbose", dest="verbose", default=False, action='store_true',
help="print contract tests for all spiders")
def run(self, args, opts):
# load contracts

10
scrapy/commands/fetch.py
View File

@ -26,11 +26,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():

20
scrapy/commands/genspider.py
View File

@ -44,16 +44,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
parser.add_argument("-l", "--list", dest="list", action="store_true",
help="List available templates")
parser.add_argument("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
parser.add_argument("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
parser.add_argument("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
parser.add_argument("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:

44
scrapy/commands/parse.py
View File

@ -32,28 +32,28 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser):
BaseRunSpiderCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
parser.add_argument("--spider", dest="spider", default=None,
help="use this spider without looking for one")
parser.add_argument("--pipelines", action="store_true",
help="process items through pipelines")
parser.add_argument("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
parser.add_argument("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
parser.add_argument("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
parser.add_argument("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
parser.add_argument("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_argument("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_argument("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
@property
def max_level(self):

20
scrapy/commands/settings.py
View File

@ -18,16 +18,16 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
parser.add_argument("--get", dest="get", metavar="SETTING",
help="print raw setting value")
parser.add_argument("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
parser.add_argument("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
parser.add_argument("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
parser.add_argument("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings

12
scrapy/commands/shell.py
View File

@ -33,12 +33,12 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_argument("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
parser.add_argument("--spider", dest="spider",
help="use this spider")
parser.add_argument("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be

4
scrapy/commands/version.py
View File

@ -16,8 +16,8 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
parser.add_argument("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
if opts.verbose:

3
scrapy/commands/view.py
View File

@ -1,3 +1,4 @@
import argparse
from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
@ -12,7 +13,7 @@ class Command(fetch.Command):
def add_options(self, parser):
super().add_options(parser)
parser.remove_option("--headers")
parser.add_argument('--headers', help=argparse.SUPPRESS)
def _print_response(self, response, opts):
open_in_browser(response)

19
tests/test_command_parse.py
View File

@ -1,6 +1,9 @@
import os
import argparse
from os.path import join, abspath, isfile, exists
from twisted.internet import defer
from scrapy.commands import parse
from scrapy.settings import Settings
from scrapy.utils.testsite import SiteTest
from scrapy.utils.testproc import ProcessTest
from scrapy.utils.python import to_unicode
@ -239,3 +242,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
content = '[\n{},\n{"foo": "bar"}\n]'
with open(file_path, 'r') as f:
self.assertEqual(f.read(), content)
def test_parse_add_options(self):
command = parse.Command()
command.settings = Settings()
parser = argparse.ArgumentParser(
prog='scrapy', formatter_class=argparse.HelpFormatter,
conflict_handler='resolve', prefix_chars='-'
)
command.add_options(parser)
namespace = parser.parse_args(
['--verbose', '--nolinks', '-d', '2', '--spider', self.spider_name]
)
self.assertTrue(namespace.nolinks)
self.assertEqual(namespace.depth, 2)
self.assertEqual(namespace.spider, self.spider_name)
self.assertTrue(namespace.verbose)

38
tests/test_commands.py
View File

@ -1,6 +1,6 @@
import inspect
import json
import optparse
import argparse
import os
import platform
import re
@ -23,7 +23,7 @@ from twisted.python.versions import Version
from twisted.trial import unittest
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.commands import view, ScrapyCommand, ScrapyHelpFormatter
from scrapy.commands.startproject import IGNORE
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
@ -37,19 +37,28 @@ class CommandSettings(unittest.TestCase):
def setUp(self):
self.command = ScrapyCommand()
self.command.settings = Settings()
self.parser = optparse.OptionParser(
formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve',
)
self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
conflict_handler='resolve')
self.command.add_options(self.parser)
def test_settings_json_string(self):
feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}'
opts, args = self.parser.parse_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
opts, args = self.parser.parse_known_args(args=['-s', f'FEEDS={feeds_json}', 'spider.py'])
self.command.process_options(args, opts)
self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings)
self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json))
def test_help_formatter(self):
formatter = ScrapyHelpFormatter(prog='scrapy')
part_strings = ['usage: scrapy genspider [options] <name> <domain>\n\n',
'\n', 'optional arguments:\n', '\n', 'Global Options:\n']
self.assertEqual(
formatter._join_parts(part_strings),
('Usage\n=====\n scrapy genspider [options] <name> <domain>\n\n\n'
'Optional Arguments\n==================\n\n'
'Global Options\n--------------\n')
)
class ProjectTest(unittest.TestCase):
project_name = 'testproject'
@ -812,6 +821,21 @@ class BenchCommandTest(CommandTest):
self.assertNotIn('Unhandled Error', log)
class ViewCommandTest(CommandTest):
def test_methods(self):
command = view.Command()
command.settings = Settings()
parser = argparse.ArgumentParser(prog='scrapy', prefix_chars='-',
formatter_class=ScrapyHelpFormatter,
conflict_handler='resolve')
command.add_options(parser)
self.assertEqual(command.short_desc(),
"Open URL in browser, as seen by Scrapy")
self.assertIn("URL using the Scrapy downloader and show its",
command.long_desc())
class CrawlCommandTest(CommandTest):
def crawl(self, code, args=()):