From dcf7235f0e44c0199360cea9472b7a0da6bab1a7 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Fri, 8 May 2020 01:48:23 +0530 Subject: [PATCH 1/9] Code sharing between crawl and runspider command --- scrapy/commands/common_commands.py | 29 +++++++++++++++++++++++++++++ scrapy/commands/crawl.py | 27 ++++----------------------- scrapy/commands/runspider.py | 24 ++---------------------- 3 files changed, 35 insertions(+), 45 deletions(-) create mode 100644 scrapy/commands/common_commands.py diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py new file mode 100644 index 000000000..7da7494ac --- /dev/null +++ b/scrapy/commands/common_commands.py @@ -0,0 +1,29 @@ +from scrapy.commands import ScrapyCommand +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.exceptions import UsageError + + +class CommonCommands(ScrapyCommand): + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-a", dest="spargs", action="append", default=[], + metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", action="append", + help="dump scraped items into FILE" + + "(use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + feeds = feed_process_params_from_cli( + self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b2f9484b..b477d7c71 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,9 +1,8 @@ -from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError +from scrapy.commands.common_commands import CommonCommands -class Command(ScrapyCommand): +class Command(CommonCommands): requires_project = True @@ -13,30 +12,12 @@ class Command(ScrapyCommand): def short_desc(self): return "Run a spider" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError( + "running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 62510609a..9959f6b0d 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,9 +3,8 @@ import os from importlib import import_module from scrapy.utils.spider import iter_spider_classes -from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.commands.common_commands import CommonCommands def _import_file(filepath): @@ -24,7 +23,7 @@ def _import_file(filepath): return module -class Command(ScrapyCommand): +class Command(CommonCommands): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} @@ -38,25 +37,6 @@ class Command(ScrapyCommand): def long_desc(self): return "Run the spider defined in the given file" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) != 1: raise UsageError() From 92792cc3f7c66a81a9e7a0576be0848266950c7e Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sat, 9 May 2020 17:28:10 +0530 Subject: [PATCH 2/9] Moved common_commands.py to __init__.py --- scrapy/commands/__init__.py | 52 ++++++++++++++++++++++++------ scrapy/commands/common_commands.py | 29 ----------------- scrapy/commands/crawl.py | 4 +-- scrapy/commands/runspider.py | 4 +-- 4 files changed, 47 insertions(+), 42 deletions(-) delete mode 100644 scrapy/commands/common_commands.py diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9f8e6986a..b95d395a1 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -5,7 +5,7 @@ import os from optparse import OptionGroup from twisted.python import failure -from scrapy.utils.conf import arglist_to_dict +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError @@ -59,18 +59,19 @@ class ScrapyCommand: """ group = OptionGroup(parser, "Global Options") group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") + help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help="log level (default: %s)" % self.settings['LOG_LEVEL']) group.add_option("--nolog", action="store_true", - help="disable logging completely") + help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") + help="write python cProfile stats to FILE") group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") + help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") + help="set/override setting (may be repeated)") + group.add_option("--pdb", action="store_true", + help="enable pdb on failure") parser.add_option_group(group) @@ -79,7 +80,8 @@ class ScrapyCommand: self.settings.setdict(arglist_to_dict(opts.set), priority='cmdline') except ValueError: - raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -s value, use -s NAME=VALUE", print_help=False) if opts.logfile: self.settings.set('LOG_ENABLED', True, priority='cmdline') @@ -104,3 +106,35 @@ class ScrapyCommand: Entry point for running commands """ raise NotImplementedError + + +'''' +The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for +performing common functionality between crawl.py and runspider.py +''' + + +class BaseRunSpiderCommands(ScrapyCommand): + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-a", dest="spargs", action="append", default=[], + metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", action="append", + help="dump scraped items into FILE" + + "(use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + feeds = feed_process_params_from_cli( + self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py deleted file mode 100644 index 7da7494ac..000000000 --- a/scrapy/commands/common_commands.py +++ /dev/null @@ -1,29 +0,0 @@ -from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli -from scrapy.exceptions import UsageError - - -class CommonCommands(ScrapyCommand): - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], - metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE" - + "(use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError( - "Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli( - self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index b477d7c71..ca19b1367 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,8 @@ from scrapy.exceptions import UsageError -from scrapy.commands.common_commands import CommonCommands +from scrapy.commands import BaseRunSpiderCommands -class Command(CommonCommands): +class Command(BaseRunSpiderCommands): requires_project = True diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9959f6b0d..bd24a369e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError -from scrapy.commands.common_commands import CommonCommands +from scrapy.commands import BaseRunSpiderCommands def _import_file(filepath): @@ -23,7 +23,7 @@ def _import_file(filepath): return module -class Command(CommonCommands): +class Command(BaseRunSpiderCommands): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} From ed4f4f84082f7023734d1200d88e8f1aec57904c Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sun, 10 May 2020 00:08:34 +0530 Subject: [PATCH 3/9] Applied suggested format changes --- scrapy/commands/__init__.py | 22 ++++++++-------------- scrapy/commands/crawl.py | 5 ++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index b95d395a1..1dada1ceb 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -80,8 +80,7 @@ class ScrapyCommand: self.settings.setdict(arglist_to_dict(opts.set), priority='cmdline') except ValueError: - raise UsageError( - "Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) if opts.logfile: self.settings.set('LOG_ENABLED', True, priority='cmdline') @@ -108,18 +107,15 @@ class ScrapyCommand: raise NotImplementedError -'''' -The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for -performing common functionality between crawl.py and runspider.py -''' - - class BaseRunSpiderCommands(ScrapyCommand): + '''' + The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for + performing common functionality between crawl.py and runspider.py + ''' def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], - metavar="NAME=VALUE", + parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", help="dump scraped items into FILE" @@ -132,9 +128,7 @@ class BaseRunSpiderCommands(ScrapyCommand): try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: - raise UsageError( - "Invalid -a value, use -a NAME=VALUE", print_help=False) + raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: - feeds = feed_process_params_from_cli( - self.settings, opts.output, opts.output_format) + feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index ca19b1367..c39c5a9ef 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,5 +1,5 @@ -from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommands +from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommands): @@ -16,8 +16,7 @@ class Command(BaseRunSpiderCommands): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError( - "running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) From 02ac6664a5c3510104b67f7cfc9a71c831d5bd5f Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sun, 10 May 2020 00:26:48 +0530 Subject: [PATCH 4/9] Travis CI fixes --- scrapy/commands/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 1dada1ceb..38b1b77bd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,7 +115,7 @@ class BaseRunSpiderCommands(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE", + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", help="dump scraped items into FILE" From e01c30f0d54ba838aa3212634cddf3630f77b2d4 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Tue, 12 May 2020 01:05:20 +0530 Subject: [PATCH 5/9] Update scrapy/commands/__init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 38b1b77bd..2ba796562 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -108,10 +108,10 @@ class ScrapyCommand: class BaseRunSpiderCommands(ScrapyCommand): - '''' + """ The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for performing common functionality between crawl.py and runspider.py - ''' + """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) From 28d223dd87fb655743f67087be0009b094ddad10 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Tue, 12 May 2020 15:28:22 +0530 Subject: [PATCH 6/9] Update __init__.py --- scrapy/commands/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2ba796562..99276515b 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -118,8 +118,7 @@ class BaseRunSpiderCommands(ScrapyCommand): parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE" - + "(use - for stdout)") + help="dump scraped items into FILE (use - for stdout)") parser.add_option("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items with -o") From 07e125f4c916fcd2d4d978a89c7d7e508cc63620 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Tue, 12 May 2020 16:31:56 +0530 Subject: [PATCH 7/9] Travis CI fixes in __init__.py --- scrapy/commands/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 99276515b..0ddbf2ca0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -112,7 +112,6 @@ class BaseRunSpiderCommands(ScrapyCommand): The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for performing common functionality between crawl.py and runspider.py """ - def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", From 4cdd00e21f4bfe22ba9b8fabe034a5e4d34dab75 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sat, 16 May 2020 00:25:57 +0530 Subject: [PATCH 8/9] Changed BaseRunSpiderCommands to BaseRunSpiderCommand --- scrapy/commands/__init__.py | 2 +- scrapy/commands/crawl.py | 4 ++-- scrapy/commands/runspider.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0ddbf2ca0..81fae089b 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -107,7 +107,7 @@ class ScrapyCommand: raise NotImplementedError -class BaseRunSpiderCommands(ScrapyCommand): +class BaseRunSpiderCommand(ScrapyCommand): """ The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for performing common functionality between crawl.py and runspider.py diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index c39c5a9ef..e1724c1e6 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,8 @@ -from scrapy.commands import BaseRunSpiderCommands +from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -class Command(BaseRunSpiderCommands): +class Command(BaseRunSpiderCommand): requires_project = True diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index bd24a369e..befee021b 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError -from scrapy.commands import BaseRunSpiderCommands +from scrapy.commands import BaseRunSpiderCommand def _import_file(filepath): @@ -23,7 +23,7 @@ def _import_file(filepath): return module -class Command(BaseRunSpiderCommands): +class Command(BaseRunSpiderCommand): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} From 604fe33bad36f1269677e98d0bfec1f60c95aa53 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sat, 16 May 2020 01:53:49 +0530 Subject: [PATCH 9/9] Update scrapy/commands/__init__.py Changed typo in a comment for BaseRunSpiderCommand Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- scrapy/commands/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 49152ca28..ab850dcb3 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -108,8 +108,7 @@ class ScrapyCommand: class BaseRunSpiderCommand(ScrapyCommand): """ - The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for - performing common functionality between crawl.py and runspider.py + Common class used to share functionality between the crawl and runspider commands """ def add_options(self, parser): ScrapyCommand.add_options(self, parser)