From dcf7235f0e44c0199360cea9472b7a0da6bab1a7 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Fri, 8 May 2020 01:48:23 +0530 Subject: [PATCH 01/28] Code sharing between crawl and runspider command --- scrapy/commands/common_commands.py | 29 +++++++++++++++++++++++++++++ scrapy/commands/crawl.py | 27 ++++----------------------- scrapy/commands/runspider.py | 24 ++---------------------- 3 files changed, 35 insertions(+), 45 deletions(-) create mode 100644 scrapy/commands/common_commands.py diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py new file mode 100644 index 000000000..7da7494ac --- /dev/null +++ b/scrapy/commands/common_commands.py @@ -0,0 +1,29 @@ +from scrapy.commands import ScrapyCommand +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.exceptions import UsageError + + +class CommonCommands(ScrapyCommand): + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-a", dest="spargs", action="append", default=[], + metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", action="append", + help="dump scraped items into FILE" + + "(use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + feeds = feed_process_params_from_cli( + self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b2f9484b..b477d7c71 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,9 +1,8 @@ -from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError +from scrapy.commands.common_commands import CommonCommands -class Command(ScrapyCommand): +class Command(CommonCommands): requires_project = True @@ -13,30 +12,12 @@ class Command(ScrapyCommand): def short_desc(self): return "Run a spider" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError( + "running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 62510609a..9959f6b0d 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,9 +3,8 @@ import os from importlib import import_module from scrapy.utils.spider import iter_spider_classes -from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli +from scrapy.commands.common_commands import CommonCommands def _import_file(filepath): @@ -24,7 +23,7 @@ def _import_file(filepath): return module -class Command(ScrapyCommand): +class Command(CommonCommands): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} @@ -38,25 +37,6 @@ class Command(ScrapyCommand): def long_desc(self): return "Run the spider defined in the given file" - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE (use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') - def run(self, args, opts): if len(args) != 1: raise UsageError() From 92792cc3f7c66a81a9e7a0576be0848266950c7e Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sat, 9 May 2020 17:28:10 +0530 Subject: [PATCH 02/28] Moved common_commands.py to __init__.py --- scrapy/commands/__init__.py | 52 ++++++++++++++++++++++++------ scrapy/commands/common_commands.py | 29 ----------------- scrapy/commands/crawl.py | 4 +-- scrapy/commands/runspider.py | 4 +-- 4 files changed, 47 insertions(+), 42 deletions(-) delete mode 100644 scrapy/commands/common_commands.py diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9f8e6986a..b95d395a1 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -5,7 +5,7 @@ import os from optparse import OptionGroup from twisted.python import failure -from scrapy.utils.conf import arglist_to_dict +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError @@ -59,18 +59,19 @@ class ScrapyCommand: """ group = OptionGroup(parser, "Global Options") group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") + help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help="log level (default: %s)" % self.settings['LOG_LEVEL']) group.add_option("--nolog", action="store_true", - help="disable logging completely") + help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") + help="write python cProfile stats to FILE") group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") + help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", help="enable pdb on failure") + help="set/override setting (may be repeated)") + group.add_option("--pdb", action="store_true", + help="enable pdb on failure") parser.add_option_group(group) @@ -79,7 +80,8 @@ class ScrapyCommand: self.settings.setdict(arglist_to_dict(opts.set), priority='cmdline') except ValueError: - raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -s value, use -s NAME=VALUE", print_help=False) if opts.logfile: self.settings.set('LOG_ENABLED', True, priority='cmdline') @@ -104,3 +106,35 @@ class ScrapyCommand: Entry point for running commands """ raise NotImplementedError + + +'''' +The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for +performing common functionality between crawl.py and runspider.py +''' + + +class BaseRunSpiderCommands(ScrapyCommand): + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-a", dest="spargs", action="append", default=[], + metavar="NAME=VALUE", + help="set spider argument (may be repeated)") + parser.add_option("-o", "--output", metavar="FILE", action="append", + help="dump scraped items into FILE" + + "(use - for stdout)") + parser.add_option("-t", "--output-format", metavar="FORMAT", + help="format to use for dumping items with -o") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + try: + opts.spargs = arglist_to_dict(opts.spargs) + except ValueError: + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False) + if opts.output: + feeds = feed_process_params_from_cli( + self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/common_commands.py b/scrapy/commands/common_commands.py deleted file mode 100644 index 7da7494ac..000000000 --- a/scrapy/commands/common_commands.py +++ /dev/null @@ -1,29 +0,0 @@ -from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli -from scrapy.exceptions import UsageError - - -class CommonCommands(ScrapyCommand): - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], - metavar="NAME=VALUE", - help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE" - + "(use - for stdout)") - parser.add_option("-t", "--output-format", metavar="FORMAT", - help="format to use for dumping items with -o") - - def process_options(self, args, opts): - ScrapyCommand.process_options(self, args, opts) - try: - opts.spargs = arglist_to_dict(opts.spargs) - except ValueError: - raise UsageError( - "Invalid -a value, use -a NAME=VALUE", print_help=False) - if opts.output: - feeds = feed_process_params_from_cli( - self.settings, opts.output, opts.output_format) - self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index b477d7c71..ca19b1367 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,8 @@ from scrapy.exceptions import UsageError -from scrapy.commands.common_commands import CommonCommands +from scrapy.commands import BaseRunSpiderCommands -class Command(CommonCommands): +class Command(BaseRunSpiderCommands): requires_project = True diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 9959f6b0d..bd24a369e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError -from scrapy.commands.common_commands import CommonCommands +from scrapy.commands import BaseRunSpiderCommands def _import_file(filepath): @@ -23,7 +23,7 @@ def _import_file(filepath): return module -class Command(CommonCommands): +class Command(BaseRunSpiderCommands): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} From ed4f4f84082f7023734d1200d88e8f1aec57904c Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sun, 10 May 2020 00:08:34 +0530 Subject: [PATCH 03/28] Applied suggested format changes --- scrapy/commands/__init__.py | 22 ++++++++-------------- scrapy/commands/crawl.py | 5 ++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index b95d395a1..1dada1ceb 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -80,8 +80,7 @@ class ScrapyCommand: self.settings.setdict(arglist_to_dict(opts.set), priority='cmdline') except ValueError: - raise UsageError( - "Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) if opts.logfile: self.settings.set('LOG_ENABLED', True, priority='cmdline') @@ -108,18 +107,15 @@ class ScrapyCommand: raise NotImplementedError -'''' -The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for -performing common functionality between crawl.py and runspider.py -''' - - class BaseRunSpiderCommands(ScrapyCommand): + '''' + The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for + performing common functionality between crawl.py and runspider.py + ''' def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[], - metavar="NAME=VALUE", + parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", help="dump scraped items into FILE" @@ -132,9 +128,7 @@ class BaseRunSpiderCommands(ScrapyCommand): try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: - raise UsageError( - "Invalid -a value, use -a NAME=VALUE", print_help=False) + raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: - feeds = feed_process_params_from_cli( - self.settings, opts.output, opts.output_format) + feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) self.settings.set('FEEDS', feeds, priority='cmdline') diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index ca19b1367..c39c5a9ef 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,5 +1,5 @@ -from scrapy.exceptions import UsageError from scrapy.commands import BaseRunSpiderCommands +from scrapy.exceptions import UsageError class Command(BaseRunSpiderCommands): @@ -16,8 +16,7 @@ class Command(BaseRunSpiderCommands): if len(args) < 1: raise UsageError() elif len(args) > 1: - raise UsageError( - "running 'scrapy crawl' with more than one spider is no longer supported") + raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) From 02ac6664a5c3510104b67f7cfc9a71c831d5bd5f Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sun, 10 May 2020 00:26:48 +0530 Subject: [PATCH 04/28] Travis CI fixes --- scrapy/commands/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 1dada1ceb..38b1b77bd 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -115,7 +115,7 @@ class BaseRunSpiderCommands(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("-a", dest="spargs", action="append", default=[],metavar="NAME=VALUE", + parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", help="dump scraped items into FILE" From 1bfbcc618090d4e71e06471b78aaac3071a51956 Mon Sep 17 00:00:00 2001 From: Will Beaufoy Date: Sun, 10 May 2020 13:48:09 +0100 Subject: [PATCH 05/28] Extend hoverxref_roles to custom crossrefs (#4495) --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 8ab38a090..3ae709a04 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -300,3 +300,4 @@ hoverxref_role_types = { "mod": "tooltip", "ref": "tooltip", } +hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal'] From e01c30f0d54ba838aa3212634cddf3630f77b2d4 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Tue, 12 May 2020 01:05:20 +0530 Subject: [PATCH 06/28] Update scrapy/commands/__init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/commands/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 38b1b77bd..2ba796562 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -108,10 +108,10 @@ class ScrapyCommand: class BaseRunSpiderCommands(ScrapyCommand): - '''' + """ The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for performing common functionality between crawl.py and runspider.py - ''' + """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) From 28d223dd87fb655743f67087be0009b094ddad10 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Tue, 12 May 2020 15:28:22 +0530 Subject: [PATCH 07/28] Update __init__.py --- scrapy/commands/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 2ba796562..99276515b 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -118,8 +118,7 @@ class BaseRunSpiderCommands(ScrapyCommand): parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") parser.add_option("-o", "--output", metavar="FILE", action="append", - help="dump scraped items into FILE" - + "(use - for stdout)") + help="dump scraped items into FILE (use - for stdout)") parser.add_option("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items with -o") From 07e125f4c916fcd2d4d978a89c7d7e508cc63620 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Tue, 12 May 2020 16:31:56 +0530 Subject: [PATCH 08/28] Travis CI fixes in __init__.py --- scrapy/commands/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 99276515b..0ddbf2ca0 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -112,7 +112,6 @@ class BaseRunSpiderCommands(ScrapyCommand): The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for performing common functionality between crawl.py and runspider.py """ - def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", From 4cdd00e21f4bfe22ba9b8fabe034a5e4d34dab75 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sat, 16 May 2020 00:25:57 +0530 Subject: [PATCH 09/28] Changed BaseRunSpiderCommands to BaseRunSpiderCommand --- scrapy/commands/__init__.py | 2 +- scrapy/commands/crawl.py | 4 ++-- scrapy/commands/runspider.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0ddbf2ca0..81fae089b 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -107,7 +107,7 @@ class ScrapyCommand: raise NotImplementedError -class BaseRunSpiderCommands(ScrapyCommand): +class BaseRunSpiderCommand(ScrapyCommand): """ The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for performing common functionality between crawl.py and runspider.py diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index c39c5a9ef..e1724c1e6 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,8 +1,8 @@ -from scrapy.commands import BaseRunSpiderCommands +from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError -class Command(BaseRunSpiderCommands): +class Command(BaseRunSpiderCommand): requires_project = True diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index bd24a369e..befee021b 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.exceptions import UsageError -from scrapy.commands import BaseRunSpiderCommands +from scrapy.commands import BaseRunSpiderCommand def _import_file(filepath): @@ -23,7 +23,7 @@ def _import_file(filepath): return module -class Command(BaseRunSpiderCommands): +class Command(BaseRunSpiderCommand): requires_project = False default_settings = {'SPIDER_LOADER_WARN_ONLY': True} From 604fe33bad36f1269677e98d0bfec1f60c95aa53 Mon Sep 17 00:00:00 2001 From: Jay Rajput Date: Sat, 16 May 2020 01:53:49 +0530 Subject: [PATCH 10/28] Update scrapy/commands/__init__.py Changed typo in a comment for BaseRunSpiderCommand Co-authored-by: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> --- scrapy/commands/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 49152ca28..ab850dcb3 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -108,8 +108,7 @@ class ScrapyCommand: class BaseRunSpiderCommand(ScrapyCommand): """ - The BaseRunSpiderCommands class inherits the ScrapyCommand class and it Used for - performing common functionality between crawl.py and runspider.py + Common class used to share functionality between the crawl and runspider commands """ def add_options(self, parser): ScrapyCommand.add_options(self, parser) From a915af2e4592e8a0367c44a73c95cee3f835887d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 11 May 2020 16:13:54 -0300 Subject: [PATCH 11/28] Remove monkeypatches module, E402 flake8 code --- pytest.ini | 4 +-- scrapy/__init__.py | 47 ++++++++++++++++--------------- scrapy/_monkeypatches.py | 11 -------- scrapy/linkextractors/__init__.py | 2 +- scrapy/spiders/__init__.py | 6 ++-- 5 files changed, 30 insertions(+), 40 deletions(-) delete mode 100644 scrapy/_monkeypatches.py diff --git a/pytest.ini b/pytest.ini index 0e289866c..0797e8ac8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -27,18 +27,16 @@ flake8-ignore = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file # F401: Module imported but unused + scrapy/__init__.py E402 scrapy/core/downloader/handlers/http.py F401 scrapy/http/__init__.py F401 scrapy/linkextractors/__init__.py E402 F401 scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/__init__.py E402 scrapy/selector/__init__.py F403 - scrapy/spiders/__init__.py E402 scrapy/utils/http.py F403 scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 scrapy/utils/url.py F403 F405 tests/test_loader.py E741 - tests/test_webclient.py E402 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index fb8357f3c..e791deaa6 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -2,33 +2,11 @@ Scrapy - a web crawling and web scraping framework written for Python """ -__all__ = ['__version__', 'version_info', 'twisted_version', - 'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field'] - -# Scrapy version import pkgutil -__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() -version_info = tuple(int(v) if v.isdigit() else v - for v in __version__.split('.')) -del pkgutil - -# Check minimum required Python version import sys -if sys.version_info < (3, 5): - print("Scrapy %s requires Python 3.5" % __version__) - sys.exit(1) - -# Ignore noisy twisted deprecation warnings import warnings -warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') -del warnings - -# Apply monkey patches to fix issues in external libraries -from scrapy import _monkeypatches -del _monkeypatches from twisted import version as _txv -twisted_version = (_txv.major, _txv.minor, _txv.micro) # Declare top-level shortcuts from scrapy.spiders import Spider @@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest from scrapy.selector import Selector from scrapy.item import Item, Field + +__all__ = [ + '__version__', 'version_info', 'twisted_version', 'Spider', + 'Request', 'FormRequest', 'Selector', 'Item', 'Field', +] + + +# Scrapy and Twisted versions +__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip() +version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.')) +twisted_version = (_txv.major, _txv.minor, _txv.micro) + + +# Check minimum required Python version +if sys.version_info < (3, 5): + print("Scrapy %s requires Python 3.5" % __version__) + sys.exit(1) + + +# Ignore noisy twisted deprecation warnings +warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') + + +del pkgutil del sys +del warnings diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py deleted file mode 100644 index f74f89bda..000000000 --- a/scrapy/_monkeypatches.py +++ /dev/null @@ -1,11 +0,0 @@ -import copyreg - - -# Undo what Twisted's perspective broker adds to pickle register -# to prevent bugs like Twisted#7989 while serializing requests -import twisted.persisted.styles # NOQA -# Remove only entries with twisted serializers for non-twisted types. -for k, v in frozenset(copyreg.dispatch_table.items()): - if not str(getattr(k, '__module__', '')).startswith('twisted') \ - and str(getattr(v, '__module__', '')).startswith('twisted'): - copyreg.dispatch_table.pop(k) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ae019c70f..984a5c4e1 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -133,4 +133,4 @@ class FilteringLinkExtractor: # Top-level imports -from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index ba1c866f8..02f87f8f5 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -110,6 +110,6 @@ class Spider(object_ref): # Top-level imports -from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401 -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401 -from scrapy.spiders.sitemap import SitemapSpider # noqa: F401 +from scrapy.spiders.crawl import CrawlSpider, Rule +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider +from scrapy.spiders.sitemap import SitemapSpider From a22f97052e9b14631a977600c1f59bd468c85601 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 May 2020 21:18:26 -0300 Subject: [PATCH 12/28] Cleanup import in selector module --- pytest.ini | 2 +- scrapy/selector/__init__.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 0797e8ac8..663c5cc78 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,10 +31,10 @@ flake8-ignore = scrapy/core/downloader/handlers/http.py F401 scrapy/http/__init__.py F401 scrapy/linkextractors/__init__.py E402 F401 + scrapy/selector/__init__.py F401 scrapy/spiders/__init__.py E402 F401 # Issues pending a review: - scrapy/selector/__init__.py F403 scrapy/utils/http.py F403 scrapy/utils/markup.py F403 scrapy/utils/multipart.py F403 diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index a9240c1f6..85c500d66 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,4 +1,6 @@ """ Selectors """ -from scrapy.selector.unified import * # noqa: F401 + +# top-level imports +from scrapy.selector.unified import Selector, SelectorList From afb2e5011fb9d9aa0ca568d376b0c4cef4193a9e Mon Sep 17 00:00:00 2001 From: nsirletti Date: Sat, 16 May 2020 12:48:21 +0200 Subject: [PATCH 13/28] Allow doc to be downloadable on readthedocs.org --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 17eba34f3..e4d3f02cc 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,5 @@ version: 2 +formats: all sphinx: configuration: docs/conf.py fail_on_warning: true From c2a0cca0fe6bc8342efa6034a78b9b8161aa2177 Mon Sep 17 00:00:00 2001 From: nsirletti Date: Sat, 16 May 2020 15:07:16 +0200 Subject: [PATCH 14/28] Suppress Sphinx warnings about MIME types other than .rst --- docs/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 8ab38a090..468c1d190 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -100,6 +100,9 @@ exclude_trees = ['.build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' +# List of Sphinx warnings that will not be raised +suppress_warnings = ['epub.unknown_project_files'] + # Options for HTML output # ----------------------- From f418c6d5f7307c3beae57dfc519be04657a8e1b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 19 May 2020 20:04:47 +0200 Subject: [PATCH 15/28] logging.rst: remove unused, misleading import --- docs/topics/logging.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 675e65ef1..e81091651 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -262,7 +262,6 @@ scrapy.utils.log module This is an example on how to redirect ``INFO`` or higher messages to a file:: import logging - from scrapy.utils.log import configure_logging logging.basicConfig( filename='log.txt', From eb8493f09c56dea4dad129882fbf7b4a4da08e1a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 20 May 2020 09:21:34 -0400 Subject: [PATCH 16/28] Incompatible with pytest 5.4 and 5.4.1 --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d207c5fb0..05b794d81 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,7 +2,7 @@ jmespath mitmproxy; python_version >= '3.6' mitmproxy<4.0.0; python_version < '3.6' -pytest < 5.4 +pytest != 5.4, != 5.4.1 pytest-cov pytest-twisted >= 1.11 pytest-xdist From 276721a5dc96a4239645db1a78fe056df671e10e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 9 May 2020 22:02:40 -0300 Subject: [PATCH 17/28] Stop response download --- docs/faq.rst | 13 +++++ docs/topics/exceptions.rst | 46 +++++++++++++++--- docs/topics/request-response.rst | 49 ++++++++++++++++++- docs/topics/signals.rst | 11 ++++- scrapy/core/downloader/handlers/http11.py | 59 +++++++++++++---------- scrapy/core/scraper.py | 10 +++- scrapy/exceptions.py | 13 +++++ scrapy/utils/signal.py | 7 +-- tests/spiders.py | 33 +++++++++++++ tests/test_crawl.py | 40 +++++++++++++-- tests/test_engine.py | 57 ++++++++++++++++++++-- 11 files changed, 291 insertions(+), 47 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 0b6bd6a86..c06cb945b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -371,6 +371,19 @@ Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switch different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +.. _faq-stop-response-download: + +How can I cancel the download of a given response? +-------------------------------------------------- + +In some situations, it might be useful to stop the download of a certain response. +For instance, if you only need the first part of a large response and you would like +to save resources by avoiding the download of the whole body. +In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received` +signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to +the :ref:`topics-stop-response-download` topic for additional information and examples. + + .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 09cb8ed66..10b23edc4 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -14,13 +14,6 @@ Built-in Exceptions reference Here's a list of all exceptions included in Scrapy and their usage. -DropItem --------- - -.. exception:: DropItem - -The exception that must be raised by item pipeline stages to stop processing an -Item. For more information see :ref:`topics-item-pipeline`. CloseSpider ----------- @@ -47,6 +40,14 @@ DontCloseSpider This exception can be raised in a :signal:`spider_idle` signal handler to prevent the spider from being closed. +DropItem +-------- + +.. exception:: DropItem + +The exception that must be raised by item pipeline stages to stop processing an +Item. For more information see :ref:`topics-item-pipeline`. + IgnoreRequest ------------- @@ -77,3 +78,34 @@ NotSupported This exception is raised to indicate an unsupported feature. +StopDownload +------------- + +.. versionadded:: 2.2 + +.. exception:: StopDownload(fail=True) + +Raised from a :class:`~scrapy.signals.bytes_received` signal handler to +indicate that no further bytes should be downloaded for a response. + +The ``fail`` boolean parameter controls which method will handle the resulting +response: + +* If ``fail=True``, the request errback is called. The response object is + available as the ``response`` attribute of the received + :class:`~twisted.python.failure.Failure` object. This is the default behaviour. + +* If ``fail=False``, the request callback is called instead. + +In both cases, the response could have its body truncated: the body contains +all bytes received up until the exception is raised, including the bytes +received in the signal handler that raises the exception. Also, the response +object is marked with ``"download_stopped"`` in its :attr:`Response.flags` +attribute. + +.. note:: ``fail`` is a keyword-only parameter, i.e. raising + ``StopDownload(False)`` or ``StopDownload(True)`` will raise + a :class:`TypeError`. + +See the documentation for the :class:`~scrapy.signals.bytes_received` signal +and the :ref:`topics-stop-response-download` topic for additional information and examples. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 15a83f453..ae25ff7e4 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -385,6 +385,51 @@ The meta key is used set retry times per request. When initialized, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. + +.. _topics-stop-response-download: + +Stopping the download of a Response +=================================== + +Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a +:class:`~scrapy.signals.bytes_received` signal handler will stop the +download of a given response. See the following example:: + + import scrapy + + + class StopSpider(scrapy.Spider): + name = "stop" + start_urls = ["https://docs.scrapy.org/en/latest/"] + + @classmethod + def from_crawler(cls, crawler): + spider = super().from_crawler(crawler) + crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received) + return spider + + def parse(self, response): + # 'last_chars' show that the full response was not downloaded + yield {"len": len(response.text), "last_chars": response.text[-40:]} + + def on_bytes_received(self, data, request, spider): + raise scrapy.exceptions.StopDownload(fail=False) + +which produces the following output:: + + 2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened + 2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min) + 2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for from signal handler StopSpider.on_bytes_received + 2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) (referer: None) ['download_stopped'] + 2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/> + {'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n \n Scr'} + 2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished) + +By default, resulting responses are handled by their corresponding errbacks. To +call their callback instead, like in this example, pass ``fail=False`` to the +:exc:`~scrapy.exceptions.StopDownload` exception. + + .. _topics-request-response-ref-request-subclasses: Request subclasses @@ -716,9 +761,9 @@ Response objects .. versionadded:: 2.1.0 The IP address of the server from which the Response originated. - + This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. For other handlers, + handler, i.e. for ``http(s)`` responses. For other handlers, :attr:`ip_address` is always ``None``. .. method:: Response.copy() diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 7fe63a7b0..fe4fb0834 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -373,6 +373,8 @@ request_left_downloader bytes_received ~~~~~~~~~~~~~~ +.. versionadded:: 2.2 + .. signal:: bytes_received .. function:: bytes_received(data, request, spider) @@ -385,14 +387,19 @@ bytes_received This signal does not support returning deferreds from its handlers. :param data: the data received by the download handler - :type spider: :class:`bytes` object + :type data: :class:`bytes` object - :param request: the request that generated the response + :param request: the request that generated the download :type request: :class:`~scrapy.http.Request` object :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: Handlers of this signal can stop the download of a response while it + is in progress by raising the :exc:`~scrapy.exceptions.StopDownload` + exception. Please refer to the :ref:`topics-stop-response-download` topic + for additional information and examples. + Response signals ---------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c21491f52..555a3a032 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,6 +12,7 @@ from urllib.parse import urldefrag from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError +from twisted.python.failure import Failure from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http_headers import Headers as TxHeaders @@ -21,7 +22,7 @@ from zope.interface import implementer from scrapy import signals from scrapy.core.downloader.tls import openssl_methods from scrapy.core.downloader.webclient import _parse -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.misc import create_instance, load_object @@ -431,7 +432,7 @@ class ScrapyAgent: def _cb_bodydone(self, result, request, url): headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) - return respcls( + response = respcls( url=url, status=int(result["txresponse"].code), headers=headers, @@ -440,6 +441,14 @@ class ScrapyAgent: certificate=result["certificate"], ip_address=result["ip_address"], ) + if result.get("failure"): + # This failure is not the same object that will reach the errback, + # so we need to temporarily store the response in the exception. + # It will be moved to the failure in core/scraper.py + failure = result["failure"] + failure.value.response = response + return failure + return response @implementer(IBodyProducer) @@ -477,6 +486,16 @@ class _ResponseReader(protocol.Protocol): self._ip_address = None self._crawler = crawler + def _finish_response(self, flags=None, failure=None): + self._finished.callback({ + "txresponse": self._txresponse, + "body": self._bodybuf.getvalue(), + "flags": flags, + "certificate": self._certificate, + "ip_address": self._ip_address, + "failure": failure, + }) + def connectionMade(self): if self._certificate is None: with suppress(AttributeError): @@ -493,12 +512,21 @@ class _ResponseReader(protocol.Protocol): self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) - self._crawler.signals.send_catch_log( + bytes_received_result = self._crawler.signals.send_catch_log( signal=signals.bytes_received, data=bodyBytes, request=self._request, spider=self._crawler.spider, ) + for handler, result in bytes_received_result: + if isinstance(result, Failure) and isinstance(result.value, StopDownload): + logger.debug("Download stopped for %(request)s from signal handler %(handler)s", + {"request": self._request, "handler": handler.__qualname__}) + self.transport._producer.loseConnection() + self._finish_response( + flags=["download_stopped"], + failure=result if result.value.fail else None, + ) if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " @@ -521,36 +549,17 @@ class _ResponseReader(protocol.Protocol): if self._finished.called: return - body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback({ - "txresponse": self._txresponse, - "body": body, - "flags": None, - "certificate": self._certificate, - "ip_address": self._ip_address, - }) + self._finish_response() return if reason.check(PotentialDataLoss): - self._finished.callback({ - "txresponse": self._txresponse, - "body": body, - "flags": ["partial"], - "certificate": self._certificate, - "ip_address": self._ip_address, - }) + self._finish_response(flags=["partial"]) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback({ - "txresponse": self._txresponse, - "body": body, - "flags": ["dataloss"], - "certificate": self._certificate, - "ip_address": self._ip_address, - }) + self._finish_response(flags=["dataloss"]) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6785e103d..c52ada83b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -11,7 +11,7 @@ from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errba from scrapy.utils.spider import iterate_spider_output from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.log import logformatter_adapter, failure_to_exc_info -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest +from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest, StopDownload from scrapy import signals from scrapy.http import Request, Response from scrapy.item import _BaseItem @@ -147,6 +147,14 @@ class Scraper: def call_spider(self, result, request, spider): result.request = request + # StopDownload exceptions: make the partial response an attribute of the failure + if ( + isinstance(result, Failure) + and isinstance(result.value, StopDownload) + and hasattr(result.value, "response") + ): + result.response = result.value.response + delattr(result.value, "response") dfd = defer_result(result) callback = request.callback or spider.parse warn_on_generator_with_return_value(spider, callback) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 7c4bb3d00..45f152321 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -41,6 +41,18 @@ class CloseSpider(Exception): self.reason = reason +class StopDownload(Exception): + """ + Stop the download of the body for a given response. + The 'fail' boolean parameter indicates whether or not the resulting partial response + should be handled by the request errback. Note that 'fail' is a keyword-only argument. + """ + + def __init__(self, *, fail=True): + super().__init__() + self.fail = fail + + # Items @@ -59,6 +71,7 @@ class NotSupported(Exception): class UsageError(Exception): """To indicate a command-line usage error""" + def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) super(UsageError, self).__init__(*a, **kw) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index a311e9257..115707182 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -5,13 +5,14 @@ import logging from twisted.internet.defer import DeferredList, Deferred from twisted.python.failure import Failure -from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \ - getAllReceivers, disconnect +from pydispatch.dispatcher import Anonymous, Any, disconnect, getAllReceivers, liveReceivers from pydispatch.robustapply import robustApply +from scrapy.exceptions import StopDownload from scrapy.utils.defer import maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info + logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. """ - dont_log = named.pop('dont_log', _IgnoredException) + dont_log = (named.pop('dont_log', _IgnoredException), StopDownload) spider = named.get('spider', None) responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): diff --git a/tests/spiders.py b/tests/spiders.py index 33d5d02e1..ebe81a1a9 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -7,6 +7,8 @@ from urllib.parse import urlencode from twisted.internet import defer +from scrapy import signals +from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.item import Item from scrapy.linkextractors import LinkExtractor @@ -267,3 +269,34 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider): def errback(self, failure): self.logger.info('[errback] status %i', failure.value.response.status) + + +class BytesReceivedCallbackSpider(MetaSpider): + + full_response_length = 2**18 + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.bytes_received, signals.bytes_received) + return spider + + def start_requests(self): + body = b"a" * self.full_response_length + url = self.mockserver.url("/alpayload") + yield Request(url, method="POST", body=body, errback=self.errback) + + def parse(self, response): + self.meta["response"] = response + + def errback(self, failure): + self.meta["failure"] = failure + + def bytes_received(self, data, request, spider): + raise StopDownload(fail=False) + + +class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): + + def bytes_received(self, data, request, spider): + raise StopDownload(fail=True) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 84f80d103..5e70064c2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -9,17 +9,31 @@ from pytest import mark from testfixtures import LogCapture from twisted.internet import defer from twisted.internet.ssl import Certificate +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy import signals from scrapy.crawler import CrawlerRunner +from scrapy.exceptions import StopDownload from scrapy.http import Request +from scrapy.http.response import Response from scrapy.utils.python import to_unicode from tests.mockserver import MockServer -from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, - SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback, - AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider, - AsyncDefAsyncioReqsReturnSpider) +from tests.spiders import ( + AsyncDefAsyncioReqsReturnSpider, + AsyncDefAsyncioReturnSpider, + AsyncDefAsyncioSpider, + AsyncDefSpider, + BrokenStartRequestsSpider, + BytesReceivedCallbackSpider, + BytesReceivedErrbackSpider, + CrawlSpiderWithErrback, + DelaySpider, + DuplicateStartRequestsSpider, + FollowAllSpider, + SimpleSpider, + SingleRequestSpider, +) class CrawlTestCase(TestCase): @@ -457,3 +471,21 @@ with multiples lines ip_address = crawler.spider.meta['responses'][0].ip_address self.assertIsInstance(ip_address, IPv4Address) self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) + + @defer.inlineCallbacks + def test_stop_download_callback(self): + crawler = self.runner.create_crawler(BytesReceivedCallbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("failure")) + self.assertIsInstance(crawler.spider.meta["response"], Response) + self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length) + + @defer.inlineCallbacks + def test_stop_download_errback(self): + crawler = self.runner.create_crawler(BytesReceivedErrbackSpider) + yield crawler.crawl(mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta.get("response")) + self.assertIsInstance(crawler.spider.meta["failure"], Failure) + self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) + self.assertIsInstance(crawler.spider.meta["failure"].response, Response) + self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length) diff --git a/tests/test_engine.py b/tests/test_engine.py index d781665dc..6696ee52e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -16,13 +16,15 @@ import sys from collections import defaultdict from urllib.parse import urlparse +from pydispatch import dispatcher +from testfixtures import LogCapture from twisted.internet import reactor, defer from twisted.trial import unittest from twisted.web import server, static, util -from pydispatch import dispatcher from scrapy import signals from scrapy.core.engine import ExecutionEngine +from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor @@ -90,7 +92,7 @@ def start_test_site(debug=False): r = static.File(root_dir) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - numbers = [str(x).encode("utf8") for x in range(2**14)] + numbers = [str(x).encode("utf8") for x in range(2**18)] r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") @@ -188,6 +190,16 @@ class CrawlerRun: self.signals_caught[sig] = signalargs +class StopDownloadCrawlerRun(CrawlerRun): + """ + Make sure raising the StopDownload exception stops the download of the response body + """ + + def bytes_received(self, data, request, spider): + super().bytes_received(data, request, spider) + raise StopDownload(fail=False) + + class EngineTest(unittest.TestCase): @defer.inlineCallbacks @@ -316,7 +328,7 @@ class EngineTest(unittest.TestCase): # signal was fired multiple times self.assertTrue(len(data) > 1) # bytes were received in order - numbers = [str(x).encode("utf8") for x in range(2**14)] + numbers = [str(x).encode("utf8") for x in range(2**18)] self.assertEqual(joined_data, b"".join(numbers)) def _assert_signals_caught(self): @@ -357,6 +369,45 @@ class EngineTest(unittest.TestCase): self.assertEqual(len(e.open_spiders), 0) +class StopDownloadEngineTest(EngineTest): + + @defer.inlineCallbacks + def test_crawler(self): + for spider in TestSpider, DictItemsSpider: + self.run = StopDownloadCrawlerRun(spider) + with LogCapture() as log: + yield self.run.run() + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + "Download stopped for <GET http://localhost:{}/redirected> from signal handler" + " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + "Download stopped for <GET http://localhost:{}/> from signal handler" + " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + log.check_present(("scrapy.core.downloader.handlers.http11", + "DEBUG", + "Download stopped for <GET http://localhost:{}/numbers> from signal handler" + " StopDownloadCrawlerRun.bytes_received".format(self.run.portno))) + self._assert_visited_urls() + self._assert_scheduled_requests(urls_to_visit=9) + self._assert_downloaded_responses() + self._assert_signals_caught() + self._assert_bytes_received() + + def _assert_bytes_received(self): + self.assertEqual(9, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + joined_data = b"".join(data) + self.assertTrue(len(data) == 1) # signal was fired only once + if self.run.getpath(request.url) == "/numbers": + # Received bytes are not the complete response. The exact amount depends + # on the buffer size, which can vary, so we only check that the amount + # of received bytes is strictly less than the full response. + numbers = [str(x).encode("utf8") for x in range(2**18)] + self.assertTrue(len(joined_data) < len(b"".join(numbers))) + + if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': start_test_site(debug=True) From 9514393bd1c9137c6a3b9964679de04bccbd0570 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 20 May 2020 15:28:54 -0300 Subject: [PATCH 18/28] Reduce amount of lines --- scrapy/core/downloader/handlers/http11.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 555a3a032..a710761c5 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -523,10 +523,8 @@ class _ResponseReader(protocol.Protocol): logger.debug("Download stopped for %(request)s from signal handler %(handler)s", {"request": self._request, "handler": handler.__qualname__}) self.transport._producer.loseConnection() - self._finish_response( - flags=["download_stopped"], - failure=result if result.value.fail else None, - ) + failure = result if result.value.fail else None + self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " From fbe915d9cbd8ab801a679b05265cff416764d558 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 20 May 2020 20:20:30 -0300 Subject: [PATCH 19/28] StopDownload: check partial response contents --- tests/spiders.py | 2 ++ tests/test_crawl.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index ebe81a1a9..05078cc04 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -293,10 +293,12 @@ class BytesReceivedCallbackSpider(MetaSpider): self.meta["failure"] = failure def bytes_received(self, data, request, spider): + self.meta["bytes_received"] = data raise StopDownload(fail=False) class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider): def bytes_received(self, data, request, spider): + self.meta["bytes_received"] = data raise StopDownload(fail=True) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5e70064c2..e2426e0da 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -478,7 +478,8 @@ with multiples lines yield crawler.crawl(mockserver=self.mockserver) self.assertIsNone(crawler.spider.meta.get("failure")) self.assertIsInstance(crawler.spider.meta["response"], Response) - self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length) + self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received")) + self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length) @defer.inlineCallbacks def test_stop_download_errback(self): @@ -488,4 +489,5 @@ with multiples lines self.assertIsInstance(crawler.spider.meta["failure"], Failure) self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) self.assertIsInstance(crawler.spider.meta["failure"].response, Response) - self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length) + self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received")) + self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length) From f6ed5edc31e7cc66225c0860e1534a6230511954 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 18 Nov 2016 09:14:54 -0300 Subject: [PATCH 20/28] CookiesMiddleware: keep cookies from 'Cookie' request header --- docs/topics/downloader-middleware.rst | 5 + docs/topics/logging.rst | 3 + scrapy/downloadermiddlewares/cookies.py | 78 ++++++++--- tests/test_downloadermiddleware_cookies.py | 145 ++++++++++++++++++--- 4 files changed, 190 insertions(+), 41 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1a87d07b6..323e553e5 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -202,6 +202,11 @@ CookiesMiddleware sends them back on subsequent requests (from that spider), just like web browsers do. + .. caution:: When non-UTF8 encoded byte sequences are passed to a + :class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log + a warning. Refer to :ref:`topics-logging-advanced-customization` + to customize the logging behaviour. + The following settings can be used to configure the cookie middleware: * :setting:`COOKIES_ENABLED` diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index e81091651..55065a1a3 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -202,6 +202,9 @@ A custom log format can be set for different actions by extending .. autoclass:: scrapy.logformatter.LogFormatter :members: + +.. _topics-logging-advanced-customization: + Advanced customization ---------------------- diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d57f04bc3..77048f389 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -29,8 +29,7 @@ class CookiesMiddleware: cookiejarkey = request.meta.get("cookiejar") jar = self.jars[cookiejarkey] - cookies = self._get_request_cookies(jar, request) - for cookie in cookies: + for cookie in self._get_request_cookies(jar, request): jar.set_cookie_if_ok(cookie, request) # set Cookie header @@ -68,28 +67,65 @@ class CookiesMiddleware: msg = "Received cookies from: {}\n{}".format(response, cookies) logger.debug(msg, extra={'spider': spider}) - def _format_cookie(self, cookie): - # build cookie string - cookie_str = '%s=%s' % (cookie['name'], cookie['value']) - - if cookie.get('path', None): - cookie_str += '; Path=%s' % cookie['path'] - if cookie.get('domain', None): - cookie_str += '; Domain=%s' % cookie['domain'] + def _format_cookie(self, cookie, request): + """ + Given a dict consisting of cookie components, return its string representation. + Decode from bytes if necessary. + """ + decoded = {} + for key in ("name", "value", "path", "domain"): + if not cookie.get(key): + if key in ("name", "value"): + msg = "Invalid cookie found in request {}: {} ('{}' is missing)" + logger.warning(msg.format(request, cookie, key)) + return + continue + if isinstance(cookie[key], str): + decoded[key] = cookie[key] + else: + try: + decoded[key] = cookie[key].decode("utf8") + except UnicodeDecodeError: + logger.warning("Non UTF-8 encoded cookie found in request %s: %s", + request, cookie) + decoded[key] = cookie[key].decode("latin1", errors="replace") + cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value")) + for key, value in decoded.items(): # path, domain + cookie_str += "; {}={}".format(key.capitalize(), value) return cookie_str def _get_request_cookies(self, jar, request): - if isinstance(request.cookies, dict): - cookie_list = [ - {'name': k, 'value': v} - for k, v in request.cookies.items() - ] - else: - cookie_list = request.cookies + """ + Extract cookies from a Request. Values from the `Request.cookies` attribute + take precedence over values from the `Cookie` request header. + """ + def get_cookies_from_header(jar, request): + cookie_header = request.headers.get("Cookie") + if not cookie_header: + return [] + cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";")) + cookie_list_unicode = [] + for cookie_bytes in cookie_gen_bytes: + try: + cookie_unicode = cookie_bytes.decode("utf8") + except UnicodeDecodeError: + logger.warning("Non UTF-8 encoded cookie found in request %s: %s", + request, cookie_bytes) + cookie_unicode = cookie_bytes.decode("latin1", errors="replace") + cookie_list_unicode.append(cookie_unicode) + response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode}) + return jar.make_cookies(response, request) - cookies = [self._format_cookie(x) for x in cookie_list] - headers = {'Set-Cookie': cookies} - response = Response(request.url, headers=headers) + def get_cookies_from_attribute(jar, request): + if not request.cookies: + return [] + elif isinstance(request.cookies, dict): + cookies = ({"name": k, "value": v} for k, v in request.cookies.items()) + else: + cookies = request.cookies + formatted = filter(None, (self._format_cookie(c, request) for c in cookies)) + response = Response(request.url, headers={"Set-Cookie": formatted}) + return jar.make_cookies(response, request) - return jar.make_cookies(response, request) + return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index d54434c8f..9ccc2110b 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,20 +1,21 @@ -import re import logging -from unittest import TestCase from testfixtures import LogCapture +from unittest import TestCase +from scrapy.downloadermiddlewares.cookies import CookiesMiddleware +from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from scrapy.exceptions import NotConfigured -from scrapy.downloadermiddlewares.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): def split_cookies(cookies): - return sorted(re.split(r";\s*", cookies.decode("latin1"))) + return sorted([s.strip() for s in to_bytes(cookies).split(b";")]) return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg) def setUp(self): @@ -61,12 +62,13 @@ class CookiesMiddlewareTest(TestCase): def test_setting_enabled_cookies_debug(self): crawler = get_crawler(settings_dict={'COOKIES_DEBUG': True}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture('scrapy.downloadermiddlewares.cookies', - propagate=False, - level=logging.DEBUG) as log: + with LogCapture( + 'scrapy.downloadermiddlewares.cookies', + propagate=False, + level=logging.DEBUG, + ) as log: req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', - headers={'Set-Cookie': 'C1=value1; path=/'}) + res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) mw.process_response(req, res, crawler.spider) req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) @@ -85,12 +87,13 @@ class CookiesMiddlewareTest(TestCase): def test_setting_disabled_cookies_debug(self): crawler = get_crawler(settings_dict={'COOKIES_DEBUG': False}) mw = CookiesMiddleware.from_crawler(crawler) - with LogCapture('scrapy.downloadermiddlewares.cookies', - propagate=False, - level=logging.DEBUG) as log: + with LogCapture( + 'scrapy.downloadermiddlewares.cookies', + propagate=False, + level=logging.DEBUG, + ) as log: req = Request('http://scrapytest.org/') - res = Response('http://scrapytest.org/', - headers={'Set-Cookie': 'C1=value1; path=/'}) + res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) mw.process_response(req, res, crawler.spider) req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) @@ -102,8 +105,7 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req, self.spider) is None assert 'Cookie' not in req.headers - headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', - 'Other': b'ignore\xa3me'} + headers = {'Set-Cookie': b'C1=in\xa3valid; path=/', 'Other': b'ignore\xa3me'} res = Response('http://scrapytest.org/', headers=headers) assert self.mw.process_response(req, res, self.spider) is res @@ -124,7 +126,10 @@ class CookiesMiddlewareTest(TestCase): assert 'Cookie' not in req.headers # check that returned cookies are not merged back to jar - res = Response('http://scrapytest.org/dontmerge', headers={'Set-Cookie': 'dont=mergeme; path=/'}) + res = Response( + 'http://scrapytest.org/dontmerge', + headers={'Set-Cookie': 'dont=mergeme; path=/'}, + ) assert self.mw.process_response(req, res, self.spider) is res # check that cookies are merged back @@ -179,7 +184,11 @@ class CookiesMiddlewareTest(TestCase): self.assertCookieValEqual(req2.headers.get('Cookie'), b"C1=value1; galleta=salada") def test_cookiejar_key(self): - req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"}) + req = Request( + 'http://scrapytest.org/', + cookies={'galleta': 'salada'}, + meta={'cookiejar': "store1"}, + ) assert self.mw.process_request(req, self.spider) is None self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') @@ -191,7 +200,11 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(req2, self.spider) is None self.assertCookieValEqual(req2.headers.get('Cookie'), b'C1=value1; galleta=salada') - req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"}) + req3 = Request( + 'http://scrapytest.org/', + cookies={'galleta': 'dulce'}, + meta={'cookiejar': "store2"}, + ) assert self.mw.process_request(req3, self.spider) is None self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce') @@ -229,3 +242,95 @@ class CookiesMiddlewareTest(TestCase): assert self.mw.process_request(request, self.spider) is None self.assertIn('Cookie', request.headers) self.assertEqual(b'currencyCookie=USD', request.headers['Cookie']) + + def test_keep_cookie_from_default_request_headers_middleware(self): + DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty') + mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items()) + # overwrite with values from 'cookies' request argument + req1 = Request('http://example.org', cookies={'default': 'something'}) + assert mw_default_headers.process_request(req1, self.spider) is None + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'default=something; asdf=qwerty') + # keep both + req2 = Request('http://example.com', cookies={'a': 'b'}) + assert mw_default_headers.process_request(req2, self.spider) is None + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty') + + def test_keep_cookie_header(self): + # keep only cookies from 'Cookie' request header + req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], 'a=b; c=d') + # keep cookies from both 'Cookie' request header and 'cookies' keyword + req2 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'}, cookies={'e': 'f'}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], 'a=b; c=d; e=f') + # overwrite values from 'Cookie' request header with 'cookies' keyword + req3 = Request( + 'http://scrapytest.org', + headers={'Cookie': 'a=b; c=d'}, + cookies={'a': 'new', 'e': 'f'}, + ) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], 'a=new; c=d; e=f') + + def test_request_cookies_encoding(self): + # 1) UTF8-encoded bytes + req1 = Request('http://example.org', cookies={'a': u'á'.encode('utf8')}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') + + # 2) Non UTF8-encoded bytes + req2 = Request('http://example.org', cookies={'a': u'á'.encode('latin1')}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') + + # 3) Unicode string + req3 = Request('http://example.org', cookies={'a': u'á'}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + + def test_request_headers_cookie_encoding(self): + # 1) UTF8-encoded bytes + req1 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('utf8')}) + assert self.mw.process_request(req1, self.spider) is None + self.assertCookieValEqual(req1.headers['Cookie'], b'a=\xc3\xa1') + + # 2) Non UTF8-encoded bytes + req2 = Request('http://example.org', headers={'Cookie': u'a=á'.encode('latin1')}) + assert self.mw.process_request(req2, self.spider) is None + self.assertCookieValEqual(req2.headers['Cookie'], b'a=\xc3\xa1') + + # 3) Unicode string + req3 = Request('http://example.org', headers={'Cookie': u'a=á'}) + assert self.mw.process_request(req3, self.spider) is None + self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1') + + def test_invalid_cookies(self): + """ + Invalid cookies are logged as warnings and discarded + """ + with LogCapture( + 'scrapy.downloadermiddlewares.cookies', + propagate=False, + level=logging.INFO, + ) as lc: + cookies1 = [{'value': 'bar'}, {'name': 'key', 'value': 'value1'}] + req1 = Request('http://example.org/1', cookies=cookies1) + assert self.mw.process_request(req1, self.spider) is None + cookies2 = [{'name': 'foo'}, {'name': 'key', 'value': 'value2'}] + req2 = Request('http://example.org/2', cookies=cookies2) + assert self.mw.process_request(req2, self.spider) is None + lc.check( + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request <GET http://example.org/1>:" + " {'value': 'bar'} ('name' is missing)"), + ("scrapy.downloadermiddlewares.cookies", + "WARNING", + "Invalid cookie found in request <GET http://example.org/2>:" + " {'name': 'foo'} ('value' is missing)"), + ) + self.assertCookieValEqual(req1.headers['Cookie'], 'key=value1') + self.assertCookieValEqual(req2.headers['Cookie'], 'key=value2') From 08756cd87a7699dd0f73dcdc3178671fb6fec2d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Fri, 22 May 2020 18:06:54 +0200 Subject: [PATCH 21/28] Fix the description of the ScreenshotPipeline example item pipeline --- docs/topics/item-pipeline.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 533f84630..c9194caa1 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -167,11 +167,13 @@ method and how to clean up the resources properly.:: Take screenshot of item ----------------------- -This example demonstrates how to return a -:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method. -It uses Splash_ to render screenshot of item url. Pipeline -makes request to locally running instance of Splash_. After request is downloaded, -it saves the screenshot to a file and adds filename to the item. +This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in +the :meth:`process_item` method. + +This item pipeline makes a request to a locally-running instance of Splash_ to +render a screenshot of the item URL. After the request response is downloaded, +the item pipeline saves the screenshot to a file and adds the filename to the +item. :: From 5b2af852892faa3bf293225e6198f0cb1bc3bd85 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf <sda@fstab.net> Date: Fri, 22 May 2020 15:09:58 -0400 Subject: [PATCH 22/28] Link to issue describing troublesome pytest versions --- tests/requirements-py3.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 05b794d81..91fa1c5b5 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,6 +2,7 @@ jmespath mitmproxy; python_version >= '3.6' mitmproxy<4.0.0; python_version < '3.6' +# https://github.com/pytest-dev/pytest-twisted/issues/93 pytest != 5.4, != 5.4.1 pytest-cov pytest-twisted >= 1.11 From 0cc0e51ca330ae5d88d7daaf027d13dd8783603f Mon Sep 17 00:00:00 2001 From: Rohit Sharma <rohit_sh1@yahoo.ca> Date: Sun, 24 May 2020 13:38:44 -0700 Subject: [PATCH 23/28] remove uneeded escape sequence removed uneeded wscape sequences from method arguments in the docs folder --- docs/topics/api.rst | 2 +- docs/topics/contracts.rst | 2 +- docs/topics/exporters.rst | 12 ++++++------ docs/topics/loaders.rst | 24 ++++++++++++------------ docs/topics/spiders.rst | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 1c461a511..52509ffdf 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -91,7 +91,7 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(\*args, \**kwargs) + .. method:: crawl(*args, **kwargs) Starts the crawler by instantiating its spider class with the given ``args`` and ``kwargs`` arguments, while setting the execution engine in diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 319f577bc..b8b3078c4 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -78,7 +78,7 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, \*args) +.. class:: Contract(method, *args) :param method: callback function to which the contract is associated :type method: function diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index de8b51195..7daf25ab3 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -236,7 +236,7 @@ PythonItemExporter XmlItemExporter --------------- -.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs) +.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs) Exports Items in XML format to the specified file object. @@ -290,7 +290,7 @@ XmlItemExporter CsvItemExporter --------------- -.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs) +.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs) Exports Items in CSV format to the given file-like object. If the :attr:`fields_to_export` attribute is set, it will be used to define the @@ -323,7 +323,7 @@ CsvItemExporter PickleItemExporter ------------------ -.. class:: PickleItemExporter(file, protocol=0, \**kwargs) +.. class:: PickleItemExporter(file, protocol=0, **kwargs) Exports Items in pickle format to the given file-like object. @@ -343,7 +343,7 @@ PickleItemExporter PprintItemExporter ------------------ -.. class:: PprintItemExporter(file, \**kwargs) +.. class:: PprintItemExporter(file, **kwargs) Exports Items in pretty print format to the specified file object. @@ -363,7 +363,7 @@ PprintItemExporter JsonItemExporter ---------------- -.. class:: JsonItemExporter(file, \**kwargs) +.. class:: JsonItemExporter(file, **kwargs) Exports Items in JSON format to the specified file-like object, writing all objects as a list of objects. The additional ``__init__`` method arguments are @@ -392,7 +392,7 @@ JsonItemExporter JsonLinesItemExporter --------------------- -.. class:: JsonLinesItemExporter(file, \**kwargs) +.. class:: JsonLinesItemExporter(file, **kwargs) Exports Items in JSON format to the specified file-like object, writing one JSON-encoded item per line. The additional ``__init__`` method arguments are passed diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 5f75ccbff..eb804f1db 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -273,7 +273,7 @@ There are several ways to modify Item Loader context values: ItemLoader objects ================== -.. class:: ItemLoader([item, selector, response], \**kwargs) +.. class:: ItemLoader([item, selector, response], **kwargs) Return a new Item Loader for populating the given Item. If no item is given, one is instantiated automatically using the class in @@ -303,7 +303,7 @@ ItemLoader objects :class:`ItemLoader` instances have the following methods: - .. method:: get_value(value, \*processors, \**kwargs) + .. method:: get_value(value, *processors, **kwargs) Process the given ``value`` by the given ``processors`` and keyword arguments. @@ -321,7 +321,7 @@ ItemLoader objects >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') 'FOO` - .. method:: add_value(field_name, value, \*processors, \**kwargs) + .. method:: add_value(field_name, value, *processors, **kwargs) Process and then add the given ``value`` for the given field. @@ -343,11 +343,11 @@ ItemLoader objects loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)') loader.add_value(None, {'name': u'foo', 'sex': u'male'}) - .. method:: replace_value(field_name, value, \*processors, \**kwargs) + .. method:: replace_value(field_name, value, *processors, **kwargs) Similar to :meth:`add_value` but replaces the collected data with the new value instead of adding it. - .. method:: get_xpath(xpath, \*processors, \**kwargs) + .. method:: get_xpath(xpath, *processors, **kwargs) Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a value, which is used to extract a list of unicode strings from the @@ -367,7 +367,7 @@ ItemLoader objects # HTML snippet: <p id="price">the price is $1200</p> loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)') - .. method:: add_xpath(field_name, xpath, \*processors, \**kwargs) + .. method:: add_xpath(field_name, xpath, *processors, **kwargs) Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a value, which is used to extract a list of unicode strings from the @@ -385,12 +385,12 @@ ItemLoader objects # HTML snippet: <p id="price">the price is $1200</p> loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - .. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs) + .. method:: replace_xpath(field_name, xpath, *processors, **kwargs) Similar to :meth:`add_xpath` but replaces collected data instead of adding it. - .. method:: get_css(css, \*processors, \**kwargs) + .. method:: get_css(css, *processors, **kwargs) Similar to :meth:`ItemLoader.get_value` but receives a CSS selector instead of a value, which is used to extract a list of unicode strings @@ -410,7 +410,7 @@ ItemLoader objects # HTML snippet: <p id="price">the price is $1200</p> loader.get_css('p#price', TakeFirst(), re='the price is (.*)') - .. method:: add_css(field_name, css, \*processors, \**kwargs) + .. method:: add_css(field_name, css, *processors, **kwargs) Similar to :meth:`ItemLoader.add_value` but receives a CSS selector instead of a value, which is used to extract a list of unicode strings @@ -428,7 +428,7 @@ ItemLoader objects # HTML snippet: <p id="price">the price is $1200</p> loader.add_css('price', 'p#price', re='the price is (.*)') - .. method:: replace_css(field_name, css, \*processors, \**kwargs) + .. method:: replace_css(field_name, css, *processors, **kwargs) Similar to :meth:`add_css` but replaces collected data instead of adding it. @@ -678,7 +678,7 @@ Here is a list of all built-in processors: >>> proc(['one', 'two', 'three']) 'one<br>two<br>three' -.. class:: Compose(\*functions, \**default_loader_context) +.. class:: Compose(*functions, **default_loader_context) A processor which is constructed from the composition of the given functions. This means that each input value of this processor is passed to @@ -706,7 +706,7 @@ Here is a list of all built-in processors: active Loader context accessible through the :meth:`ItemLoader.context` attribute. -.. class:: MapCompose(\*functions, \**default_loader_context) +.. class:: MapCompose(*functions, **default_loader_context) A processor which is constructed from the composition of the given functions, similar to the :class:`Compose` processor. The difference with diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 231db6cea..8ff5e7292 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -121,7 +121,7 @@ scrapy.Spider send log messages through it as described on :ref:`topics-logging-from-spiders`. - .. method:: from_crawler(crawler, \*args, \**kwargs) + .. method:: from_crawler(crawler, *args, **kwargs) This is the class method used by Scrapy to create your spiders. From 898bdd3feb23de793ae0b4c725545c32050fc0ea Mon Sep 17 00:00:00 2001 From: Will Beaufoy <will@willbeaufoy.net> Date: Sun, 10 May 2020 13:49:43 +0100 Subject: [PATCH 24/28] Update docs README with build instructions using tox None of the existing commands built the docs properly for me (I had to revert the changes in 901892d to docs/conf.py to get them to build properly, and even then no tooltips displayed). Building them with tox worked for me, but other developers say they can still use the original method, so the docs now contain both. --- docs/README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/README.rst b/docs/README.rst index 0a343cd19..0b7afa548 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you need to install watchdog (``pip install watchdog``) and then use:: make watch + +Alternative method using tox +---------------------------- + +To compile the documentation to HTML run the following command:: + + tox -e docs + +Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. From 492197e4773798fcb19ba934c03d5b48e0b7c1f2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 27 May 2020 10:45:07 -0300 Subject: [PATCH 25/28] StopDownload: store response in the exception instead of the failure --- docs/topics/exceptions.rst | 9 ++++++--- scrapy/core/downloader/handlers/http11.py | 8 ++------ scrapy/core/scraper.py | 10 +--------- tests/test_crawl.py | 10 +++++++--- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index 10b23edc4..583a50ab8 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -91,9 +91,12 @@ indicate that no further bytes should be downloaded for a response. The ``fail`` boolean parameter controls which method will handle the resulting response: -* If ``fail=True``, the request errback is called. The response object is - available as the ``response`` attribute of the received - :class:`~twisted.python.failure.Failure` object. This is the default behaviour. +* If ``fail=True`` (default), the request errback is called. The response object is + available as the ``response`` attribute of the ``StopDownload`` exception, + which is in turn stored as the ``value`` attribute of the received + :class:`~twisted.python.failure.Failure` object. This means that in an errback + defined as ``def errback(self, failure)``, the response can be accessed though + ``failure.value.response``. * If ``fail=False``, the request callback is called instead. diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a710761c5..22c9ac520 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -442,12 +442,8 @@ class ScrapyAgent: ip_address=result["ip_address"], ) if result.get("failure"): - # This failure is not the same object that will reach the errback, - # so we need to temporarily store the response in the exception. - # It will be moved to the failure in core/scraper.py - failure = result["failure"] - failure.value.response = response - return failure + result["failure"].value.response = response + return result["failure"] return response diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c52ada83b..6785e103d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -11,7 +11,7 @@ from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errba from scrapy.utils.spider import iterate_spider_output from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.log import logformatter_adapter, failure_to_exc_info -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest, StopDownload +from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy import signals from scrapy.http import Request, Response from scrapy.item import _BaseItem @@ -147,14 +147,6 @@ class Scraper: def call_spider(self, result, request, spider): result.request = request - # StopDownload exceptions: make the partial response an attribute of the failure - if ( - isinstance(result, Failure) - and isinstance(result.value, StopDownload) - and hasattr(result.value, "response") - ): - result.response = result.value.response - delattr(result.value, "response") dfd = defer_result(result) callback = request.callback or spider.parse warn_on_generator_with_return_value(spider, callback) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index e2426e0da..0115b8fb9 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -488,6 +488,10 @@ with multiples lines self.assertIsNone(crawler.spider.meta.get("response")) self.assertIsInstance(crawler.spider.meta["failure"], Failure) self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload) - self.assertIsInstance(crawler.spider.meta["failure"].response, Response) - self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received")) - self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length) + self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response) + self.assertEqual( + crawler.spider.meta["failure"].value.response.body, + crawler.spider.meta.get("bytes_received")) + self.assertLess( + len(crawler.spider.meta["failure"].value.response.body), + crawler.spider.full_response_length) From 44d5801b05280ad633e63a6b25db264d6586d882 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Wed, 27 May 2020 11:13:21 -0300 Subject: [PATCH 26/28] Fix KeyError in LocalWeakReferencedCache --- scrapy/utils/datatypes.py | 4 ++-- tests/test_utils_datatypes.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index f59f4cc55..2a92d0588 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -105,8 +105,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): def __getitem__(self, key): try: return super(LocalWeakReferencedCache, self).__getitem__(key) - except TypeError: - return None # key is not weak-referenceable, it's not cached + except (TypeError, KeyError): + return None # key is either not weak-referenceable or not cached class SequenceExclude: diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 0a4c6034a..aa18ef1f3 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -271,6 +271,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): self.assertNotIn(r1, cache) self.assertIn(r2, cache) self.assertIn(r3, cache) + self.assertEqual(cache[r1], None) self.assertEqual(cache[r2], 2) self.assertEqual(cache[r3], 3) del r2 From 5cef927944152f11d296566767c1d794b34b38e4 Mon Sep 17 00:00:00 2001 From: Bulat Khabibullin <xb.bolo@gmail.com> Date: Mon, 1 Jun 2020 07:57:23 +0300 Subject: [PATCH 27/28] Implement TextResponse.json() (#4574) --- docs/topics/request-response.rst | 5 +++++ scrapy/http/response/text.py | 12 ++++++++++++ tests/test_http_response.py | 21 +++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ae25ff7e4..bbd715766 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -879,6 +879,11 @@ TextResponse objects .. automethod:: TextResponse.follow_all + .. automethod:: TextResponse.json() + + Returns a Python object from deserialized JSON document. + The result is cached after the first call. + HtmlResponse objects -------------------- diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0603b6653..40cf3f483 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +import json import warnings from contextlib import suppress from typing import Generator @@ -21,10 +22,13 @@ from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url +_NONE = object() + class TextResponse(Response): _DEFAULT_ENCODING = 'ascii' + _cached_decoded_json = _NONE def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) @@ -68,6 +72,14 @@ class TextResponse(Response): ScrapyDeprecationWarning, stacklevel=2) return self.text + def json(self): + """ + Deserialize a JSON document to a Python object. + """ + if self._cached_decoded_json is _NONE: + self._cached_decoded_json = json.loads(self.text) + return self._cached_decoded_json + @property def text(self): """ Body as unicode """ diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 039e863f4..e0ca3c0e6 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,4 +1,5 @@ import unittest +from unittest import mock from warnings import catch_warnings from w3lib.encoding import resolve_encoding @@ -685,6 +686,26 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + def test_json_response(self): + json_body = b"""{"ip": "109.187.217.200"}""" + json_response = self.response_class("http://www.example.com", body=json_body) + self.assertEqual(json_response.json(), {'ip': '109.187.217.200'}) + + text_body = b"""<html><body>text</body></html>""" + text_response = self.response_class("http://www.example.com", body=text_body) + with self.assertRaises(ValueError): + text_response.json() + + def test_cache_json_response(self): + json_valid_bodies = [b"""{"ip": "109.187.217.200"}""", b"""null"""] + for json_body in json_valid_bodies: + json_response = self.response_class("http://www.example.com", body=json_body) + + with mock.patch('json.loads') as mock_json: + for _ in range(2): + json_response.json() + mock_json.assert_called_once_with(json_body.decode()) + class HtmlResponseTest(TextResponseTest): From 91e505edfe811c8faa6e457ecdae42c3644e6537 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com> Date: Tue, 2 Jun 2020 05:32:14 -0300 Subject: [PATCH 28/28] Return single element from coroutine callback (#4609) --- scrapy/utils/spider.py | 8 ++++++-- tests/spiders.py | 11 +++++++++++ tests/test_crawl.py | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 1b8a82829..7e7a50c88 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,5 +1,5 @@ -import logging import inspect +import logging from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro @@ -18,7 +18,11 @@ def iterate_spider_output(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d - return arg_to_iter(deferred_from_coro(result)) + elif inspect.iscoroutine(result): + d = deferred_from_coro(result) + d.addCallback(iterate_spider_output) + return d + return arg_to_iter(result) def iter_spider_classes(module): diff --git a/tests/spiders.py b/tests/spiders.py index 05078cc04..a360d8206 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -119,6 +119,17 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): return [{'id': 1}, {'id': 2}] +class AsyncDefAsyncioReturnSingleElementSpider(SimpleSpider): + + name = "asyncdef_asyncio_return_single_element" + + async def parse(self, response): + await asyncio.sleep(0.1) + status = await get_from_asyncio_queue(response.status) + self.logger.info("Got response %d" % status) + return {"foo": 42} + + class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): name = 'asyncdef_asyncio_reqs_return' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 0115b8fb9..df920f2a2 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -21,6 +21,7 @@ from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( AsyncDefAsyncioReqsReturnSpider, + AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, AsyncDefAsyncioSpider, AsyncDefSpider, @@ -364,6 +365,21 @@ with multiples lines self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncio_parse_items_single_element(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + self.assertIn({"foo": 42}, items) + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks