From 56929e77d98391255b77ffd3350abb49da18009e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 18:34:55 -0300 Subject: [PATCH] Rule.process_request: deprecate the use of functions taking only one argument --- scrapy/spiders/crawl.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f474b0a18..f469891d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst """ import copy +import warnings + import six +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_func_args from scrapy.spiders import Spider -def identity(x): - return x +def _identity(request, response): + return request class Rule(object): @@ -25,19 +28,21 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request = process_request or identity - if follow is None: - self.follow = False if callback else True - else: - self.follow = follow + self.process_request = process_request or _identity + self.follow = follow if follow is not None else not callback def _process_request(self, request, response): """ - Wrapper around the request processing function to maintain backward compatibility - with functions that do not take a Response object as parameter. + Wrapper around the request processing function to maintain backward + compatibility with functions that do not take a Response object """ arg_count = len(get_func_args(self.process_request)) - args = [request] if arg_count == 1 else [request, response] + if arg_count == 1: + args = [request] + msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' + warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + else: + args = [request, response] return self.process_request(*args)