From 43fd6229684b3ccca564524fc92faf009a8c4c97 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 10:21:50 +0000
Subject: [PATCH 01/10] Rule.process_request: optionally take a Response object

---
 scrapy/spiders/crawl.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e5ac72e18..5aec0fd83 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -24,12 +24,23 @@ class Rule(object):
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request = process_request
+        self.process_request_function = process_request
         if follow is None:
             self.follow = False if callback else True
         else:
             self.follow = follow
 
+    def process_request(self, request, response):
+        """
+        Wrapper around the request processing function to maintain backward compatibility
+        with functions that do not take a Response object as parameter.
+        """
+        argcount = self.process_request_function.__code__.co_argcount
+        if getattr(self.process_request_function, '__self__', None):
+            argcount = argcount - 1
+        args = [request] if argcount == 1 else [request, response]
+        return self.process_request_function(*args)
+
 
 class CrawlSpider(Spider):
 
@@ -65,7 +76,7 @@ class CrawlSpider(Spider):
             for link in links:
                 seen.add(link)
                 r = self._build_request(n, link)
-                yield rule.process_request(r)
+                yield rule.process_request(r, response)
 
     def _response_downloaded(self, response):
         rule = self._rules[response.meta['rule']]
@@ -93,7 +104,7 @@ class CrawlSpider(Spider):
         for rule in self._rules:
             rule.callback = get_method(rule.callback)
             rule.process_links = get_method(rule.process_links)
-            rule.process_request = get_method(rule.process_request)
+            rule.process_request_function = get_method(rule.process_request_function)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):

From 22fda61d62a2b230b0e8588eabb0d71cb77141b7 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 10:54:38 +0000
Subject: [PATCH 02/10] Rule.process_request: tests

---
 tests/test_spider.py | 98 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index fefdaa403..5e20e0d99 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -263,6 +263,104 @@ class CrawlSpiderTest(SpiderTest):
                            'http://example.org/about.html',
                            'http://example.org/nofollow.html'])
 
+    def test_process_request(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        def process_request_change_domain(request):
+            return request.replace(url=request.url.replace('.org', '.com'))
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request=process_request_change_domain),
+            )
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.com/somepage/item/12.html',
+                          'http://example.com/about.html',
+                          'http://example.com/nofollow.html'])
+
+    def test_process_request_with_response(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        def process_request_meta_response_class(request, response):
+            request.meta['response_class'] = response.__class__.__name__
+            return request
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request=process_request_meta_response_class),
+            )
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
+        self.assertEqual([r.meta['response_class'] for r in output],
+                         ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+
+    def test_process_request_instance_method(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request='process_request_upper'),
+            )
+
+            def process_request_upper(self, request):
+                return request.replace(url=request.url.upper())
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                          'http://EXAMPLE.ORG/ABOUT.HTML',
+                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+
+    def test_process_request_instance_method_with_response(self):
+
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
+
+        class _CrawlSpider(self.spider_class):
+            name="test"
+            allowed_domains=['example.org']
+            rules = (
+                Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
+            )
+
+            def process_request_meta_response_class(self, request, response):
+                request.meta['response_class'] = response.__class__.__name__
+                return request
+
+        spider = _CrawlSpider()
+        output = list(spider._requests_to_follow(response))
+        self.assertEqual(len(output), 3)
+        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertEqual([r.url for r in output],
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
+        self.assertEqual([r.meta['response_class'] for r in output],
+                         ['HtmlResponse', 'HtmlResponse', 'HtmlResponse'])
+
     def test_follow_links_attribute_population(self):
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, 'example.com')

From b30ca379b6785c7ceb75e12285fe7865b4f607d1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 11:02:51 +0000
Subject: [PATCH 03/10] Rule.process_request: docs

---
 docs/topics/spiders.rst | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 742a88659..24b6f7ec9 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -402,10 +402,12 @@ Crawling rules
    of links extracted from each response using the specified ``link_extractor``.
    This is mainly used for filtering purposes.
 
-   ``process_request`` is a callable, or a string (in which case a method from
-   the spider object with that name will be used) which will be called with
-   every request extracted by this rule, and must return a request or None (to
-   filter out the request).
+   ``process_request`` is a callable (or a string, in which case a method from
+   the spider object with that name will be used) which will be called for
+   every request extracted by this rule. This callable should take a Request object
+   as first positional argument and, optionally, the Response object from which the
+   Request originated as second positional argument. It must return a request or None
+   (to filter out the request).
 
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~

From 83ec947fe732035e147c21df352e199ce2cce5c8 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Wed, 13 Mar 2019 11:23:51 +0000
Subject: [PATCH 04/10] Rule.process_request defaults to None in the docs

---
 scrapy/spiders/crawl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 5aec0fd83..ad86fc19d 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -19,12 +19,12 @@ def identity(x):
 
 class Rule(object):
 
-    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
+    def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
         self.link_extractor = link_extractor
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request_function = process_request
+        self.process_request_function = process_request or identity
         if follow is None:
             self.follow = False if callback else True
         else:

From 01ed605d02013b1d7955369562b2443d2a561599 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 15 Mar 2019 16:54:14 +0000
Subject: [PATCH 05/10] PEP8 changes to test_spider.py

---
 tests/test_spider.py | 60 +++++++++++++++++++++-----------------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/tests/test_spider.py b/tests/test_spider.py
index 5e20e0d99..c9af7a2d7 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -105,11 +105,11 @@ class SpiderTest(unittest.TestCase):
 
     def test_logger(self):
         spider = self.spider_class('example.com')
-        with LogCapture() as l:
+        with LogCapture() as lc:
             spider.logger.info('test log msg')
-        l.check(('example.com', 'INFO', 'test log msg'))
+        lc.check(('example.com', 'INFO', 'test log msg'))
 
-        record = l.records[0]
+        record = lc.records[0]
         self.assertIn('spider', record.__dict__)
         self.assertIs(record.spider, spider)
 
@@ -190,12 +190,11 @@ class CrawlSpiderTest(SpiderTest):
 
     def test_process_links(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html",
-            body=self.test_body)
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_links="dummy_process_links"),
             )
@@ -208,24 +207,24 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                          ['http://example.org/somepage/item/12.html',
-                           'http://example.org/about.html',
-                           'http://example.org/nofollow.html'])
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
 
     def test_process_links_filter(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html",
-            body=self.test_body)
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
             import re
 
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_links="filter_process_links"),
             )
             _test_regex = re.compile('nofollow')
+
             def filter_process_links(self, links):
                 return [link for link in links
                         if not self._test_regex.search(link.url)]
@@ -235,17 +234,16 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 2)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                          ['http://example.org/somepage/item/12.html',
-                           'http://example.org/about.html'])
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html'])
 
     def test_process_links_generator(self):
 
-        response = HtmlResponse("http://example.org/somepage/index.html",
-            body=self.test_body)
+        response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_links="dummy_process_links"),
             )
@@ -259,9 +257,9 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                          ['http://example.org/somepage/item/12.html',
-                           'http://example.org/about.html',
-                           'http://example.org/nofollow.html'])
+                         ['http://example.org/somepage/item/12.html',
+                          'http://example.org/about.html',
+                          'http://example.org/nofollow.html'])
 
     def test_process_request(self):
 
@@ -271,8 +269,8 @@ class CrawlSpiderTest(SpiderTest):
             return request.replace(url=request.url.replace('.org', '.com'))
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
@@ -295,8 +293,8 @@ class CrawlSpiderTest(SpiderTest):
             return request
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request=process_request_meta_response_class),
             )
@@ -317,8 +315,8 @@ class CrawlSpiderTest(SpiderTest):
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request='process_request_upper'),
             )
@@ -340,8 +338,8 @@ class CrawlSpiderTest(SpiderTest):
         response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
 
         class _CrawlSpider(self.spider_class):
-            name="test"
-            allowed_domains=['example.org']
+            name = "test"
+            allowed_domains = ['example.org']
             rules = (
                 Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
             )

From 92bbc5290d2b381ea60d68442a887d1ba020874e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Sat, 16 Mar 2019 05:41:40 +0000
Subject: [PATCH 06/10] Rule.process_request - Renaming

---
 scrapy/spiders/crawl.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index ad86fc19d..c01f75798 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -24,22 +24,22 @@ class Rule(object):
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request_function = process_request or identity
+        self.process_request = process_request or identity
         if follow is None:
             self.follow = False if callback else True
         else:
             self.follow = follow
 
-    def process_request(self, request, response):
+    def _process_request(self, request, response):
         """
         Wrapper around the request processing function to maintain backward compatibility
         with functions that do not take a Response object as parameter.
         """
-        argcount = self.process_request_function.__code__.co_argcount
-        if getattr(self.process_request_function, '__self__', None):
+        argcount = self.process_request.__code__.co_argcount
+        if hasattr(self.process_request, '__self__'):
             argcount = argcount - 1
         args = [request] if argcount == 1 else [request, response]
-        return self.process_request_function(*args)
+        return self.process_request(*args)
 
 
 class CrawlSpider(Spider):
@@ -75,8 +75,8 @@ class CrawlSpider(Spider):
                 links = rule.process_links(links)
             for link in links:
                 seen.add(link)
-                r = self._build_request(n, link)
-                yield rule.process_request(r, response)
+                request = self._build_request(n, link)
+                yield rule._process_request(request, response)
 
     def _response_downloaded(self, response):
         rule = self._rules[response.meta['rule']]
@@ -104,7 +104,7 @@ class CrawlSpider(Spider):
         for rule in self._rules:
             rule.callback = get_method(rule.callback)
             rule.process_links = get_method(rule.process_links)
-            rule.process_request_function = get_method(rule.process_request_function)
+            rule.process_request = get_method(rule.process_request)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):

From bbf24b7a1ce2e91eab57d1b8524d398822a1ddd1 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 18:02:31 -0300
Subject: [PATCH 07/10] Rule.process_request: use
 scrapy.utils.python.get_func_args

---
 scrapy/spiders/crawl.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index c01f75798..f474b0a18 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -10,6 +10,7 @@ import six
 
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
+from scrapy.utils.python import get_func_args
 from scrapy.spiders import Spider
 
 
@@ -35,10 +36,8 @@ class Rule(object):
         Wrapper around the request processing function to maintain backward compatibility
         with functions that do not take a Response object as parameter.
         """
-        argcount = self.process_request.__code__.co_argcount
-        if hasattr(self.process_request, '__self__'):
-            argcount = argcount - 1
-        args = [request] if argcount == 1 else [request, response]
+        arg_count = len(get_func_args(self.process_request))
+        args = [request] if arg_count == 1 else [request, response]
         return self.process_request(*args)
 
 

From 56929e77d98391255b77ffd3350abb49da18009e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 18:34:55 -0300
Subject: [PATCH 08/10] Rule.process_request: deprecate the use of functions
 taking only one argument

---
 scrapy/spiders/crawl.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f474b0a18..f469891d0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
+import warnings
+
 import six
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_func_args
 from scrapy.spiders import Spider
 
 
-def identity(x):
-    return x
+def _identity(request, response):
+    return request
 
 
 class Rule(object):
@@ -25,19 +28,21 @@ class Rule(object):
         self.callback = callback
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
-        self.process_request = process_request or identity
-        if follow is None:
-            self.follow = False if callback else True
-        else:
-            self.follow = follow
+        self.process_request = process_request or _identity
+        self.follow = follow if follow is not None else not callback
 
     def _process_request(self, request, response):
         """
-        Wrapper around the request processing function to maintain backward compatibility
-        with functions that do not take a Response object as parameter.
+        Wrapper around the request processing function to maintain backward
+        compatibility with functions that do not take a Response object
         """
         arg_count = len(get_func_args(self.process_request))
-        args = [request] if arg_count == 1 else [request, response]
+        if arg_count == 1:
+            args = [request]
+            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
+            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+        else:
+            args = [request, response]
         return self.process_request(*args)
 
 

From 174ba3cc5671cdc9e66cb29275986ff7481affc5 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 19:16:18 -0300
Subject: [PATCH 09/10] Rule.process_request: update docs

---
 docs/topics/spiders.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 24b6f7ec9..30e15906e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -403,11 +403,11 @@ Crawling rules
    This is mainly used for filtering purposes.
 
    ``process_request`` is a callable (or a string, in which case a method from
-   the spider object with that name will be used) which will be called for
-   every request extracted by this rule. This callable should take a Request object
-   as first positional argument and, optionally, the Response object from which the
-   Request originated as second positional argument. It must return a request or None
-   (to filter out the request).
+   the spider object with that name will be used) which will be called for every
+   :class:`~scrapy.http.Request` extracted by this rule. This callable should
+   take said request as first argument and the :class:`~scrapy.http.Response`
+   from which the request originated as second argument. It must return a
+   ``Request`` object or ``None`` (to filter out the request).
 
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~

From 1b4385b7e3f78694c0378455644b539d80d293a2 Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>
Date: Fri, 22 Mar 2019 19:46:17 -0300
Subject: [PATCH 10/10] Rule.process_request: move deprecation warnings and
 compiling code, update tests

---
 scrapy/spiders/crawl.py | 35 +++++++++++++++++++----------------
 tests/test_spider.py    | 38 ++++++++++++++++++++++----------------
 2 files changed, 41 insertions(+), 32 deletions(-)

diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f469891d0..6db3a1e06 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -21,6 +21,13 @@ def _identity(request, response):
     return request
 
 
+def _get_method(method, spider):
+    if callable(method):
+        return method
+    elif isinstance(method, six.string_types):
+        return getattr(spider, method, None)
+
+
 class Rule(object):
 
     def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
@@ -29,20 +36,24 @@ class Rule(object):
         self.cb_kwargs = cb_kwargs or {}
         self.process_links = process_links
         self.process_request = process_request or _identity
+        self.process_request_argcount = None
         self.follow = follow if follow is not None else not callback
 
+    def _compile(self, spider):
+        self.callback = _get_method(self.callback, spider)
+        self.process_links = _get_method(self.process_links, spider)
+        self.process_request = _get_method(self.process_request, spider)
+        self.process_request_argcount = len(get_func_args(self.process_request))
+        if self.process_request_argcount == 1:
+            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
+            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+
     def _process_request(self, request, response):
         """
         Wrapper around the request processing function to maintain backward
         compatibility with functions that do not take a Response object
         """
-        arg_count = len(get_func_args(self.process_request))
-        if arg_count == 1:
-            args = [request]
-            msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
-            warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
-        else:
-            args = [request, response]
+        args = [request] if self.process_request_argcount == 1 else [request, response]
         return self.process_request(*args)
 
 
@@ -98,17 +109,9 @@ class CrawlSpider(Spider):
                 yield request_or_item
 
     def _compile_rules(self):
-        def get_method(method):
-            if callable(method):
-                return method
-            elif isinstance(method, six.string_types):
-                return getattr(self, method, None)
-
         self._rules = [copy.copy(r) for r in self.rules]
         for rule in self._rules:
-            rule.callback = get_method(rule.callback)
-            rule.process_links = get_method(rule.process_links)
-            rule.process_request = get_method(rule.process_request)
+            rule._compile(self)
 
     @classmethod
     def from_crawler(cls, crawler, *args, **kwargs):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index c9af7a2d7..83fb68c2f 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -275,14 +275,17 @@ class CrawlSpiderTest(SpiderTest):
                 Rule(LinkExtractor(), process_request=process_request_change_domain),
             )
 
-        spider = _CrawlSpider()
-        output = list(spider._requests_to_follow(response))
-        self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://example.com/somepage/item/12.html',
-                          'http://example.com/about.html',
-                          'http://example.com/nofollow.html'])
+        with warnings.catch_warnings(record=True) as cw:
+            spider = _CrawlSpider()
+            output = list(spider._requests_to_follow(response))
+            self.assertEqual(len(output), 3)
+            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+            self.assertEqual([r.url for r in output],
+                             ['http://example.com/somepage/item/12.html',
+                              'http://example.com/about.html',
+                              'http://example.com/nofollow.html'])
+            self.assertEqual(len(cw), 1)
+            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_process_request_with_response(self):
 
@@ -324,14 +327,17 @@ class CrawlSpiderTest(SpiderTest):
             def process_request_upper(self, request):
                 return request.replace(url=request.url.upper())
 
-        spider = _CrawlSpider()
-        output = list(spider._requests_to_follow(response))
-        self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
-        self.assertEqual([r.url for r in output],
-                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                          'http://EXAMPLE.ORG/ABOUT.HTML',
-                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+        with warnings.catch_warnings(record=True) as cw:
+            spider = _CrawlSpider()
+            output = list(spider._requests_to_follow(response))
+            self.assertEqual(len(output), 3)
+            self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+            self.assertEqual([r.url for r in output],
+                             ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
+                              'http://EXAMPLE.ORG/ABOUT.HTML',
+                              'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+            self.assertEqual(len(cw), 1)
+            self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
 
     def test_process_request_instance_method_with_response(self):