From 47a992615a046d20c31bfca6f8b65c3194e9fd30 Mon Sep 17 00:00:00 2001
From: Victor Torres <vpaivatorres@gmail.com>
Date: Wed, 15 Apr 2020 19:57:34 -0300
Subject: [PATCH] serialize requests with callback references as spider
 attribute

You could define a spider attribute that references a callback method
but if this method has a different name than your spider attribute,
the request serializer is not able to find it on the spider class.

With this commit we're fixing this behavior as we're searching for
callback references in the spider object itself instead of looking
for attributes with the same function's name, that could be different.
---
 scrapy/utils/reqser.py     | 12 ++++++++----
 tests/test_utils_reqser.py | 40 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 749bbc387..78e13ec10 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,6 +1,8 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
+import inspect
+
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
@@ -90,10 +92,12 @@ def _find_method(obj, func):
             pass
         else:
             if func_self is obj:
-                name = func.__func__.__name__
-                if _is_private_method(name):
-                    return _mangle_private_name(obj, func, name)
-                return name
+                members = inspect.getmembers(obj, predicate=inspect.ismethod)
+                for name, obj_func in members:
+                    if obj_func.__func__ is func.__func__:
+                        if _is_private_method(name):
+                            return _mangle_private_name(obj, func, name)
+                        return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
 
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index c7572f02c..cf84f8fbd 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
+    def test_reference_callback_serialization(self):
+        r = Request("http://www.example.com",
+                    callback=self.spider.parse_item_reference,
+                    errback=self.spider.handle_error_reference)
+        self._assert_serializes_ok(r, spider=self.spider)
+        request_dict = request_to_dict(r, self.spider)
+        self.assertEqual(request_dict['callback'], 'parse_item_reference')
+        self.assertEqual(request_dict['errback'], 'handle_error_reference')
+
+    def test_private_reference_callback_serialization(self):
+        r = Request("http://www.example.com",
+                    callback=self.spider._TestSpider__parse_item_reference,
+                    errback=self.spider._TestSpider__handle_error_reference)
+        self._assert_serializes_ok(r, spider=self.spider)
+        request_dict = request_to_dict(r, self.spider)
+        self.assertEqual(request_dict['callback'],
+                         '_TestSpider__parse_item_reference')
+        self.assertEqual(request_dict['errback'],
+                         '_TestSpider__handle_error_reference')
+
     def test_private_callback_serialization(self):
         r = Request("http://www.example.com",
                     callback=self.spider._TestSpider__parse_item_private,
@@ -131,8 +151,28 @@ class TestSpiderMixin:
         pass
 
 
+def parse_item(response):
+    pass
+
+
+def handle_error(failure):
+    pass
+
+
+def private_parse_item(response):
+    pass
+
+
+def private_handle_error(failure):
+    pass
+
+
 class TestSpider(Spider, TestSpiderMixin):
     name = 'test'
+    parse_item_reference = parse_item
+    handle_error_reference = handle_error
+    __parse_item_reference = private_parse_item
+    __handle_error_reference = private_handle_error
 
     def parse_item(self, response):
         pass