From 3e726b9df721f2288f4ae9a685de9bdff0762c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 11:22:05 +0100 Subject: [PATCH 1/3] Support for delegated methods as callbacks It can be useful to structure the spiders code around some helper classes. --- scrapy/utils/reqser.py | 29 ++++++++++++++--------------- tests/test_utils_reqser.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 5ea2aafb8..35a4fc72c 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -73,23 +73,22 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): if obj: try: - func_self = func.__self__ - except AttributeError: # func has no __self__ + func.__func__ + except AttributeError: # func is not a instance method. Not supported. pass else: - if func_self is obj: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name - raise ValueError("Function %s is not a method of: %s" % (func, obj)) + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name + raise ValueError("Function %s is not an instance method in: %s" % (func, obj)) def _get_method(obj, name): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index de94ec960..c8d1db138 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -102,6 +102,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_delegated_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider.delegated_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -131,6 +137,9 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass +class TestSpiderDelegation: + def delegated_callback(self, response): + pass def parse_item(response): pass @@ -155,6 +164,9 @@ class TestSpider(Spider, TestSpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error + def __init__(self): + self.delegated_callback = TestSpiderDelegation().delegated_callback + def parse_item(self, response): pass From 39affea93c5b60cde89e000486c39c3c0ce87dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Aug 2020 13:57:48 +0200 Subject: [PATCH 2/3] Fix style issues --- tests/test_utils_reqser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index c8d1db138..ee68cf6b1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -137,10 +137,12 @@ class TestSpiderMixin: def __mixin_callback(self, response): pass + class TestSpiderDelegation: def delegated_callback(self, response): pass + def parse_item(response): pass From 0524df866936506ae9438a5565fc4733fa5ba5b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20de=20Prado?= Date: Tue, 25 Aug 2020 14:36:38 +0100 Subject: [PATCH 3/3] Code simplification. Thanks @victor-torres for the suggestion --- scrapy/utils/reqser.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 35a4fc72c..503d7b133 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,23 +71,19 @@ def request_from_dict(d, spider=None): def _find_method(obj, func): - if obj: - try: - func.__func__ - except AttributeError: # func is not a instance method. Not supported. - pass - else: - members = inspect.getmembers(obj, predicate=inspect.ismethod) - for name, obj_func in members: - # We need to use __func__ to access the original - # function object because instance method objects - # are generated each time attribute is retrieved from - # instance. - # - # Reference: The standard type hierarchy - # https://docs.python.org/3/reference/datamodel.html - if obj_func.__func__ is func.__func__: - return name + # Only instance methods contain ``__func__`` + if obj and hasattr(func, '__func__'): + members = inspect.getmembers(obj, predicate=inspect.ismethod) + for name, obj_func in members: + # We need to use __func__ to access the original + # function object because instance method objects + # are generated each time attribute is retrieved from + # instance. + # + # Reference: The standard type hierarchy + # https://docs.python.org/3/reference/datamodel.html + if obj_func.__func__ is func.__func__: + return name raise ValueError("Function %s is not an instance method in: %s" % (func, obj))