diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 959dddbd5..3c463cfed 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -67,6 +67,20 @@ def request_from_dict(d, spider=None): flags=d.get('flags')) +def _is_private_method(name): + return name.startswith('__') and not name.endswith('__') + + +def _mangle_private_name(obj, func, name): + qualname = getattr(func, '__qualname__', None) + if qualname is None: + classname = obj.__class__.__name__.lstrip('_') + return '_%s%s' % (classname, name) + else: + splits = qualname.split('.') + return '_%s%s' % (splits[-2], splits[-1]) + + def _find_method(obj, func): if obj: try: @@ -75,7 +89,10 @@ def _find_method(obj, func): pass else: if func_self is obj: - return six.get_method_function(func).__name__ + name = six.get_method_function(func).__name__ + if _is_private_method(name): + return _mangle_private_name(obj, func, name) + return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index dcc070b8f..e5a09dcf1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- import unittest +import sys + +import six from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name class RequestSerializationTest(unittest.TestCase): @@ -68,6 +71,56 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider._TestSpider__parse_item_private, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + + def test_mixin_private_callback_serialization(self): + if sys.version_info[0] < 3: + return + r = Request("http://www.example.com", + callback=self.spider._TestSpiderMixin__mixin_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + + def test_private_callback_name_matching(self): + self.assertTrue(_is_private_method('__a')) + self.assertTrue(_is_private_method('__a_')) + self.assertTrue(_is_private_method('__a_a')) + self.assertTrue(_is_private_method('__a_a_')) + self.assertTrue(_is_private_method('__a__a')) + self.assertTrue(_is_private_method('__a__a_')) + self.assertTrue(_is_private_method('__a___a')) + self.assertTrue(_is_private_method('__a___a_')) + self.assertTrue(_is_private_method('___a')) + self.assertTrue(_is_private_method('___a_')) + self.assertTrue(_is_private_method('___a_a')) + self.assertTrue(_is_private_method('___a_a_')) + self.assertTrue(_is_private_method('____a_a_')) + + self.assertFalse(_is_private_method('_a')) + self.assertFalse(_is_private_method('_a_')) + self.assertFalse(_is_private_method('__a__')) + self.assertFalse(_is_private_method('__')) + self.assertFalse(_is_private_method('___')) + self.assertFalse(_is_private_method('____')) + + def _assert_mangles_to(self, obj, name): + func = getattr(obj, name) + self.assertEqual( + _mangle_private_name(obj, func, func.__name__), + name + ) + + def test_private_name_mangling(self): + self._assert_mangles_to( + self.spider, '_TestSpider__parse_item_private') + if sys.version_info[0] >= 3: + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -78,7 +131,12 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpider(Spider): +class TestSpiderMixin(object): + def __mixin_callback(self, response): + pass + + +class TestSpider(Spider, TestSpiderMixin): name = 'test' def parse_item(self, response): @@ -87,6 +145,9 @@ class TestSpider(Spider): def handle_error(self, failure): pass + def __parse_item_private(self, response): + pass + class CustomRequest(Request): pass