From 09c3f536939e3a23b388a009a0fee6d41999e7b6 Mon Sep 17 00:00:00 2001 From: Beng Hee Eu Date: Mon, 30 Dec 2013 03:46:10 +0800 Subject: [PATCH 1/3] Fixes #368: Support functools.partial in scrapy.utils.python.get_func_args() --- scrapy/tests/test_utils_python.py | 3 +++ scrapy/utils/python.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index 24e962754..ca73bc5ac 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -1,3 +1,4 @@ +import functools import operator import unittest from itertools import count @@ -174,12 +175,14 @@ class UtilsPythonTestCase(unittest.TestCase): pass a = A(1, 2, 3) + partial_f1 = functools.partial(f1, None) cal = Callable() self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) self.assertEqual(get_func_args(A), ['a', 'b', 'c']) self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) + self.assertEqual(get_func_args(partial_f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(cal), ['a', 'b', 'c']) self.assertEqual(get_func_args(object), []) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 834773072..454f5a59f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,7 @@ import re import inspect import weakref import errno -from functools import wraps +from functools import partial, wraps from sgmllib import SGMLParser @@ -156,6 +156,8 @@ def get_func_args(func, stripself=False): return get_func_args(func.__func__, True) elif inspect.ismethoddescriptor(func): return [] + elif isinstance(func, partial): + return get_func_args(func.func) elif hasattr(func, '__call__'): if inspect.isroutine(func): return [] From 589fd037d9a156733580bc4b3f065c86aa54af23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 30 Dec 2013 09:25:11 -0200 Subject: [PATCH 2/3] do not return applied arguments on partial functions --- scrapy/tests/test_utils_python.py | 10 +++++++--- scrapy/utils/python.py | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index ca73bc5ac..9122cf1be 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -65,7 +65,7 @@ class UtilsPythonTestCase(unittest.TestCase): assert not isbinarytext("hello") # utf-16 strings contain null bytes - assert not isbinarytext(u"hello".encode('utf-16')) + assert not isbinarytext(u"hello".encode('utf-16')) # one with encoding assert not isbinarytext("
Price \xa3
") @@ -175,14 +175,18 @@ class UtilsPythonTestCase(unittest.TestCase): pass a = A(1, 2, 3) - partial_f1 = functools.partial(f1, None) cal = Callable() + partial_f1 = functools.partial(f1, None) + partial_f2 = functools.partial(f1, b=None) + partial_f3 = functools.partial(partial_f2, None) self.assertEqual(get_func_args(f1), ['a', 'b', 'c']) self.assertEqual(get_func_args(f2), ['a', 'b', 'c']) self.assertEqual(get_func_args(A), ['a', 'b', 'c']) self.assertEqual(get_func_args(a.method), ['a', 'b', 'c']) - self.assertEqual(get_func_args(partial_f1), ['a', 'b', 'c']) + self.assertEqual(get_func_args(partial_f1), ['b', 'c']) + self.assertEqual(get_func_args(partial_f2), ['a', 'c']) + self.assertEqual(get_func_args(partial_f3), ['c']) self.assertEqual(get_func_args(cal), ['a', 'b', 'c']) self.assertEqual(get_func_args(object), []) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 454f5a59f..566b50f17 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -157,7 +157,8 @@ def get_func_args(func, stripself=False): elif inspect.ismethoddescriptor(func): return [] elif isinstance(func, partial): - return get_func_args(func.func) + return [x for x in get_func_args(func.func)[len(func.args):] + if not (func.keywords and x in func.keywords)] elif hasattr(func, '__call__'): if inspect.isroutine(func): return [] From 22e107194bb35f83066c06e176417791f9129b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 30 Dec 2013 09:25:57 -0200 Subject: [PATCH 3/3] test loader processors using partial functions --- scrapy/tests/test_contrib_loader.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py index afd740a86..734bc31d1 100644 --- a/scrapy/tests/test_contrib_loader.py +++ b/scrapy/tests/test_contrib_loader.py @@ -1,4 +1,5 @@ import unittest +from functools import partial from scrapy.contrib.loader import ItemLoader from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \ @@ -332,6 +333,29 @@ class BasicItemLoaderTest(unittest.TestCase): item = il.load_item() self.assertEqual(item['name'], u'Mart') + def test_partial_processor(self): + def join(values, sep=None, loader_context=None, ignored=None): + if sep is not None: + return sep.join(values) + elif loader_context and 'sep' in loader_context: + return loader_context['sep'].join(values) + else: + return ''.join(values) + + class TestItemLoader(NameItemLoader): + name_out = Compose(partial(join, sep='+')) + url_out = Compose(partial(join, loader_context={'sep': '.'})) + summary_out = Compose(partial(join, ignored='foo')) + + il = TestItemLoader() + il.add_value('name', [u'rabbit', u'hole']) + il.add_value('url', [u'rabbit', u'hole']) + il.add_value('summary', [u'rabbit', u'hole']) + item = il.load_item() + self.assertEqual(item['name'], u'rabbit+hole') + self.assertEqual(item['url'], u'rabbit.hole') + self.assertEqual(item['summary'], u'rabbithole') + class ProcessorsTest(unittest.TestCase):