Merge pull request #506 from dangra/ahbeng-368-get-func-args-partial-support

get func args partial support - based on #504

scrapy/tests/test_contrib_loader.py

@@ -1,4 +1,5 @@
 import unittest
+from functools import partial
 
 from scrapy.contrib.loader import ItemLoader
 from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \
@@ -332,6 +333,29 @@ class BasicItemLoaderTest(unittest.TestCase):
         item = il.load_item()
         self.assertEqual(item['name'], u'Mart')
 
+    def test_partial_processor(self):
+        def join(values, sep=None, loader_context=None, ignored=None):
+            if sep is not None:
+                return sep.join(values)
+            elif loader_context and 'sep' in loader_context:
+                return loader_context['sep'].join(values)
+            else:
+                return ''.join(values)
+
+        class TestItemLoader(NameItemLoader):
+            name_out = Compose(partial(join, sep='+'))
+            url_out = Compose(partial(join, loader_context={'sep': '.'}))
+            summary_out = Compose(partial(join, ignored='foo'))
+
+        il = TestItemLoader()
+        il.add_value('name', [u'rabbit', u'hole'])
+        il.add_value('url', [u'rabbit', u'hole'])
+        il.add_value('summary', [u'rabbit', u'hole'])
+        item = il.load_item()
+        self.assertEqual(item['name'], u'rabbit+hole')
+        self.assertEqual(item['url'], u'rabbit.hole')
+        self.assertEqual(item['summary'], u'rabbithole')
+
 
 class ProcessorsTest(unittest.TestCase):
 

scrapy/tests/test_utils_python.py

@@ -1,3 +1,4 @@
+import functools
 import operator
 import unittest
 from itertools import count
@@ -64,7 +65,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         assert not isbinarytext("hello")
 
         # utf-16 strings contain null bytes
-        assert not isbinarytext(u"hello".encode('utf-16')) 
+        assert not isbinarytext(u"hello".encode('utf-16'))
 
         # one with encoding
         assert not isbinarytext("<div>Price \xa3</div>")
@@ -175,11 +176,17 @@ class UtilsPythonTestCase(unittest.TestCase):
 
         a = A(1, 2, 3)
         cal = Callable()
+        partial_f1 = functools.partial(f1, None)
+        partial_f2 = functools.partial(f1, b=None)
+        partial_f3 = functools.partial(partial_f2, None)
 
         self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
+        self.assertEqual(get_func_args(partial_f1), ['b', 'c'])
+        self.assertEqual(get_func_args(partial_f2), ['a', 'c'])
+        self.assertEqual(get_func_args(partial_f3), ['c'])
         self.assertEqual(get_func_args(cal), ['a', 'b', 'c'])
         self.assertEqual(get_func_args(object), [])
 

scrapy/utils/python.py

@@ -10,7 +10,7 @@ import re
 import inspect
 import weakref
 import errno
-from functools import wraps
+from functools import partial, wraps
 from sgmllib import SGMLParser
 
 
@@ -156,6 +156,9 @@ def get_func_args(func, stripself=False):
         return get_func_args(func.__func__, True)
     elif inspect.ismethoddescriptor(func):
         return []
+    elif isinstance(func, partial):
+        return [x for x in get_func_args(func.func)[len(func.args):]
+                if not (func.keywords and x in func.keywords)]
     elif hasattr(func, '__call__'):
         if inspect.isroutine(func):
             return []