Merge pull request #1016 from SudShekhar/jsonProcessor

[MRG+1] Added JmesSelect

docs/topics/loaders.rst

@@ -675,3 +675,27 @@ Here is a list of all built-in processors:
     constructor keyword arguments are used as default context values. See
     :class:`Compose` processor for more info.
 
+.. class:: SelectJmes(json_path)
+
+    Queries the value using the json path provided to the constructor and returns the output.
+    Requires jmespath (https://github.com/jmespath/jmespath) to run.
+    This processor takes only one input at a time.
+
+    Example::
+
+        >>> from scrapy.contrib.loader.processor import SelectJmes, Compose, MapCompose
+        >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
+        >>> proc({'foo': 'bar'})
+        'bar'
+        >>> proc({'foo': {'bar': 'baz'}})
+        {'bar': 'baz'}
+
+    Working with Json::
+
+        >>> import json
+        >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
+        >>> proc_single_json_str('{"foo": "bar"}')
+        u'bar'
+        >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
+        >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
+        [u'bar']

scrapy/contrib/loader/processor.py

@@ -8,6 +8,7 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.datatypes import MergeDict
 from .common import wrap_loader_context
 
+
 class MapCompose(object):
 
     def __init__(self, *functions, **default_loader_context):
@@ -63,6 +64,26 @@ class Identity(object):
         return values
 
 
+class SelectJmes(object):
+    """
+        Query the input string for the jmespath (given at instantiation),
+        and return the answer
+        Requires : jmespath(https://github.com/jmespath/jmespath)
+        Note: SelectJmes accepts only one input element at a time.
+    """
+    def __init__(self, json_path):
+        self.json_path = json_path
+        import jmespath
+        self.compiled_path = jmespath.compile(self.json_path)
+
+    def __call__(self, value):
+        """Query value for the jmespath query and return answer
+        :param str value: a string with JSON data to extract from
+        :return: Element extracted according to jmespath query
+        """
+        return self.compiled_path.search(value)
+
+
 class Join(object):
 
     def __init__(self, separator=u' '):

tests/requirements.txt

@@ -3,3 +3,4 @@ mock
 mitmproxy==0.10.1
 netlib==0.10.1
 pytest-twisted
+jmespath

tests/test_contrib_loader.py

@@ -3,12 +3,11 @@ from functools import partial
 
 from scrapy.contrib.loader import ItemLoader
 from scrapy.contrib.loader.processor import Join, Identity, TakeFirst, \
-    Compose, MapCompose
+    Compose, MapCompose, SelectJmes
 from scrapy.item import Item, Field
 from scrapy.selector import Selector
 from scrapy.http import HtmlResponse
 
-
 # test items
 class NameItem(Item):
     name = Field()
@@ -579,5 +578,30 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
 
 
+class SelectJmesTestCase(unittest.TestCase):
+        test_list_equals = {
+            'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+            'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
+            'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
+            'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
+            'dict': (
+                'foo.bar[*].name',
+                {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
+                ['one', 'two']
+            ),
+            'list': ('[1]', [1, 2], 2)
+        }
+
+        def test_output(self):
+            for l in self.test_list_equals:
+                expr, test_list, expected = self.test_list_equals[l]
+                test = SelectJmes(expr)(test_list)
+                self.assertEqual(
+                    test,
+                    expected,
+                    msg='test "{}" got {} expected {}'.format(l, test, expected)
+                )
+
+
 if __name__ == "__main__":
     unittest.main()