Merge branch 'ananana-selectorlist-extract-first' from pull request #624

docs/topics/selectors.rst

@@ -139,6 +139,16 @@ method, as follows::
     >>> response.xpath('//title/text()').extract()
     [u'Example website']
 
+If you want to extract only first matched element, you can call the selector ``.extract_first()``
+
+    >>> sel.xpath('//div[@id="images"]/a/text()').extract_first()
+    u'Name: My image 1 '
+
+It returns ``None`` if no element was found:
+
+    >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first() is None
+    True
+
 Notice that CSS selectors can select text or attribute nodes using CSS3
 pseudo-elements::
 
@@ -226,6 +236,12 @@ Here's an example used to extract images names from the :ref:`HTML code
      u'My image 4',
      u'My image 5']
 
+There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
+named ``.re_first()``. Use it to extract just the first matching string::
+
+    >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
+    u'My image 1'
+
 .. _topics-selectors-relative-xpaths:
 
 Working with relative XPaths

scrapy/selector/unified.py

@@ -6,7 +6,7 @@ from lxml import etree
 
 from scrapy.utils.misc import extract_regex
 from scrapy.utils.trackref import object_ref
-from scrapy.utils.python import unicode_to_str, flatten
+from scrapy.utils.python import unicode_to_str, flatten, iflatten
 from scrapy.utils.decorator import deprecated
 from scrapy.http import HtmlResponse, XmlResponse
 from .lxmldocument import LxmlDocument
@@ -175,9 +175,17 @@ class SelectorList(list):
     def re(self, regex):
         return flatten([x.re(regex) for x in self])
 
+    def re_first(self, regex):
+        for el in iflatten(x.re(regex) for x in self):
+            return el
+
     def extract(self):
         return [x.extract() for x in self]
 
+    def extract_first(self):
+        for x in self:
+            return x.extract()
+
     @deprecated(use_instead='.extract()')
     def extract_unquoted(self):
         return [x.extract_unquoted() for x in self]

scrapy/utils/python.py

@@ -27,13 +27,20 @@ def flatten(x):
     >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
     [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]"""
 
-    result = []
+    return list(iflatten(x))
+
+
+def iflatten(x):
+    """iflatten(sequence) -> iterator
+
+    Similar to ``.flatten()``, but returns iterator instead"""
+
     for el in x:
         if hasattr(el, "__iter__"):
-            result.extend(flatten(el))
+            for el_ in flatten(el):
+                yield el_
         else:
-            result.append(el)
-    return result
+            yield el
 
 
 def unique(list_, key=lambda x: x):

tests/test_selector.py

@@ -55,6 +55,41 @@ class SelectorTestCase(unittest.TestCase):
             ["<Selector xpath=u'//input[@value=\"\\xa9\"]/@value' data=u'\\xa9'>"]
         )
 
+    def test_extract_first(self):
+        """Test if extract_first() returns first element"""
+        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        self.assertEqual(sel.xpath('//ul/li/text()').extract_first(),
+                         sel.xpath('//ul/li/text()').extract()[0])
+
+        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(),
+                         sel.xpath('//ul/li[@id="1"]/text()').extract()[0])
+
+        self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(),
+                         sel.xpath('//ul/li/text()').extract()[1])
+
+        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None)
+
+    def test_re_first(self):
+        """Test if re_first() returns first matched element"""
+        body = '<ul><li id="1">1</li><li id="2">2</li></ul>'
+        response = TextResponse(url="http://example.com", body=body)
+        sel = self.sscls(response)
+
+        self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
+                         sel.xpath('//ul/li/text()').re('\d')[0])
+
+        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
+                         sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])
+
+        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
+                         sel.xpath('//ul/li/text()').re('\d')[1])
+
+        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
+        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)
+
     def test_select_unicode_query(self):
         body = u"<p><input name='\xa9' value='1'/></p>"
         response = TextResponse(url="http://example.com", body=body, encoding='utf8')