diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 00ed8152c..33958cee5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -139,6 +139,16 @@ method, as follows:: >>> response.xpath('//title/text()').extract() [u'Example website'] +If you want to extract only first matched element, you can call the selector ``.extract_first()`` + + >>> sel.xpath('//div[@id="images"]/a/text()').extract_first() + u'Name: My image 1 ' + +It returns ``None`` if no element was found: + + >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first() is None + True + Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: @@ -226,6 +236,12 @@ Here's an example used to extract images names from the :ref:`HTML code u'My image 4', u'My image 5'] +There's an additional helper reciprocating ``.extract_first()`` for ``.re()``, +named ``.re_first()``. Use it to extract just the first matching string:: + + >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') + u'My image 1' + .. _topics-selectors-relative-xpaths: Working with relative XPaths diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index b8a3678a8..c0eefb85e 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -6,7 +6,7 @@ from lxml import etree from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref -from scrapy.utils.python import unicode_to_str, flatten +from scrapy.utils.python import unicode_to_str, flatten, iflatten from scrapy.utils.decorator import deprecated from scrapy.http import HtmlResponse, XmlResponse from .lxmldocument import LxmlDocument @@ -175,9 +175,17 @@ class SelectorList(list): def re(self, regex): return flatten([x.re(regex) for x in self]) + def re_first(self, regex): + for el in iflatten(x.re(regex) for x in self): + return el + def extract(self): return [x.extract() for x in self] + def extract_first(self): + for x in self: + return x.extract() + @deprecated(use_instead='.extract()') def extract_unquoted(self): return [x.extract_unquoted() for x in self] diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 551d337eb..b6100f899 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -27,13 +27,20 @@ def flatten(x): >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]""" - result = [] + return list(iflatten(x)) + + +def iflatten(x): + """iflatten(sequence) -> iterator + + Similar to ``.flatten()``, but returns iterator instead""" + for el in x: if hasattr(el, "__iter__"): - result.extend(flatten(el)) + for el_ in flatten(el): + yield el_ else: - result.append(el) - return result + yield el def unique(list_, key=lambda x: x): diff --git a/tests/test_selector.py b/tests/test_selector.py index 6fbb451a6..9b8613319 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -55,6 +55,41 @@ class SelectorTestCase(unittest.TestCase): [""] ) + def test_extract_first(self): + """Test if extract_first() returns first element""" + body = '' + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + self.assertEqual(sel.xpath('//ul/li/text()').extract_first(), + sel.xpath('//ul/li/text()').extract()[0]) + + self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(), + sel.xpath('//ul/li[@id="1"]/text()').extract()[0]) + + self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(), + sel.xpath('//ul/li/text()').extract()[1]) + + self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None) + + def test_re_first(self): + """Test if re_first() returns first matched element""" + body = '' + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'), + sel.xpath('//ul/li/text()').re('\d')[0]) + + self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'), + sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0]) + + self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'), + sel.xpath('//ul/li/text()').re('\d')[1]) + + self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None) + self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None) + def test_select_unicode_query(self): body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf8')