From 6c7bd54fc37d2f21bbbd0927eab2da827f070852 Mon Sep 17 00:00:00 2001 From: Mateusz Golewski Date: Thu, 30 Jan 2014 21:33:46 +0100 Subject: [PATCH 1/8] Add extract_first() method to SelectorList --- scrapy/selector/unified.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index b8a3678a8..7b877153d 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -178,6 +178,10 @@ class SelectorList(list): def extract(self): return [x.extract() for x in self] + def extract_first(self): + for x in self.extract(): + return x + @deprecated(use_instead='.extract()') def extract_unquoted(self): return [x.extract_unquoted() for x in self] From bd126be3569ddd77c458c79ef9e066cacf3a3af1 Mon Sep 17 00:00:00 2001 From: Mateusz Golewski Date: Thu, 30 Jan 2014 21:48:50 +0100 Subject: [PATCH 2/8] Optimize extract_first() --- scrapy/selector/unified.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 7b877153d..3d9435665 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -179,8 +179,8 @@ class SelectorList(list): return [x.extract() for x in self] def extract_first(self): - for x in self.extract(): - return x + for x in self: + return x.extract() @deprecated(use_instead='.extract()') def extract_unquoted(self): From 2742b4d8c26f946c20767fd7fd3f227d00002597 Mon Sep 17 00:00:00 2001 From: Mateusz Golewski Date: Thu, 30 Jan 2014 23:10:53 +0100 Subject: [PATCH 3/8] Add tests to extract_first() --- tests/test_selector.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_selector.py b/tests/test_selector.py index 6fbb451a6..80a9a4672 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -55,6 +55,23 @@ class SelectorTestCase(unittest.TestCase): [""] ) + def test_extract_first(self): + """Test if extract_first() returns first element""" + body = '
  • 1
  • 2
' + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + self.assertEqual(sel.xpath('//ul/li/text()').extract_first(), + sel.xpath('//ul/li/text()').extract()[0]) + + self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').extract_first(), + sel.xpath('//ul/li[@id="1"]/text()').extract()[0]) + + self.assertEqual(sel.xpath('//ul/li[2]/text()').extract_first(), + sel.xpath('//ul/li/text()').extract()[1]) + + self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None) + def test_select_unicode_query(self): body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf8') From 012211accda0fb5ce3af2d4010e2b51db33bdd02 Mon Sep 17 00:00:00 2001 From: Mateusz Golewski Date: Thu, 30 Jan 2014 23:39:15 +0100 Subject: [PATCH 4/8] Add docs for extract_first() --- docs/topics/selectors.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 00ed8152c..0ce0f084e 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -139,6 +139,16 @@ method, as follows:: >>> response.xpath('//title/text()').extract() [u'Example website'] +If you want to extract only first matched element, you must call the selector ``.extract_first()`` + + >>> sel.xpath('//ul/li').extract_first() + u'First list element' + +It returns ``None`` if no element was found: + + >>> sel.xpath('//ul/li[999]').extract_first() + None + Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: From 127c6c694a75e1448ddbd3d0f699ca8074c46761 Mon Sep 17 00:00:00 2001 From: Mateusz Golewski Date: Sun, 2 Feb 2014 15:02:25 +0100 Subject: [PATCH 5/8] Fix extract_first() docs --- docs/topics/selectors.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 0ce0f084e..92e092246 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -139,15 +139,15 @@ method, as follows:: >>> response.xpath('//title/text()').extract() [u'Example website'] -If you want to extract only first matched element, you must call the selector ``.extract_first()`` +If you want to extract only first matched element, you can call the selector ``.extract_first()`` - >>> sel.xpath('//ul/li').extract_first() - u'First list element' + >>> sel.xpath('//div[@id="images"]/a/text()').extract_first() + u'Name: My image 1 ' It returns ``None`` if no element was found: - >>> sel.xpath('//ul/li[999]').extract_first() - None + >>> sel.xpath('//div/[id="not-exists"]/text()').extract_first() is None + True Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: From f92bc09bf433c43ab5669f7bc14108ac6fd49e5c Mon Sep 17 00:00:00 2001 From: Mateusz Golewski Date: Sun, 2 Feb 2014 15:45:43 +0100 Subject: [PATCH 6/8] Add re_first() to SelectorList and iflatten() to utils.python --- scrapy/selector/unified.py | 6 +++++- scrapy/utils/python.py | 15 +++++++++++---- tests/test_selector.py | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 3d9435665..889c349e3 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -6,7 +6,7 @@ from lxml import etree from scrapy.utils.misc import extract_regex from scrapy.utils.trackref import object_ref -from scrapy.utils.python import unicode_to_str, flatten +from scrapy.utils.python import unicode_to_str, flatten, iflatten from scrapy.utils.decorator import deprecated from scrapy.http import HtmlResponse, XmlResponse from .lxmldocument import LxmlDocument @@ -175,6 +175,10 @@ class SelectorList(list): def re(self, regex): return flatten([x.re(regex) for x in self]) + def re_first(self, regex): + for el in iflatten((x.re(regex) for x in self)): + return el + def extract(self): return [x.extract() for x in self] diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 551d337eb..b6100f899 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -27,13 +27,20 @@ def flatten(x): >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]""" - result = [] + return list(iflatten(x)) + + +def iflatten(x): + """iflatten(sequence) -> iterator + + Similar to ``.flatten()``, but returns iterator instead""" + for el in x: if hasattr(el, "__iter__"): - result.extend(flatten(el)) + for el_ in flatten(el): + yield el_ else: - result.append(el) - return result + yield el def unique(list_, key=lambda x: x): diff --git a/tests/test_selector.py b/tests/test_selector.py index 80a9a4672..9b8613319 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -72,6 +72,24 @@ class SelectorTestCase(unittest.TestCase): self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').extract_first(), None) + def test_re_first(self): + """Test if re_first() returns first matched element""" + body = '
  • 1
  • 2
' + response = TextResponse(url="http://example.com", body=body) + sel = self.sscls(response) + + self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'), + sel.xpath('//ul/li/text()').re('\d')[0]) + + self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'), + sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0]) + + self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'), + sel.xpath('//ul/li/text()').re('\d')[1]) + + self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None) + self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None) + def test_select_unicode_query(self): body = u"

" response = TextResponse(url="http://example.com", body=body, encoding='utf8') From 0dade7315bc59ce6b2cfacdd17895985882f5ae1 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 18 Mar 2015 20:50:17 -0300 Subject: [PATCH 7/8] Use generator sintax in re_first --- scrapy/selector/unified.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 889c349e3..c0eefb85e 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -176,7 +176,7 @@ class SelectorList(list): return flatten([x.re(regex) for x in self]) def re_first(self, regex): - for el in iflatten((x.re(regex) for x in self)): + for el in iflatten(x.re(regex) for x in self): return el def extract(self): From 959aaad20554f1ad89704229594d2efb7a835bd3 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Wed, 18 Mar 2015 21:04:15 -0300 Subject: [PATCH 8/8] Document `re_first` --- docs/topics/selectors.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 92e092246..33958cee5 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -236,6 +236,12 @@ Here's an example used to extract images names from the :ref:`HTML code u'My image 4', u'My image 5'] +There's an additional helper reciprocating ``.extract_first()`` for ``.re()``, +named ``.re_first()``. Use it to extract just the first matching string:: + + >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') + u'My image 1' + .. _topics-selectors-relative-xpaths: Working with relative XPaths